2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2014-10-01 23:19:16 +00:00
|
|
|
|
2017-04-06 02:02:00 +00:00
|
|
|
#include "options/cf_options.h"
|
2015-09-11 21:31:23 +00:00
|
|
|
|
2014-10-01 23:19:16 +00:00
|
|
|
#include <cassert>
|
2019-09-20 19:00:55 +00:00
|
|
|
#include <cinttypes>
|
2016-09-14 04:11:59 +00:00
|
|
|
#include <limits>
|
2014-10-17 00:22:28 +00:00
|
|
|
#include <string>
|
2020-04-03 17:48:46 +00:00
|
|
|
|
2021-09-29 11:01:57 +00:00
|
|
|
#include "logging/logging.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "options/configurable_helper.h"
|
2017-04-06 02:02:00 +00:00
|
|
|
#include "options/db_options.h"
|
2020-04-03 17:48:46 +00:00
|
|
|
#include "options/options_helper.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "options/options_parser.h"
|
2015-07-11 16:54:33 +00:00
|
|
|
#include "port/port.h"
|
2023-02-09 20:12:02 +00:00
|
|
|
#include "rocksdb/advanced_cache.h"
|
2021-08-06 15:26:23 +00:00
|
|
|
#include "rocksdb/compaction_filter.h"
|
2019-09-20 19:00:55 +00:00
|
|
|
#include "rocksdb/concurrent_task_limiter.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "rocksdb/configurable.h"
|
2020-04-03 17:48:46 +00:00
|
|
|
#include "rocksdb/convenience.h"
|
2014-10-17 00:22:28 +00:00
|
|
|
#include "rocksdb/env.h"
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
2019-12-13 22:47:08 +00:00
|
|
|
#include "rocksdb/file_system.h"
|
2020-04-03 17:48:46 +00:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-10-01 23:19:16 +00:00
|
|
|
#include "rocksdb/options.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "rocksdb/table.h"
|
2020-04-03 17:48:46 +00:00
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "rocksdb/utilities/options_type.h"
|
2020-04-03 17:48:46 +00:00
|
|
|
#include "util/cast_util.h"
|
2014-10-01 23:19:16 +00:00
|
|
|
|
2022-02-04 13:31:07 +00:00
|
|
|
// NOTE: in this file, many option flags that were deprecated
|
|
|
|
|
// and removed from the rest of the code have to be kept here
|
|
|
|
|
// and marked as kDeprecated in order to be able to read old
|
|
|
|
|
// OPTIONS files.
|
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2020-04-03 17:48:46 +00:00
|
|
|
|
2020-04-29 01:02:11 +00:00
|
|
|
static Status ParseCompressionOptions(const std::string& value,
|
|
|
|
|
const std::string& name,
|
|
|
|
|
CompressionOptions& compression_opts) {
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
const char kDelimiter = ':';
|
|
|
|
|
std::istringstream field_stream(value);
|
|
|
|
|
std::string field;
|
|
|
|
|
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.window_bits = ParseInt(field);
|
|
|
|
|
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.level = ParseInt(field);
|
|
|
|
|
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.strategy = ParseInt(field);
|
|
|
|
|
|
2020-04-29 01:02:11 +00:00
|
|
|
// max_dict_bytes is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.max_dict_bytes = ParseInt(field);
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
|
2020-04-29 01:02:11 +00:00
|
|
|
// zstd_max_train_bytes is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.zstd_max_train_bytes = ParseInt(field);
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
2020-07-17 02:04:44 +00:00
|
|
|
|
|
|
|
|
// parallel_threads is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-07-17 02:04:44 +00:00
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
|
|
|
|
// Since parallel_threads comes before enabled but was added optionally
|
|
|
|
|
// later, we need to check if this is the final token (meaning it is the
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
// enabled bit), or if there are more tokens (meaning this one is
|
|
|
|
|
// parallel_threads).
|
|
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
compression_opts.parallel_threads = ParseInt(field);
|
2020-07-17 02:04:44 +00:00
|
|
|
} else {
|
|
|
|
|
// parallel_threads is not serialized with this format, but enabled is
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.enabled = ParseBoolean("", field);
|
2020-07-17 02:04:44 +00:00
|
|
|
}
|
|
|
|
|
}
|
2020-04-30 23:59:16 +00:00
|
|
|
|
2020-04-29 01:02:11 +00:00
|
|
|
// enabled is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
|
|
|
|
compression_opts.enabled = ParseBoolean("", field);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// max_dict_buffer_bytes is optional for backwards compatibility
|
|
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
compression_opts.max_dict_buffer_bytes = ParseUint64(field);
|
|
|
|
|
}
|
|
|
|
|
|
Support using ZDICT_finalizeDictionary to generate zstd dictionary (#9857)
Summary:
An untrained dictionary is currently simply the concatenation of several samples. The ZSTD API, ZDICT_finalizeDictionary(), can improve such a dictionary's effectiveness at low cost. This PR changes how dictionary is created by calling the ZSTD ZDICT_finalizeDictionary() API instead of creating raw content dictionary (when max_dict_buffer_bytes > 0), and pass in all buffered uncompressed data blocks as samples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9857
Test Plan:
#### db_bench test for cpu/memory of compression+decompression and space saving on synthetic data:
Set up: change the parameter [here](https://github.com/facebook/rocksdb/blob/fb9a167a55e0970b1ef6f67c1600c8d9c4c6114f/tools/db_bench_tool.cc#L1766) to 16384 to make synthetic data more compressible.
```
# linked local ZSTD with version 1.5.2
# DEBUG_LEVEL=0 ROCKSDB_NO_FBCODE=1 ROCKSDB_DISABLE_ZSTD=1 EXTRA_CXXFLAGS="-DZSTD_STATIC_LINKING_ONLY -DZSTD -I/data/users/changyubi/install/include/" EXTRA_LDFLAGS="-L/data/users/changyubi/install/lib/ -l:libzstd.a" make -j32 db_bench
dict_bytes=16384
train_bytes=1048576
echo "========== No Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== Raw Content Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench_main -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench_main -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== FinalizeDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== TrainDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
# Result: TrainDictionary is much better on space saving, but FinalizeDictionary seems to use less memory.
# before compression data size: 1.2GB
dict_bytes=16384
max_dict_buffer_bytes = 1048576
space cpu/memory
No Dictionary 468M 14.93user 1.00system 0:15.92elapsed 100%CPU (0avgtext+0avgdata 23904maxresident)k
Raw Dictionary 251M 15.81user 0.80system 0:16.56elapsed 100%CPU (0avgtext+0avgdata 156808maxresident)k
FinalizeDictionary 236M 11.93user 0.64system 0:12.56elapsed 100%CPU (0avgtext+0avgdata 89548maxresident)k
TrainDictionary 84M 7.29user 0.45system 0:07.75elapsed 100%CPU (0avgtext+0avgdata 97288maxresident)k
```
#### Benchmark on 10 sample SST files for spacing saving and CPU time on compression:
FinalizeDictionary is comparable to TrainDictionary in terms of space saving, and takes less time in compression.
```
dict_bytes=16384
train_bytes=1048576
for sst_file in `ls ../temp/myrock-sst/`
do
echo "********** $sst_file **********"
echo "========== No Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD
echo "========== Raw Content Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes
echo "========== FinalizeDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes --compression_use_zstd_finalize_dict
echo "========== TrainDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes
done
010240.sst (Size/Time) 011029.sst 013184.sst 021552.sst 185054.sst 185137.sst 191666.sst 7560381.sst 7604174.sst 7635312.sst
No Dictionary 28165569 / 2614419 32899411 / 2976832 32977848 / 3055542 31966329 / 2004590 33614351 / 1755877 33429029 / 1717042 33611933 / 1776936 33634045 / 2771417 33789721 / 2205414 33592194 / 388254
Raw Content Dictionary 28019950 / 2697961 33748665 / 3572422 33896373 / 3534701 26418431 / 2259658 28560825 / 1839168 28455030 / 1846039 28494319 / 1861349 32391599 / 3095649 33772142 / 2407843 33592230 / 474523
FinalizeDictionary 27896012 / 2650029 33763886 / 3719427 33904283 / 3552793 26008225 / 2198033 28111872 / 1869530 28014374 / 1789771 28047706 / 1848300 32296254 / 3204027 33698698 / 2381468 33592344 / 517433
TrainDictionary 28046089 / 2740037 33706480 / 3679019 33885741 / 3629351 25087123 / 2204558 27194353 / 1970207 27234229 / 1896811 27166710 / 1903119 32011041 / 3322315 32730692 / 2406146 33608631 / 570593
```
#### Decompression/Read test:
With FinalizeDictionary/TrainDictionary, some data structure used for decompression are in stored in dictionary, so they are expected to be faster in terms of decompression/reads.
```
dict_bytes=16384
train_bytes=1048576
echo "No Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=0 > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=0 2>&1 | grep MB/s
echo "Raw Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes 2>&1 | grep MB/s
echo "FinalizeDict"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false 2>&1 | grep MB/s
echo "Train Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes 2>&1 | grep MB/s
No Dictionary
readrandom : 12.183 micros/op 82082 ops/sec 12.183 seconds 1000000 operations; 9.1 MB/s (1000000 of 1000000 found)
Raw Dictionary
readrandom : 12.314 micros/op 81205 ops/sec 12.314 seconds 1000000 operations; 9.0 MB/s (1000000 of 1000000 found)
FinalizeDict
readrandom : 9.787 micros/op 102180 ops/sec 9.787 seconds 1000000 operations; 11.3 MB/s (1000000 of 1000000 found)
Train Dictionary
readrandom : 9.698 micros/op 103108 ops/sec 9.699 seconds 1000000 operations; 11.4 MB/s (1000000 of 1000000 found)
```
Reviewed By: ajkr
Differential Revision: D35720026
Pulled By: cbi42
fbshipit-source-id: 24d230fdff0fd28a1bb650658798f00dfcfb2a1f
2022-05-20 19:09:09 +00:00
|
|
|
// use_zstd_dict_trainer is optional for backwards compatibility
|
|
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
|
|
|
|
compression_opts.use_zstd_dict_trainer = ParseBoolean("", field);
|
|
|
|
|
}
|
|
|
|
|
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
|
|
|
|
return Status::OK();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::string kOptNameBMCompOpts = "bottommost_compression_opts";
|
|
|
|
|
const std::string kOptNameCompOpts = "compression_opts";
|
2020-04-03 17:48:46 +00:00
|
|
|
|
2020-07-17 02:04:44 +00:00
|
|
|
// OptionTypeInfo map for CompressionOptions
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
compression_options_type_info = {
|
|
|
|
|
{"window_bits",
|
|
|
|
|
{offsetof(struct CompressionOptions, window_bits), OptionType::kInt,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
{"level",
|
|
|
|
|
{offsetof(struct CompressionOptions, level), OptionType::kInt,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
{"strategy",
|
|
|
|
|
{offsetof(struct CompressionOptions, strategy), OptionType::kInt,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
2023-04-22 04:57:40 +00:00
|
|
|
{"max_compressed_bytes_per_kb",
|
|
|
|
|
{offsetof(struct CompressionOptions, max_compressed_bytes_per_kb),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
{"max_dict_bytes",
|
|
|
|
|
{offsetof(struct CompressionOptions, max_dict_bytes), OptionType::kInt,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
{"zstd_max_train_bytes",
|
|
|
|
|
{offsetof(struct CompressionOptions, zstd_max_train_bytes),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
{"parallel_threads",
|
|
|
|
|
{offsetof(struct CompressionOptions, parallel_threads),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
{"enabled",
|
|
|
|
|
{offsetof(struct CompressionOptions, enabled), OptionType::kBoolean,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
|
|
|
{"max_dict_buffer_bytes",
|
|
|
|
|
{offsetof(struct CompressionOptions, max_dict_buffer_bytes),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Support using ZDICT_finalizeDictionary to generate zstd dictionary (#9857)
Summary:
An untrained dictionary is currently simply the concatenation of several samples. The ZSTD API, ZDICT_finalizeDictionary(), can improve such a dictionary's effectiveness at low cost. This PR changes how dictionary is created by calling the ZSTD ZDICT_finalizeDictionary() API instead of creating raw content dictionary (when max_dict_buffer_bytes > 0), and pass in all buffered uncompressed data blocks as samples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9857
Test Plan:
#### db_bench test for cpu/memory of compression+decompression and space saving on synthetic data:
Set up: change the parameter [here](https://github.com/facebook/rocksdb/blob/fb9a167a55e0970b1ef6f67c1600c8d9c4c6114f/tools/db_bench_tool.cc#L1766) to 16384 to make synthetic data more compressible.
```
# linked local ZSTD with version 1.5.2
# DEBUG_LEVEL=0 ROCKSDB_NO_FBCODE=1 ROCKSDB_DISABLE_ZSTD=1 EXTRA_CXXFLAGS="-DZSTD_STATIC_LINKING_ONLY -DZSTD -I/data/users/changyubi/install/include/" EXTRA_LDFLAGS="-L/data/users/changyubi/install/lib/ -l:libzstd.a" make -j32 db_bench
dict_bytes=16384
train_bytes=1048576
echo "========== No Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== Raw Content Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench_main -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench_main -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== FinalizeDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== TrainDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
# Result: TrainDictionary is much better on space saving, but FinalizeDictionary seems to use less memory.
# before compression data size: 1.2GB
dict_bytes=16384
max_dict_buffer_bytes = 1048576
space cpu/memory
No Dictionary 468M 14.93user 1.00system 0:15.92elapsed 100%CPU (0avgtext+0avgdata 23904maxresident)k
Raw Dictionary 251M 15.81user 0.80system 0:16.56elapsed 100%CPU (0avgtext+0avgdata 156808maxresident)k
FinalizeDictionary 236M 11.93user 0.64system 0:12.56elapsed 100%CPU (0avgtext+0avgdata 89548maxresident)k
TrainDictionary 84M 7.29user 0.45system 0:07.75elapsed 100%CPU (0avgtext+0avgdata 97288maxresident)k
```
#### Benchmark on 10 sample SST files for spacing saving and CPU time on compression:
FinalizeDictionary is comparable to TrainDictionary in terms of space saving, and takes less time in compression.
```
dict_bytes=16384
train_bytes=1048576
for sst_file in `ls ../temp/myrock-sst/`
do
echo "********** $sst_file **********"
echo "========== No Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD
echo "========== Raw Content Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes
echo "========== FinalizeDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes --compression_use_zstd_finalize_dict
echo "========== TrainDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes
done
010240.sst (Size/Time) 011029.sst 013184.sst 021552.sst 185054.sst 185137.sst 191666.sst 7560381.sst 7604174.sst 7635312.sst
No Dictionary 28165569 / 2614419 32899411 / 2976832 32977848 / 3055542 31966329 / 2004590 33614351 / 1755877 33429029 / 1717042 33611933 / 1776936 33634045 / 2771417 33789721 / 2205414 33592194 / 388254
Raw Content Dictionary 28019950 / 2697961 33748665 / 3572422 33896373 / 3534701 26418431 / 2259658 28560825 / 1839168 28455030 / 1846039 28494319 / 1861349 32391599 / 3095649 33772142 / 2407843 33592230 / 474523
FinalizeDictionary 27896012 / 2650029 33763886 / 3719427 33904283 / 3552793 26008225 / 2198033 28111872 / 1869530 28014374 / 1789771 28047706 / 1848300 32296254 / 3204027 33698698 / 2381468 33592344 / 517433
TrainDictionary 28046089 / 2740037 33706480 / 3679019 33885741 / 3629351 25087123 / 2204558 27194353 / 1970207 27234229 / 1896811 27166710 / 1903119 32011041 / 3322315 32730692 / 2406146 33608631 / 570593
```
#### Decompression/Read test:
With FinalizeDictionary/TrainDictionary, some data structure used for decompression are in stored in dictionary, so they are expected to be faster in terms of decompression/reads.
```
dict_bytes=16384
train_bytes=1048576
echo "No Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=0 > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=0 2>&1 | grep MB/s
echo "Raw Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes 2>&1 | grep MB/s
echo "FinalizeDict"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false 2>&1 | grep MB/s
echo "Train Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes 2>&1 | grep MB/s
No Dictionary
readrandom : 12.183 micros/op 82082 ops/sec 12.183 seconds 1000000 operations; 9.1 MB/s (1000000 of 1000000 found)
Raw Dictionary
readrandom : 12.314 micros/op 81205 ops/sec 12.314 seconds 1000000 operations; 9.0 MB/s (1000000 of 1000000 found)
FinalizeDict
readrandom : 9.787 micros/op 102180 ops/sec 9.787 seconds 1000000 operations; 11.3 MB/s (1000000 of 1000000 found)
Train Dictionary
readrandom : 9.698 micros/op 103108 ops/sec 9.699 seconds 1000000 operations; 11.4 MB/s (1000000 of 1000000 found)
```
Reviewed By: ajkr
Differential Revision: D35720026
Pulled By: cbi42
fbshipit-source-id: 24d230fdff0fd28a1bb650658798f00dfcfb2a1f
2022-05-20 19:09:09 +00:00
|
|
|
{"use_zstd_dict_trainer",
|
|
|
|
|
{offsetof(struct CompressionOptions, use_zstd_dict_trainer),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Add `CompressionOptions::checksum` for enabling ZSTD checksum (#11666)
Summary:
Optionally enable zstd checksum flag (https://github.com/facebook/zstd/blob/d857369028d997c92ff1f1861a4d7f679a125464/lib/zstd.h#L428) to detect corruption during decompression. Main changes are in compression.h:
* User can set CompressionOptions::checksum to true to enable this feature.
* We enable this feature in ZSTD by setting the checksum flag in ZSTD compression context: `ZSTD_CCtx`.
* Uses `ZSTD_compress2()` to do compression since it supports frame parameter like the checksum flag. Compression level is also set in compression context as a flag.
* Error handling during decompression to propagate error message from ZSTD.
* Updated microbench to test read performance impact.
About compatibility, the current compression decoders should continue to work with the data created by the new compression API `ZSTD_compress2()`: https://github.com/facebook/zstd/issues/3711.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11666
Test Plan:
* Existing unit tests for zstd compression
* Add unit test `DBTest2.ZSTDChecksum` to test the corruption case
* Manually tested that compression levels, parallel compression, dictionary compression, index compression all work with the new ZSTD_compress2() API.
* Manually tested with `sst_dump --command=recompress` that different compression levels and dictionary compression settings all work.
* Manually tested compiling with older versions of ZSTD: v1.3.8, v1.1.0, v0.6.2.
* Perf impact: from public benchmark data: http://fastcompression.blogspot.com/2019/03/presenting-xxh3.html for checksum and https://github.com/facebook/zstd#benchmarks, if decompression is 1700MB/s and checksum computation is 70000MB/s, checksum computation is an additional ~2.4% time for decompression. Compression is slower and checksumming should be less noticeable.
* Microbench:
```
TEST_TMPDIR=/dev/shm ./branch_db_basic_bench --benchmark_filter=DBGet/comp_style:0/max_data:1048576/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/mmap:0/compression_type:7/compression_checksum:1/no_blockcache:1/iterations:10000/threads:1 --benchmark_repetitions=100
Min out of 100 runs:
Main:
10390 10436 10456 10484 10499 10535 10544 10545 10565 10568
After this PR, checksum=false
10285 10397 10503 10508 10515 10557 10562 10635 10640 10660
After this PR, checksum=true
10827 10876 10925 10949 10971 11052 11061 11063 11100 11109
```
* db_bench:
```
Write perf
TEST_TMPDIR=/dev/shm/ ./db_bench_ichecksum --benchmarks=fillseq[-X10] --compression_type=zstd --num=10000000 --compression_checksum=..
[FillSeq checksum=0]
fillseq [AVG 10 runs] : 281635 (± 31711) ops/sec; 31.2 (± 3.5) MB/sec
fillseq [MEDIAN 10 runs] : 294027 ops/sec; 32.5 MB/sec
[FillSeq checksum=1]
fillseq [AVG 10 runs] : 286961 (± 34700) ops/sec; 31.7 (± 3.8) MB/sec
fillseq [MEDIAN 10 runs] : 283278 ops/sec; 31.3 MB/sec
Read perf
TEST_TMPDIR=/dev/shm ./db_bench_ichecksum --benchmarks=readrandom[-X20] --num=100000000 --reads=1000000 --use_existing_db=true --readonly=1
[Readrandom checksum=1]
readrandom [AVG 20 runs] : 360928 (± 3579) ops/sec; 4.0 (± 0.0) MB/sec
readrandom [MEDIAN 20 runs] : 362468 ops/sec; 4.0 MB/sec
[Readrandom checksum=0]
readrandom [AVG 20 runs] : 380365 (± 2384) ops/sec; 4.2 (± 0.0) MB/sec
readrandom [MEDIAN 20 runs] : 379800 ops/sec; 4.2 MB/sec
Compression
TEST_TMPDIR=/dev/shm ./db_bench_ichecksum --benchmarks=compress[-X20] --compression_type=zstd --num=100000000 --compression_checksum=1
checksum=1
compress [AVG 20 runs] : 54074 (± 634) ops/sec; 211.2 (± 2.5) MB/sec
compress [MEDIAN 20 runs] : 54396 ops/sec; 212.5 MB/sec
checksum=0
compress [AVG 20 runs] : 54598 (± 393) ops/sec; 213.3 (± 1.5) MB/sec
compress [MEDIAN 20 runs] : 54592 ops/sec; 213.3 MB/sec
Decompression:
TEST_TMPDIR=/dev/shm ./db_bench_ichecksum --benchmarks=uncompress[-X20] --compression_type=zstd --compression_checksum=1
checksum = 0
uncompress [AVG 20 runs] : 167499 (± 962) ops/sec; 654.3 (± 3.8) MB/sec
uncompress [MEDIAN 20 runs] : 167210 ops/sec; 653.2 MB/sec
checksum = 1
uncompress [AVG 20 runs] : 167980 (± 924) ops/sec; 656.2 (± 3.6) MB/sec
uncompress [MEDIAN 20 runs] : 168465 ops/sec; 658.1 MB/sec
```
Reviewed By: ajkr
Differential Revision: D48019378
Pulled By: cbi42
fbshipit-source-id: 674120c6e1853c2ced1436ac8138559d0204feba
2023-08-18 22:01:59 +00:00
|
|
|
{"checksum",
|
|
|
|
|
{offsetof(struct CompressionOptions, checksum), OptionType::kBoolean,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
2020-07-17 02:04:44 +00:00
|
|
|
};
|
|
|
|
|
|
2023-05-11 23:40:59 +00:00
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
file_temperature_age_type_info = {
|
|
|
|
|
{"temperature",
|
|
|
|
|
{offsetof(struct FileTemperatureAge, temperature),
|
|
|
|
|
OptionType::kTemperature, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"age",
|
|
|
|
|
{offsetof(struct FileTemperatureAge, age), OptionType::kUInt64T,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
};
|
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
fifo_compaction_options_type_info = {
|
|
|
|
|
{"max_table_files_size",
|
|
|
|
|
{offsetof(struct CompactionOptionsFIFO, max_table_files_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2021-08-09 19:50:19 +00:00
|
|
|
{"age_for_warm",
|
|
|
|
|
{offsetof(struct CompactionOptionsFIFO, age_for_warm),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"ttl",
|
|
|
|
|
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kNone}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"allow_compaction",
|
|
|
|
|
{offsetof(struct CompactionOptionsFIFO, allow_compaction),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2023-05-11 23:40:59 +00:00
|
|
|
{"file_temperature_age_thresholds",
|
|
|
|
|
OptionTypeInfo::Vector<struct FileTemperatureAge>(
|
|
|
|
|
offsetof(struct CompactionOptionsFIFO,
|
|
|
|
|
file_temperature_age_thresholds),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
|
|
|
|
|
OptionTypeInfo::Struct("file_temperature_age_thresholds",
|
|
|
|
|
&file_temperature_age_type_info, 0,
|
|
|
|
|
OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable))}};
|
2020-05-21 17:56:40 +00:00
|
|
|
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
universal_compaction_options_type_info = {
|
|
|
|
|
{"size_ratio",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, size_ratio),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"min_merge_width",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, min_merge_width),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"max_merge_width",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, max_merge_width),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"max_size_amplification_percent",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal,
|
|
|
|
|
max_size_amplification_percent),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"compression_size_percent",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, compression_size_percent),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
Improve universal compaction sorted-run trigger (#12477)
Summary:
Universal compaction currently uses `level0_file_num_compaction_trigger` for two purposes:
1. the trigger for checking if there is any compaction to do, and
2. the limit on the number of sorted runs. RocksDB will do compaction to keep the number of sorted runs no more than the value of this option.
This can make the option inflexible. A value that is too small causes higher write amp: more compactions to reduce the number of sorted runs. A value that is too big delays potential compaction work and causes worse read performance. This PR introduce an option `CompactionOptionsUniversal::max_read_amp` for only the second purpose: to specify
the hard limit on the number of sorted runs.
For backward compatibility, `max_read_amp = -1` by default, which means to fallback to the current behavior.
When `max_read_amp > 0`,`level0_file_num_compaction_trigger` will only serve as a trigger to find potential compaction.
When `max_read_amp = 0`, RocksDB will auto-tune the limit on the number of sorted runs. The estimation is based on DB size, write_buffer_size and size_ratio, so it is adaptive to the size change of the DB. See more in `UniversalCompactionBuilder::PickCompaction()`.
Alternatively, users now can configure `max_read_amp` to a very big value and keep `level0_file_num_compaction_trigger` small. This will allow `size_ratio` and `max_size_amplification_percent` to control the number of sorted runs. This essentially disables compactions with reason kUniversalSortedRunNum.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12477
Test Plan:
* new unit test
* existing unit test for default behavior
* updated crash test with the new option
* benchmark:
* Create a DB that is roughly 24GB in the last level. When `max_read_amp = 0`, we estimate that the DB needs 9 levels to avoid excessive compactions to reduce the number of sorted runs.
* We then run fillrandom to ingest another 24GB data to compare write amp.
* case 1: small level0 trigger: `level0_file_num_compaction_trigger=5, max_read_amp=-1`
* write-amp: 4.8
* case 2: auto-tune: `level0_file_num_compaction_trigger=5, max_read_amp=0`
* write-amp: 3.6
* case 3: auto-tune with minimal trigger: `level0_file_num_compaction_trigger=1, max_read_amp=0`
* write-amp: 3.8
* case 4: hard-code a good value for trigger: `level0_file_num_compaction_trigger=9`
* write-amp: 2.8
```
Case 1:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 0/0 0.00 KB 1.0 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 163.2 141.94 111.10 108 1.314 0 0 0.0 0.0
L45 8/0 1.81 GB 0.0 39.6 11.1 28.5 39.3 10.8 0.0 3.5 209.0 207.3 194.25 191.29 43 4.517 348M 2498K 0.0 0.0
L46 13/0 3.12 GB 0.0 15.3 9.5 5.8 15.0 9.3 0.0 1.6 203.1 199.3 77.13 75.88 16 4.821 134M 2362K 0.0 0.0
L47 19/0 4.68 GB 0.0 15.4 10.5 4.9 14.7 9.8 0.0 1.4 204.0 194.9 77.38 76.15 8 9.673 135M 5920K 0.0 0.0
L48 38/0 9.42 GB 0.0 19.6 11.7 7.9 17.3 9.4 0.0 1.5 206.5 182.3 97.15 95.02 4 24.287 172M 20M 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 169/0 41.74 GB 0.0 89.9 42.9 47.0 109.0 61.9 0.0 4.8 156.7 189.8 587.85 549.45 179 3.284 791M 31M 0.0 0.0
Case 2:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 1/0 214.47 MB 1.2 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 164.5 140.81 109.98 108 1.304 0 0 0.0 0.0
L44 0/0 0.00 KB 0.0 1.3 1.3 0.0 1.2 1.2 0.0 1.0 206.1 204.9 6.24 5.98 3 2.081 11M 51K 0.0 0.0
L45 4/0 844.36 MB 0.0 7.1 5.4 1.7 7.0 5.4 0.0 1.3 194.6 192.9 37.41 36.00 13 2.878 62M 489K 0.0 0.0
L46 11/0 2.57 GB 0.0 14.6 9.8 4.8 14.3 9.5 0.0 1.5 193.7 189.8 77.09 73.54 17 4.535 128M 2411K 0.0 0.0
L47 24/0 5.81 GB 0.0 19.8 12.0 7.8 18.8 11.0 0.0 1.6 191.4 181.1 106.19 101.21 9 11.799 174M 9166K 0.0 0.0
L48 38/0 9.42 GB 0.0 19.6 11.8 7.9 17.3 9.4 0.0 1.5 197.3 173.6 101.97 97.23 4 25.491 172M 20M 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 169/0 41.54 GB 0.0 62.4 40.3 22.1 81.3 59.2 0.0 3.6 136.1 177.2 469.71 423.94 154 3.050 549M 32M 0.0 0.0
Case 3:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 0/0 0.00 KB 5.0 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 163.8 141.43 111.13 108 1.310 0 0 0.0 0.0
L44 0/0 0.00 KB 0.0 0.8 0.8 0.0 0.8 0.8 0.0 1.0 201.4 200.2 4.26 4.19 2 2.130 7360K 33K 0.0 0.0
L45 4/0 844.38 MB 0.0 6.3 5.0 1.2 6.2 5.0 0.0 1.2 202.0 200.3 31.81 31.50 12 2.651 55M 403K 0.0 0.0
L46 7/0 1.62 GB 0.0 13.3 8.8 4.6 13.1 8.6 0.0 1.5 198.9 195.7 68.72 67.89 17 4.042 117M 1696K 0.0 0.0
L47 24/0 5.81 GB 0.0 21.7 12.9 8.8 20.6 11.8 0.0 1.6 198.5 188.6 112.04 109.97 12 9.336 191M 9352K 0.0 0.0
L48 41/0 10.14 GB 0.0 24.8 13.0 11.8 21.9 10.1 0.0 1.7 198.6 175.6 127.88 125.36 6 21.313 218M 25M 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 167/0 41.10 GB 0.0 67.0 40.5 26.4 85.4 58.9 0.0 3.8 141.1 179.8 486.13 450.04 157 3.096 589M 36M 0.0 0.0
Case 4:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 0/0 0.00 KB 0.7 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 158.6 146.02 114.68 108 1.352 0 0 0.0 0.0
L42 0/0 0.00 KB 0.0 1.7 1.7 0.0 1.7 1.7 0.0 1.0 185.4 184.3 9.25 8.96 4 2.314 14M 67K 0.0 0.0
L43 0/0 0.00 KB 0.0 2.5 2.5 0.0 2.5 2.5 0.0 1.0 197.8 195.6 13.01 12.65 4 3.253 22M 202K 0.0 0.0
L44 4/0 844.40 MB 0.0 4.2 4.2 0.0 4.1 4.1 0.0 1.0 188.1 185.1 22.81 21.89 5 4.562 36M 503K 0.0 0.0
L45 13/0 3.12 GB 0.0 7.5 6.5 1.0 7.2 6.2 0.0 1.1 188.7 181.8 40.69 39.32 5 8.138 65M 2282K 0.0 0.0
L46 17/0 4.18 GB 0.0 8.3 7.1 1.2 7.9 6.6 0.0 1.1 192.2 181.8 44.23 43.06 4 11.058 73M 3846K 0.0 0.0
L47 22/0 5.34 GB 0.0 8.9 7.5 1.4 8.2 6.8 0.0 1.1 189.1 174.1 48.12 45.37 3 16.041 78M 6098K 0.0 0.0
L48 27/0 6.58 GB 0.0 9.2 7.6 1.6 8.2 6.6 0.0 1.1 195.2 172.9 48.52 47.11 2 24.262 81M 9217K 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 174/0 42.74 GB 0.0 42.3 37.0 5.3 62.4 57.1 0.0 2.8 116.3 171.3 372.66 333.04 135 2.760 372M 22M 0.0 0.0
setup:
./db_bench --benchmarks=fillseq,compactall,waitforcompaction --num=200000000 --compression_type=none --disable_wal=1 --compaction_style=1 --num_levels=50 --target_file_size_base=268435456 --max_compaction_bytes=6710886400 --level0_file_num_compaction_trigger=10 --write_buffer_size=268435456 --seed 1708494134896523
benchmark:
./db_bench --benchmarks=overwrite,waitforcompaction,stats --num=200000000 --compression_type=none --disable_wal=1 --compaction_style=1 --write_buffer_size=268435456 --level0_file_num_compaction_trigger=5 --target_file_size_base=268435456 --use_existing_db=1 --num_levels=50 --writes=200000000 --universal_max_read_amp=-1 --seed=1716488324800233
```
Reviewed By: ajkr
Differential Revision: D55370922
Pulled By: cbi42
fbshipit-source-id: 9be69979126b840d08e93e7059260e76a878bb2a
2024-05-24 17:10:31 +00:00
|
|
|
{"max_read_amp",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, max_read_amp),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"stop_style",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, stop_style),
|
|
|
|
|
OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
Incremental Space Amp Compactions in Universal Style (#8655)
Summary:
This commit introduces incremental compaction in univeral style for space amplification. This follows the first improvement mentioned in https://rocksdb.org/blog/2021/04/12/universal-improvements.html . The implemention simply picks up files about size of max_compaction_bytes to compact and execute if the penalty is not too big. More optimizations can be done in the future, e.g. prioritizing between this compaction and other types. But for now, the feature is supposed to be functional and can often reduce frequency of full compactions, although it can introduce penalty.
In order to add cut files more efficiently so that more files from upper levels can be included, SST file cutting threshold (for current file + overlapping parent level files) is set to 1.5X of target file size. A 2MB target file size will generate files like this: https://gist.github.com/siying/29d2676fba417404f3c95e6c013c7de8 Number of files indeed increases but it is not out of control.
Two set of write benchmarks are run:
1. For ingestion rate limited scenario, we can see full compaction is mostly eliminated: https://gist.github.com/siying/959bc1186066906831cf4c808d6e0a19 . The write amp increased from 7.7 to 9.4, as expected. After applying file cutting, the number is improved to 8.9. In another benchmark, the write amp is even better with the incremental approach: https://gist.github.com/siying/d1c16c286d7c59c4d7bba718ca198163
2. For ingestion rate unlimited scenario, incremental compaction turns out to be too expensive most of the time and is not executed, as expected.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8655
Test Plan: Add unit tests to the functionality.
Reviewed By: ajkr
Differential Revision: D31787034
fbshipit-source-id: ce813e63b15a61d5a56e97bf8902a1b28e011beb
2021-10-20 17:03:03 +00:00
|
|
|
{"incremental",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, incremental),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-05-21 17:56:40 +00:00
|
|
|
{"allow_trivial_move",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, allow_trivial_move),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}}};
|
2020-05-21 17:56:40 +00:00
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
cf_mutable_options_type_info = {
|
2020-04-03 17:48:46 +00:00
|
|
|
{"report_bg_io_stats",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, report_bg_io_stats),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"disable_auto_compactions",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, disable_auto_compactions),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2024-10-17 21:13:20 +00:00
|
|
|
{"table_factory", OptionTypeInfo::AsCustomSharedPtr<TableFactory>(
|
|
|
|
|
offsetof(struct MutableCFOptions, table_factory),
|
|
|
|
|
OptionVerificationType::kByName,
|
|
|
|
|
(OptionTypeFlags::kCompareLoose |
|
|
|
|
|
OptionTypeFlags::kStringNameOnly |
|
|
|
|
|
OptionTypeFlags::kDontPrepare))},
|
|
|
|
|
{"block_based_table_factory",
|
|
|
|
|
{offsetof(struct MutableCFOptions, table_factory),
|
|
|
|
|
OptionType::kCustomizable, OptionVerificationType::kAlias,
|
|
|
|
|
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
|
|
|
|
|
// Parses the input value and creates a BlockBasedTableFactory
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
BlockBasedTableOptions* old_opts = nullptr;
|
|
|
|
|
auto table_factory =
|
|
|
|
|
static_cast<std::shared_ptr<TableFactory>*>(addr);
|
|
|
|
|
if (table_factory->get() != nullptr) {
|
|
|
|
|
old_opts =
|
|
|
|
|
table_factory->get()->GetOptions<BlockBasedTableOptions>();
|
|
|
|
|
}
|
|
|
|
|
if (name == "block_based_table_factory") {
|
|
|
|
|
std::unique_ptr<TableFactory> new_factory;
|
|
|
|
|
if (old_opts != nullptr) {
|
|
|
|
|
new_factory.reset(NewBlockBasedTableFactory(*old_opts));
|
|
|
|
|
} else {
|
|
|
|
|
new_factory.reset(NewBlockBasedTableFactory());
|
|
|
|
|
}
|
|
|
|
|
Status s = new_factory->ConfigureFromString(opts, value);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
table_factory->reset(new_factory.release());
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
} else if (old_opts != nullptr) {
|
|
|
|
|
return table_factory->get()->ConfigureOption(opts, name, value);
|
|
|
|
|
} else {
|
|
|
|
|
return Status::NotFound("Mismatched table option: ", name);
|
|
|
|
|
}
|
|
|
|
|
}}},
|
|
|
|
|
{"plain_table_factory",
|
|
|
|
|
{offsetof(struct MutableCFOptions, table_factory),
|
|
|
|
|
OptionType::kCustomizable, OptionVerificationType::kAlias,
|
|
|
|
|
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
|
|
|
|
|
// Parses the input value and creates a PlainTableFactory
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
PlainTableOptions* old_opts = nullptr;
|
|
|
|
|
auto table_factory =
|
|
|
|
|
static_cast<std::shared_ptr<TableFactory>*>(addr);
|
|
|
|
|
if (table_factory->get() != nullptr) {
|
|
|
|
|
old_opts = table_factory->get()->GetOptions<PlainTableOptions>();
|
|
|
|
|
}
|
|
|
|
|
if (name == "plain_table_factory") {
|
|
|
|
|
std::unique_ptr<TableFactory> new_factory;
|
|
|
|
|
if (old_opts != nullptr) {
|
|
|
|
|
new_factory.reset(NewPlainTableFactory(*old_opts));
|
|
|
|
|
} else {
|
|
|
|
|
new_factory.reset(NewPlainTableFactory());
|
|
|
|
|
}
|
|
|
|
|
Status s = new_factory->ConfigureFromString(opts, value);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
table_factory->reset(new_factory.release());
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
} else if (old_opts != nullptr) {
|
|
|
|
|
return table_factory->get()->ConfigureOption(opts, name, value);
|
|
|
|
|
} else {
|
|
|
|
|
return Status::NotFound("Mismatched table option: ", name);
|
|
|
|
|
}
|
|
|
|
|
}}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"filter_deletes",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-10-01 17:08:52 +00:00
|
|
|
{"check_flush_compaction_key_order",
|
2024-02-01 00:30:26 +00:00
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
2020-10-01 17:08:52 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"paranoid_file_checks",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, paranoid_file_checks),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"verify_checksums_in_compaction",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"soft_pending_compaction_bytes_limit",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions,
|
|
|
|
|
soft_pending_compaction_bytes_limit),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"hard_pending_compaction_bytes_limit",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions,
|
|
|
|
|
hard_pending_compaction_bytes_limit),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"hard_rate_limit",
|
|
|
|
|
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"soft_rate_limit",
|
|
|
|
|
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_compaction_bytes",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, max_compaction_bytes),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
Ignore max_compaction_bytes for compaction input that are within output key-range (#10835)
Summary:
When picking compaction input files, we sometimes stop picking a file that is fully included in the output key-range due to hitting max_compaction_bytes. Including these input files can potentially reduce WA at the expense of larger compactions. Larger compaction should be fine as files from input level are usually 10X smaller than files from output level. This PR adds a mutable CF option `ignore_max_compaction_bytes_for_input` that is enabled by default. We can remove this option once we are sure it is safe.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10835
Test Plan:
- CI, a unit test on max_compaction_bytes fails before turning this flag off.
- Benchmark does not show much difference in WA: `./db_bench --benchmarks=fillrandom,waitforcompaction,stats,levelstats -max_background_jobs=12 -num=2000000000 -target_file_size_base=33554432 --write_buffer_size=33554432`
```
main:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 3/0 91.59 MB 0.8 70.9 0.0 70.9 200.8 129.9 0.0 1.5 25.2 71.2 2886.55 2463.45 9725 0.297 1093M 254K 0.0 0.0
L1 9/0 248.03 MB 1.0 392.0 129.8 262.2 391.7 129.5 0.0 3.0 69.0 68.9 5821.71 5536.90 804 7.241 6029M 5814K 0.0 0.0
L2 87/0 2.50 GB 1.0 537.0 128.5 408.5 533.8 125.2 0.7 4.2 69.5 69.1 7912.24 7323.70 4417 1.791 8299M 36M 0.0 0.0
L3 836/0 24.99 GB 1.0 616.9 118.3 498.7 594.5 95.8 5.2 5.0 66.9 64.5 9442.38 8490.28 4204 2.246 9749M 306M 0.0 0.0
L4 2355/0 62.95 GB 0.3 67.3 37.1 30.2 54.2 24.0 38.9 1.5 72.2 58.2 954.37 821.18 917 1.041 1076M 173M 0.0 0.0
Sum 3290/0 90.77 GB 0.0 1684.2 413.7 1270.5 1775.0 504.5 44.9 13.7 63.8 67.3 27017.25 24635.52 20067 1.346 26G 522M 0.0 0.0
Cumulative compaction: 1774.96 GB write, 154.29 MB/s write, 1684.19 GB read, 146.40 MB/s read, 27017.3 seconds
This PR:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 3/0 45.71 MB 0.8 72.9 0.0 72.9 202.8 129.9 0.0 1.6 25.4 70.7 2938.16 2510.36 9741 0.302 1124M 265K 0.0 0.0
L1 8/0 234.54 MB 0.9 384.5 129.8 254.7 384.2 129.6 0.0 3.0 69.0 68.9 5708.08 5424.43 791 7.216 5913M 5753K 0.0 0.0
L2 84/0 2.47 GB 1.0 543.1 128.6 414.5 539.9 125.4 0.7 4.2 69.6 69.2 7989.31 7403.13 4418 1.808 8393M 36M 0.0 0.0
L3 839/0 24.96 GB 1.0 615.6 118.4 497.2 593.2 96.0 5.1 5.0 66.6 64.1 9471.23 8489.31 4193 2.259 9726M 306M 0.0 0.0
L4 2360/0 63.04 GB 0.3 67.6 37.3 30.3 54.4 24.1 38.9 1.5 71.5 57.6 967.30 827.99 907 1.066 1080M 173M 0.0 0.0
Sum 3294/0 90.75 GB 0.0 1683.8 414.2 1269.6 1774.5 504.9 44.8 13.7 63.7 67.1 27074.08 24655.22 20050 1.350 26G 522M 0.0 0.0
Cumulative compaction: 1774.52 GB write, 157.09 MB/s write, 1683.77 GB read, 149.06 MB/s read, 27074.1 seconds
```
Reviewed By: ajkr
Differential Revision: D40518319
Pulled By: cbi42
fbshipit-source-id: f4ea614bc0ebefe007ffaf05bb9aec9a8ca25b60
2022-10-21 17:22:41 +00:00
|
|
|
{"ignore_max_compaction_bytes_for_input",
|
2024-02-03 01:09:42 +00:00
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
Ignore max_compaction_bytes for compaction input that are within output key-range (#10835)
Summary:
When picking compaction input files, we sometimes stop picking a file that is fully included in the output key-range due to hitting max_compaction_bytes. Including these input files can potentially reduce WA at the expense of larger compactions. Larger compaction should be fine as files from input level are usually 10X smaller than files from output level. This PR adds a mutable CF option `ignore_max_compaction_bytes_for_input` that is enabled by default. We can remove this option once we are sure it is safe.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10835
Test Plan:
- CI, a unit test on max_compaction_bytes fails before turning this flag off.
- Benchmark does not show much difference in WA: `./db_bench --benchmarks=fillrandom,waitforcompaction,stats,levelstats -max_background_jobs=12 -num=2000000000 -target_file_size_base=33554432 --write_buffer_size=33554432`
```
main:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 3/0 91.59 MB 0.8 70.9 0.0 70.9 200.8 129.9 0.0 1.5 25.2 71.2 2886.55 2463.45 9725 0.297 1093M 254K 0.0 0.0
L1 9/0 248.03 MB 1.0 392.0 129.8 262.2 391.7 129.5 0.0 3.0 69.0 68.9 5821.71 5536.90 804 7.241 6029M 5814K 0.0 0.0
L2 87/0 2.50 GB 1.0 537.0 128.5 408.5 533.8 125.2 0.7 4.2 69.5 69.1 7912.24 7323.70 4417 1.791 8299M 36M 0.0 0.0
L3 836/0 24.99 GB 1.0 616.9 118.3 498.7 594.5 95.8 5.2 5.0 66.9 64.5 9442.38 8490.28 4204 2.246 9749M 306M 0.0 0.0
L4 2355/0 62.95 GB 0.3 67.3 37.1 30.2 54.2 24.0 38.9 1.5 72.2 58.2 954.37 821.18 917 1.041 1076M 173M 0.0 0.0
Sum 3290/0 90.77 GB 0.0 1684.2 413.7 1270.5 1775.0 504.5 44.9 13.7 63.8 67.3 27017.25 24635.52 20067 1.346 26G 522M 0.0 0.0
Cumulative compaction: 1774.96 GB write, 154.29 MB/s write, 1684.19 GB read, 146.40 MB/s read, 27017.3 seconds
This PR:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 3/0 45.71 MB 0.8 72.9 0.0 72.9 202.8 129.9 0.0 1.6 25.4 70.7 2938.16 2510.36 9741 0.302 1124M 265K 0.0 0.0
L1 8/0 234.54 MB 0.9 384.5 129.8 254.7 384.2 129.6 0.0 3.0 69.0 68.9 5708.08 5424.43 791 7.216 5913M 5753K 0.0 0.0
L2 84/0 2.47 GB 1.0 543.1 128.6 414.5 539.9 125.4 0.7 4.2 69.6 69.2 7989.31 7403.13 4418 1.808 8393M 36M 0.0 0.0
L3 839/0 24.96 GB 1.0 615.6 118.4 497.2 593.2 96.0 5.1 5.0 66.6 64.1 9471.23 8489.31 4193 2.259 9726M 306M 0.0 0.0
L4 2360/0 63.04 GB 0.3 67.6 37.3 30.3 54.4 24.1 38.9 1.5 71.5 57.6 967.30 827.99 907 1.066 1080M 173M 0.0 0.0
Sum 3294/0 90.75 GB 0.0 1683.8 414.2 1269.6 1774.5 504.9 44.8 13.7 63.7 67.1 27074.08 24655.22 20050 1.350 26G 522M 0.0 0.0
Cumulative compaction: 1774.52 GB write, 157.09 MB/s write, 1683.77 GB read, 149.06 MB/s read, 27074.1 seconds
```
Reviewed By: ajkr
Differential Revision: D40518319
Pulled By: cbi42
fbshipit-source-id: f4ea614bc0ebefe007ffaf05bb9aec9a8ca25b60
2022-10-21 17:22:41 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"expanded_compaction_factor",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"level0_file_num_compaction_trigger",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, level0_file_num_compaction_trigger),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"level0_slowdown_writes_trigger",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"level0_stop_writes_trigger",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, level0_stop_writes_trigger),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_grandparent_overlap_factor",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_write_buffer_number",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, max_write_buffer_number),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"source_compaction_factor",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"target_file_size_multiplier",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, target_file_size_multiplier),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"arena_block_size",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, arena_block_size),
|
|
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"inplace_update_num_locks",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, inplace_update_num_locks),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_successive_merges",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, max_successive_merges),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2024-02-21 21:15:27 +00:00
|
|
|
{"strict_max_successive_merges",
|
|
|
|
|
{offsetof(struct MutableCFOptions, strict_max_successive_merges),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_huge_page_size",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, memtable_huge_page_size),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_prefix_bloom_huge_page_tlb_size",
|
|
|
|
|
{0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"write_buffer_size",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, write_buffer_size),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_prefix_bloom_bits",
|
|
|
|
|
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_prefix_bloom_size_ratio",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_prefix_bloom_probes",
|
|
|
|
|
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_whole_key_filtering",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, memtable_whole_key_filtering),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"min_partial_merge_operands",
|
|
|
|
|
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_bytes_for_level_base",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, max_bytes_for_level_base),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"snap_refresh_nanos",
|
|
|
|
|
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_bytes_for_level_multiplier",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_bytes_for_level_multiplier_additional",
|
2020-06-03 19:19:54 +00:00
|
|
|
OptionTypeInfo::Vector<int>(
|
|
|
|
|
offsetof(struct MutableCFOptions,
|
|
|
|
|
max_bytes_for_level_multiplier_additional),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
|
|
|
|
|
{0, OptionType::kInt})},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"max_sequential_skip_in_iterations",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, max_sequential_skip_in_iterations),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"target_file_size_base",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, target_file_size_base),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"compression",
|
2020-09-14 23:59:00 +00:00
|
|
|
{offsetof(struct MutableCFOptions, compression),
|
|
|
|
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"prefix_extractor",
|
2021-09-27 14:42:36 +00:00
|
|
|
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
|
|
|
|
|
offsetof(struct MutableCFOptions, prefix_extractor),
|
|
|
|
|
OptionVerificationType::kByNameAllowNull,
|
|
|
|
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kAllowNull))},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"compaction_options_fifo",
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
"compaction_options_fifo", &fifo_compaction_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, compaction_options_fifo),
|
2022-05-13 11:57:08 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable)
|
|
|
|
|
.SetParseFunc([](const ConfigOptions& opts,
|
|
|
|
|
const std::string& name, const std::string& value,
|
|
|
|
|
void* addr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
// This is to handle backward compatibility, where
|
|
|
|
|
// compaction_options_fifo could be assigned a single scalar
|
|
|
|
|
// value, say, like "23", which would be assigned to
|
|
|
|
|
// max_table_files_size.
|
|
|
|
|
if (name == "compaction_options_fifo" &&
|
2023-12-01 19:10:30 +00:00
|
|
|
value.find('=') == std::string::npos) {
|
2020-09-14 23:59:00 +00:00
|
|
|
// Old format. Parse just a single uint64_t value.
|
2021-05-13 21:28:50 +00:00
|
|
|
auto options = static_cast<CompactionOptionsFIFO*>(addr);
|
2020-09-14 23:59:00 +00:00
|
|
|
options->max_table_files_size = ParseUint64(value);
|
|
|
|
|
return Status::OK();
|
|
|
|
|
} else {
|
|
|
|
|
return OptionTypeInfo::ParseStruct(
|
|
|
|
|
opts, "compaction_options_fifo",
|
|
|
|
|
&fifo_compaction_options_type_info, name, value, addr);
|
|
|
|
|
}
|
|
|
|
|
})},
|
|
|
|
|
{"compaction_options_universal",
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
"compaction_options_universal",
|
|
|
|
|
&universal_compaction_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, compaction_options_universal),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable)},
|
|
|
|
|
{"ttl",
|
|
|
|
|
{offsetof(struct MutableCFOptions, ttl), OptionType::kUInt64T,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
{"periodic_compaction_seconds",
|
|
|
|
|
{offsetof(struct MutableCFOptions, periodic_compaction_seconds),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2022-01-25 22:58:48 +00:00
|
|
|
{"bottommost_temperature",
|
2022-08-08 21:36:34 +00:00
|
|
|
{0, OptionType::kTemperature, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"last_level_temperature",
|
|
|
|
|
{offsetof(struct MutableCFOptions, last_level_temperature),
|
2022-01-25 22:58:48 +00:00
|
|
|
OptionType::kTemperature, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2024-02-28 22:36:13 +00:00
|
|
|
{"default_write_temperature",
|
|
|
|
|
{offsetof(struct MutableCFOptions, default_write_temperature),
|
|
|
|
|
OptionType::kTemperature, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"enable_blob_files",
|
|
|
|
|
{offsetof(struct MutableCFOptions, enable_blob_files),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"min_blob_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, min_blob_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_file_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_file_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_compression_type",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_compression_type),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kMutable}},
|
2020-11-13 02:57:20 +00:00
|
|
|
{"enable_blob_garbage_collection",
|
|
|
|
|
{offsetof(struct MutableCFOptions, enable_blob_garbage_collection),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_garbage_collection_age_cutoff",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_garbage_collection_age_cutoff),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
2021-10-12 01:00:44 +00:00
|
|
|
{"blob_garbage_collection_force_threshold",
|
|
|
|
|
{offsetof(struct MutableCFOptions,
|
|
|
|
|
blob_garbage_collection_force_threshold),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2021-11-20 01:52:42 +00:00
|
|
|
{"blob_compaction_readahead_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_compaction_readahead_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2022-06-03 03:04:33 +00:00
|
|
|
{"blob_file_starting_level",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_file_starting_level),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2022-07-17 14:13:59 +00:00
|
|
|
{"prepopulate_blob_cache",
|
|
|
|
|
OptionTypeInfo::Enum<PrepopulateBlobCache>(
|
|
|
|
|
offsetof(struct MutableCFOptions, prepopulate_blob_cache),
|
|
|
|
|
&prepopulate_blob_cache_string_map, OptionTypeFlags::kMutable)},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"sample_for_compression",
|
|
|
|
|
{offsetof(struct MutableCFOptions, sample_for_compression),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"bottommost_compression",
|
|
|
|
|
{offsetof(struct MutableCFOptions, bottommost_compression),
|
|
|
|
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2022-03-08 02:06:19 +00:00
|
|
|
{"compression_per_level",
|
|
|
|
|
OptionTypeInfo::Vector<CompressionType>(
|
|
|
|
|
offsetof(struct MutableCFOptions, compression_per_level),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
|
|
|
|
|
{0, OptionType::kCompressionType})},
|
2022-06-23 16:42:18 +00:00
|
|
|
{"experimental_mempurge_threshold",
|
|
|
|
|
{offsetof(struct MutableCFOptions, experimental_mempurge_threshold),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2022-08-12 20:51:32 +00:00
|
|
|
{"memtable_protection_bytes_per_key",
|
|
|
|
|
{offsetof(struct MutableCFOptions, memtable_protection_bytes_per_key),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Delay bottommost level single file compactions (#11701)
Summary:
For leveled compaction, RocksDB has a special kind of compaction with reason "kBottommmostFiles" that compacts bottommost level files to clear data held by snapshots (more detail in https://github.com/facebook/rocksdb/issues/3009). Such compactions can happen soon after a relevant snapshot is released. For some use cases, a bottommost file may contain only a small amount of keys that can be cleared, so compacting such a file has a high write amp. In addition, these bottommost files may be compacted in compactions with reason other than "kBottommmostFiles" if we wait for some time (so that enough data is ingested to trigger such a compaction). This PR introduces an option `bottommost_file_compaction_delay` to specify the delay of these bottommost level single file compactions.
* The main change is in `VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction()` where we only add a file to `bottommost_files_marked_for_compaction_` if it oldest_snapshot is larger than its non-zero largest_seqno **and** the file is old enough. Note that if a file is not old enough but its largest_seqno is less than oldest_snapshot, we exclude it from the calculation of `bottommost_files_mark_threshold_`. This makes the change simpler, but such a file's eligibility for compaction will only be checked the next time `ComputeBottommostFilesMarkedForCompaction()` is called. This happens when a new Version is created (compaction, flush, SetOptions()...), a new enough snapshot is released (`VersionStorageInfo::UpdateOldestSnapshot()`) or when a compaction is picked and compaction score has to be re-calculated.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11701
Test Plan:
* Add two unit tests to test when bottommost_file_compaction_delay > 0.
* Ran crash test with the new option.
Reviewed By: jaykorean, ajkr
Differential Revision: D48331564
Pulled By: cbi42
fbshipit-source-id: c584f3dc5f6354fce3ed65f4c6366dc450b15ba8
2023-08-17 00:45:44 +00:00
|
|
|
{"bottommost_file_compaction_delay",
|
|
|
|
|
{offsetof(struct MutableCFOptions, bottommost_file_compaction_delay),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Support pro-actively erasing obsolete block cache entries (#12694)
Summary:
Currently, when files become obsolete, the block cache entries associated with them just age out naturally. With pure LRU, this is not too bad, as once you "use" enough cache entries to (re-)fill the cache, you are guranteed to have purged the obsolete entries. However, HyperClockCache is a counting clock cache with a somewhat longer memory, so could be more negatively impacted by previously-hot cache entries becoming obsolete, and taking longer to age out than newer single-hit entries.
Part of the reason we still have this natural aging-out is that there's almost no connection between block cache entries and the file they are associated with. Everything is hashed into the same pool(s) of entries with nothing like a secondary index based on file. Keeping track of such an index could be expensive.
This change adds a new, mutable CF option `uncache_aggressiveness` for erasing obsolete block cache entries. The process can be speculative, lossy, or unproductive because not all potential block cache entries associated with files will be resident in memory, and attempting to remove them all could be wasted CPU time. Rather than a simple on/off switch, `uncache_aggressiveness` basically tells RocksDB how much CPU you're willing to burn trying to purge obsolete block cache entries. When such efforts are not sufficiently productive for a file, we stop and move on.
The option is in ColumnFamilyOptions so that it is dynamically changeable for already-open files, and customizeable by CF.
Note that this block cache removal happens as part of the process of purging obsolete files, which is often in a background thread (depending on `background_purge_on_iterator_cleanup` and `avoid_unnecessary_blocking_io` options) rather than along CPU critical paths.
Notable auxiliary code details:
* Possibly fixing some issues with trivial moves with `only_delete_metadata`: unnecessary TableCache::Evict in that case and missing from the ObsoleteFileInfo move operator. (Not able to reproduce an current failure.)
* Remove suspicious TableCache::Erase() from VersionSet::AddObsoleteBlobFile() (TODO follow-up item)
Marked EXPERIMENTAL until more thorough validation is complete.
Direct stats of this functionality are omitted because they could be misleading. Block cache hit rate is a better indicator of benefit, and CPU profiling a better indicator of cost.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12694
Test Plan:
* Unit tests added, including refactoring an existing test to make better use of parameterized tests.
* Added to crash test.
* Performance, sample command:
```
for I in `seq 1 10`; do for UA in 300; do for CT in lru_cache fixed_hyper_clock_cache auto_hyper_clock_cache; do rm -rf /dev/shm/test3; TEST_TMPDIR=/dev/shm/test3 /usr/bin/time ./db_bench -benchmarks=readwhilewriting -num=13000000 -read_random_exp_range=6 -write_buffer_size=10000000 -bloom_bits=10 -cache_type=$CT -cache_size=390000000 -cache_index_and_filter_blocks=1 -disable_wal=1 -duration=60 -statistics -uncache_aggressiveness=$UA 2>&1 | grep -E 'micros/op|rocksdb.block.cache.data.(hit|miss)|rocksdb.number.keys.(read|written)|maxresident' | awk '/rocksdb.block.cache.data.miss/ { miss = $4 } /rocksdb.block.cache.data.hit/ { hit = $4 } { print } END { print "hit rate = " ((hit * 1.0) / (miss + hit)) }' | tee -a results-$CT-$UA; done; done; done
```
Averaging 10 runs each case, block cache data block hit rates
```
lru_cache
UA=0 -> hit rate = 0.327, ops/s = 87668, user CPU sec = 139.0
UA=300 -> hit rate = 0.336, ops/s = 87960, user CPU sec = 139.0
fixed_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 100069, user CPU sec = 139.9
UA=300 -> hit rate = 0.343, ops/s = 100104, user CPU sec = 140.2
auto_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 97580, user CPU sec = 140.5
UA=300 -> hit rate = 0.345, ops/s = 97972, user CPU sec = 139.8
```
Conclusion: up to roughly 1 percentage point of improved block cache hit rate, likely leading to overall improved efficiency (because the foreground CPU cost of cache misses likely outweighs the background CPU cost of erasure, let alone I/O savings).
Reviewed By: ajkr
Differential Revision: D57932442
Pulled By: pdillinger
fbshipit-source-id: 84a243ca5f965f731f346a4853009780a904af6c
2024-06-07 15:57:11 +00:00
|
|
|
{"uncache_aggressiveness",
|
|
|
|
|
{offsetof(struct MutableCFOptions, uncache_aggressiveness),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2023-04-25 19:08:23 +00:00
|
|
|
{"block_protection_bytes_per_key",
|
|
|
|
|
{offsetof(struct MutableCFOptions, block_protection_bytes_per_key),
|
|
|
|
|
OptionType::kUInt8T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2024-08-19 20:53:25 +00:00
|
|
|
{"paranoid_memory_checks",
|
|
|
|
|
{offsetof(struct MutableCFOptions, paranoid_memory_checks),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{kOptNameCompOpts,
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
kOptNameCompOpts, &compression_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, compression_opts),
|
|
|
|
|
OptionVerificationType::kNormal,
|
|
|
|
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
2021-05-13 21:28:50 +00:00
|
|
|
const std::string& value, void* addr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
// This is to handle backward compatibility, where
|
|
|
|
|
// compression_options was a ":" separated list.
|
|
|
|
|
if (name == kOptNameCompOpts &&
|
2023-12-01 19:10:30 +00:00
|
|
|
value.find('=') == std::string::npos) {
|
2021-05-13 21:28:50 +00:00
|
|
|
auto* compression = static_cast<CompressionOptions*>(addr);
|
2020-09-14 23:59:00 +00:00
|
|
|
return ParseCompressionOptions(value, name, *compression);
|
|
|
|
|
} else {
|
|
|
|
|
return OptionTypeInfo::ParseStruct(
|
|
|
|
|
opts, kOptNameCompOpts, &compression_options_type_info,
|
|
|
|
|
name, value, addr);
|
|
|
|
|
}
|
|
|
|
|
})},
|
|
|
|
|
{kOptNameBMCompOpts,
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
kOptNameBMCompOpts, &compression_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, bottommost_compression_opts),
|
|
|
|
|
OptionVerificationType::kNormal,
|
|
|
|
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
2021-05-13 21:28:50 +00:00
|
|
|
const std::string& value, void* addr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
// This is to handle backward compatibility, where
|
|
|
|
|
// compression_options was a ":" separated list.
|
|
|
|
|
if (name == kOptNameBMCompOpts &&
|
2023-12-01 19:10:30 +00:00
|
|
|
value.find('=') == std::string::npos) {
|
2021-05-13 21:28:50 +00:00
|
|
|
auto* compression = static_cast<CompressionOptions*>(addr);
|
2020-09-14 23:59:00 +00:00
|
|
|
return ParseCompressionOptions(value, name, *compression);
|
|
|
|
|
} else {
|
|
|
|
|
return OptionTypeInfo::ParseStruct(
|
|
|
|
|
opts, kOptNameBMCompOpts, &compression_options_type_info,
|
|
|
|
|
name, value, addr);
|
|
|
|
|
}
|
|
|
|
|
})},
|
|
|
|
|
// End special case properties
|
2023-08-03 02:58:56 +00:00
|
|
|
{"memtable_max_range_deletions",
|
|
|
|
|
{offsetof(struct MutableCFOptions, memtable_max_range_deletions),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
cf_immutable_options_type_info = {
|
|
|
|
|
/* not yet supported
|
|
|
|
|
CompressionOptions compression_opts;
|
|
|
|
|
TablePropertiesCollectorFactories table_properties_collector_factories;
|
2021-09-07 18:31:12 +00:00
|
|
|
using TablePropertiesCollectorFactories =
|
|
|
|
|
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>;
|
2020-09-14 23:59:00 +00:00
|
|
|
UpdateStatus (*inplace_callback)(char* existing_value,
|
|
|
|
|
uint34_t* existing_value_size,
|
|
|
|
|
Slice delta_value,
|
|
|
|
|
std::string* merged_value);
|
|
|
|
|
std::vector<DbPath> cf_paths;
|
|
|
|
|
*/
|
|
|
|
|
{"compaction_measure_io_stats",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2022-01-27 06:02:02 +00:00
|
|
|
{"purge_redundant_kvs_while_flush",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"inplace_update_support",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, inplace_update_support),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"level_compaction_dynamic_level_bytes",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions,
|
|
|
|
|
level_compaction_dynamic_level_bytes),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
Align compaction output file boundaries to the next level ones (#10655)
Summary:
Try to align the compaction output file boundaries to the next level ones
(grandparent level), to reduce the level compaction write-amplification.
In level compaction, there are "wasted" data at the beginning and end of the
output level files. Align the file boundary can avoid such "wasted" compaction.
With this PR, it tries to align the non-bottommost level file boundaries to its
next level ones. It may cut file when the file size is large enough (at least
50% of target_file_size) and not too large (2x target_file_size).
db_bench shows about 12.56% compaction reduction:
```
TEST_TMPDIR=/data/dbbench2 ./db_bench --benchmarks=fillrandom,readrandom -max_background_jobs=12 -num=400000000 -target_file_size_base=33554432
# baseline:
Flush(GB): cumulative 25.882, interval 7.216
Cumulative compaction: 285.90 GB write, 162.36 MB/s write, 269.68 GB read, 153.15 MB/s read, 2926.7 seconds
# with this change:
Flush(GB): cumulative 25.882, interval 7.753
Cumulative compaction: 249.97 GB write, 141.96 MB/s write, 233.74 GB read, 132.74 MB/s read, 2534.9 seconds
```
The compaction simulator shows a similar result (14% with 100G random data).
As a side effect, with this PR, the SST file size can exceed the
target_file_size, but is capped at 2x target_file_size. And there will be
smaller files. Here are file size statistics when loading 100GB with the target
file size 32MB:
```
baseline this_PR
count 1.656000e+03 1.705000e+03
mean 3.116062e+07 3.028076e+07
std 7.145242e+06 8.046139e+06
```
The feature is enabled by default, to revert to the old behavior disable it
with `AdvancedColumnFamilyOptions.level_compaction_dynamic_file_size = false`
Also includes https://github.com/facebook/rocksdb/issues/1963 to cut file before skippable grandparent file. Which is for
use case like user adding 2 or more non-overlapping data range at the same
time, it can reduce the overlapping of 2 datasets in the lower levels.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10655
Reviewed By: cbi42
Differential Revision: D39552321
Pulled By: jay-zhuang
fbshipit-source-id: 640d15f159ab0cd973f2426cfc3af266fc8bdde2
2022-09-30 02:43:55 +00:00
|
|
|
{"level_compaction_dynamic_file_size",
|
2024-02-02 23:37:40 +00:00
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
Align compaction output file boundaries to the next level ones (#10655)
Summary:
Try to align the compaction output file boundaries to the next level ones
(grandparent level), to reduce the level compaction write-amplification.
In level compaction, there are "wasted" data at the beginning and end of the
output level files. Align the file boundary can avoid such "wasted" compaction.
With this PR, it tries to align the non-bottommost level file boundaries to its
next level ones. It may cut file when the file size is large enough (at least
50% of target_file_size) and not too large (2x target_file_size).
db_bench shows about 12.56% compaction reduction:
```
TEST_TMPDIR=/data/dbbench2 ./db_bench --benchmarks=fillrandom,readrandom -max_background_jobs=12 -num=400000000 -target_file_size_base=33554432
# baseline:
Flush(GB): cumulative 25.882, interval 7.216
Cumulative compaction: 285.90 GB write, 162.36 MB/s write, 269.68 GB read, 153.15 MB/s read, 2926.7 seconds
# with this change:
Flush(GB): cumulative 25.882, interval 7.753
Cumulative compaction: 249.97 GB write, 141.96 MB/s write, 233.74 GB read, 132.74 MB/s read, 2534.9 seconds
```
The compaction simulator shows a similar result (14% with 100G random data).
As a side effect, with this PR, the SST file size can exceed the
target_file_size, but is capped at 2x target_file_size. And there will be
smaller files. Here are file size statistics when loading 100GB with the target
file size 32MB:
```
baseline this_PR
count 1.656000e+03 1.705000e+03
mean 3.116062e+07 3.028076e+07
std 7.145242e+06 8.046139e+06
```
The feature is enabled by default, to revert to the old behavior disable it
with `AdvancedColumnFamilyOptions.level_compaction_dynamic_file_size = false`
Also includes https://github.com/facebook/rocksdb/issues/1963 to cut file before skippable grandparent file. Which is for
use case like user adding 2 or more non-overlapping data range at the same
time, it can reduce the overlapping of 2 datasets in the lower levels.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10655
Reviewed By: cbi42
Differential Revision: D39552321
Pulled By: jay-zhuang
fbshipit-source-id: 640d15f159ab0cd973f2426cfc3af266fc8bdde2
2022-09-30 02:43:55 +00:00
|
|
|
OptionTypeFlags::kNone}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"optimize_filters_for_hits",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, optimize_filters_for_hits),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"force_consistency_checks",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, force_consistency_checks),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2023-08-18 00:06:57 +00:00
|
|
|
{"default_temperature",
|
|
|
|
|
{offsetof(struct ImmutableCFOptions, default_temperature),
|
|
|
|
|
OptionType::kTemperature, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kCompareNever}},
|
2022-07-15 04:49:34 +00:00
|
|
|
{"preclude_last_level_data_seconds",
|
|
|
|
|
{offsetof(struct ImmutableCFOptions, preclude_last_level_data_seconds),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2022-10-08 01:49:40 +00:00
|
|
|
{"preserve_internal_time_seconds",
|
|
|
|
|
{offsetof(struct ImmutableCFOptions, preserve_internal_time_seconds),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2022-02-04 13:31:07 +00:00
|
|
|
// Need to keep this around to be able to read old OPTIONS files.
|
2020-09-14 23:59:00 +00:00
|
|
|
{"max_mem_compaction_level",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"max_write_buffer_number_to_maintain",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions,
|
|
|
|
|
max_write_buffer_number_to_maintain),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2023-12-01 19:10:30 +00:00
|
|
|
OptionTypeFlags::kNone, nullptr}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"max_write_buffer_size_to_maintain",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions,
|
|
|
|
|
max_write_buffer_size_to_maintain),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"min_write_buffer_number_to_merge",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, min_write_buffer_number_to_merge),
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
2023-12-01 19:10:30 +00:00
|
|
|
OptionTypeFlags::kNone, nullptr}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"num_levels",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, num_levels), OptionType::kInt,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
|
|
|
|
|
{"bloom_locality",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, bloom_locality),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2020-09-14 23:59:00 +00:00
|
|
|
{"rate_limit_delay_max_milliseconds",
|
|
|
|
|
{0, OptionType::kUInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
2020-04-29 01:02:11 +00:00
|
|
|
{"comparator",
|
2021-06-11 13:21:55 +00:00
|
|
|
OptionTypeInfo::AsCustomRawPtr<const Comparator>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions, user_comparator),
|
2022-05-13 11:57:08 +00:00
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose)
|
|
|
|
|
.SetSerializeFunc(
|
|
|
|
|
// Serializes a Comparator
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&,
|
|
|
|
|
const void* addr, std::string* value) {
|
|
|
|
|
// it's a const pointer of const Comparator*
|
|
|
|
|
const auto* ptr =
|
|
|
|
|
static_cast<const Comparator* const*>(addr);
|
|
|
|
|
if (*ptr == nullptr) {
|
|
|
|
|
*value = kNullptrString;
|
|
|
|
|
} else if (opts.mutable_options_only) {
|
|
|
|
|
*value = "";
|
|
|
|
|
} else {
|
2023-11-02 20:27:59 +00:00
|
|
|
*value = (*ptr)->ToString(opts);
|
2022-05-13 11:57:08 +00:00
|
|
|
}
|
|
|
|
|
return Status::OK();
|
|
|
|
|
})},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_insert_with_hint_prefix_extractor",
|
2021-09-27 14:42:36 +00:00
|
|
|
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions,
|
|
|
|
|
memtable_insert_with_hint_prefix_extractor),
|
2021-09-27 14:42:36 +00:00
|
|
|
OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone)},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"memtable_factory",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, memtable_factory),
|
2021-09-08 14:45:59 +00:00
|
|
|
OptionType::kCustomizable, OptionVerificationType::kByName,
|
|
|
|
|
OptionTypeFlags::kShared,
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
std::unique_ptr<MemTableRepFactory> factory;
|
|
|
|
|
auto* shared =
|
|
|
|
|
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
|
|
|
|
|
Status s =
|
2022-05-16 16:44:43 +00:00
|
|
|
MemTableRepFactory::CreateFromString(opts, value, shared);
|
2021-09-08 14:45:59 +00:00
|
|
|
return s;
|
|
|
|
|
}}},
|
2020-04-29 01:02:11 +00:00
|
|
|
{"memtable",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, memtable_factory),
|
2021-09-08 14:45:59 +00:00
|
|
|
OptionType::kCustomizable, OptionVerificationType::kAlias,
|
|
|
|
|
OptionTypeFlags::kShared,
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&,
|
2021-05-13 21:28:50 +00:00
|
|
|
const std::string& value, void* addr) {
|
2021-09-08 14:45:59 +00:00
|
|
|
std::unique_ptr<MemTableRepFactory> factory;
|
|
|
|
|
auto* shared =
|
|
|
|
|
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
|
|
|
|
|
Status s =
|
2022-05-16 16:44:43 +00:00
|
|
|
MemTableRepFactory::CreateFromString(opts, value, shared);
|
2020-04-29 01:02:11 +00:00
|
|
|
return s;
|
|
|
|
|
}}},
|
2021-09-28 12:30:32 +00:00
|
|
|
{"table_properties_collectors",
|
|
|
|
|
OptionTypeInfo::Vector<
|
|
|
|
|
std::shared_ptr<TablePropertiesCollectorFactory>>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions,
|
|
|
|
|
table_properties_collector_factories),
|
2021-09-28 12:30:32 +00:00
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kNone,
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<TablePropertiesCollectorFactory>(
|
|
|
|
|
0, OptionVerificationType::kByName, OptionTypeFlags::kNone))},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"compaction_filter",
|
2021-08-06 15:26:23 +00:00
|
|
|
OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions, compaction_filter),
|
2021-08-06 15:26:23 +00:00
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"compaction_filter_factory",
|
2021-08-06 15:26:23 +00:00
|
|
|
OptionTypeInfo::AsCustomSharedPtr<CompactionFilterFactory>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions, compaction_filter_factory),
|
2021-08-06 15:26:23 +00:00
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
|
2020-04-29 01:02:11 +00:00
|
|
|
{"merge_operator",
|
2021-08-06 15:26:23 +00:00
|
|
|
OptionTypeInfo::AsCustomSharedPtr<MergeOperator>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions, merge_operator),
|
2021-08-06 15:26:23 +00:00
|
|
|
OptionVerificationType::kByNameAllowFromNull,
|
|
|
|
|
OptionTypeFlags::kCompareLoose | OptionTypeFlags::kAllowNull)},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"compaction_style",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, compaction_style),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kCompactionStyle, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kNone}},
|
2020-04-03 17:48:46 +00:00
|
|
|
{"compaction_pri",
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
{offsetof(struct ImmutableCFOptions, compaction_pri),
|
2020-04-03 17:48:46 +00:00
|
|
|
OptionType::kCompactionPri, OptionVerificationType::kNormal,
|
2020-09-14 23:59:00 +00:00
|
|
|
OptionTypeFlags::kNone}},
|
2021-09-28 12:30:32 +00:00
|
|
|
{"sst_partitioner_factory",
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<SstPartitionerFactory>(
|
Use -Wno-invalid-offsetof instead of dangerous offset_of hack (#9563)
Summary:
After https://github.com/facebook/rocksdb/issues/9515 added a unique_ptr to Status, we see some
warnings-as-error in some internal builds like this:
```
stderr: rocksdb/src/db/compaction/compaction_job.cc:2839:7: error:
offset of on non-standard-layout type 'struct CompactionServiceResult'
[-Werror,-Winvalid-offsetof]
{offsetof(struct CompactionServiceResult, status),
^ ~~~~~~
```
I see three potential solutions to resolving this:
* Expand our use of an idiom that works around the warning (see offset_of
functions removed in this change, inspired by
https://gist.github.com/graphitemaster/494f21190bb2c63c5516) However,
this construction is invoking undefined behavior that assumes consistent
layout with no compiler-introduced indirection. A compiler incompatible
with our assumptions will likely compile the code and exhibit undefined
behavior.
* Migrate to something in place of offset, like a function mapping
CompactionServiceResult* to Status* (for the `status` field). This might
be required in the long term.
* **Selected:** Use our new C++17 dependency to use offsetof in a well-defined way
when the compiler allows it. From a comment on
https://gist.github.com/graphitemaster/494f21190bb2c63c5516:
> A final note: in C++17, offsetof is conditionally supported, which
> means that you can use it on any type (not just standard layout
> types) and the compiler will error if it can't compile it correctly.
> That appears to be the best option if you can live with C++17 and
> don't need constexpr support.
The C++17 semantics are confirmed on
https://en.cppreference.com/w/cpp/types/offsetof, so we can suppress the
warning as long as we accept that we might run into a compiler that
rejects the code, and at that point we will find a solution, such as
the more intrusive "migrate" solution above.
Although this is currently only showing in our buck build, it will
surely show up also with make and cmake, so I have updated those
configurations as well.
Also in the buck build, -Wno-expansion-to-defined does not appear to be
needed anymore (both current compiler configurations) so I
removed it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9563
Test Plan: Tried out buck builds with both current compiler configurations
Reviewed By: riversand963
Differential Revision: D34220931
Pulled By: pdillinger
fbshipit-source-id: d39436008259bd1eaaa87c77be69fb2a5b559e1f
2022-02-15 17:18:08 +00:00
|
|
|
offsetof(struct ImmutableCFOptions, sst_partitioner_factory),
|
2021-09-28 12:30:32 +00:00
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
|
2022-06-14 21:19:26 +00:00
|
|
|
{"blob_cache",
|
|
|
|
|
{offsetof(struct ImmutableCFOptions, blob_cache), OptionType::kUnknown,
|
|
|
|
|
OptionVerificationType::kNormal,
|
|
|
|
|
(OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize),
|
|
|
|
|
// Parses the input value as a Cache
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
auto* cache = static_cast<std::shared_ptr<Cache>*>(addr);
|
|
|
|
|
return Cache::CreateFromString(opts, value, cache);
|
|
|
|
|
}}},
|
2023-04-12 00:50:34 +00:00
|
|
|
{"persist_user_defined_timestamps",
|
|
|
|
|
{offsetof(struct ImmutableCFOptions, persist_user_defined_timestamps),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kCompareLoose}},
|
2020-04-29 01:02:11 +00:00
|
|
|
};
|
2020-04-03 17:48:46 +00:00
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";
|
|
|
|
|
|
|
|
|
|
class ConfigurableMutableCFOptions : public Configurable {
|
|
|
|
|
public:
|
2021-04-26 10:12:35 +00:00
|
|
|
explicit ConfigurableMutableCFOptions(const MutableCFOptions& mcf) {
|
2020-09-14 23:59:00 +00:00
|
|
|
mutable_ = mcf;
|
2021-04-26 10:12:35 +00:00
|
|
|
RegisterOptions(&mutable_, &cf_mutable_options_type_info);
|
2020-09-14 23:59:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected:
|
|
|
|
|
MutableCFOptions mutable_;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class ConfigurableCFOptions : public ConfigurableMutableCFOptions {
|
|
|
|
|
public:
|
|
|
|
|
ConfigurableCFOptions(const ColumnFamilyOptions& opts,
|
|
|
|
|
const std::unordered_map<std::string, std::string>* map)
|
|
|
|
|
: ConfigurableMutableCFOptions(MutableCFOptions(opts)),
|
2021-05-05 20:59:21 +00:00
|
|
|
immutable_(opts),
|
2020-09-14 23:59:00 +00:00
|
|
|
cf_options_(opts),
|
|
|
|
|
opt_map_(map) {
|
2021-04-26 10:12:35 +00:00
|
|
|
RegisterOptions(&immutable_, &cf_immutable_options_type_info);
|
2020-09-14 23:59:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected:
|
|
|
|
|
Status ConfigureOptions(
|
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
|
std::unordered_map<std::string, std::string>* unused) override {
|
2021-02-19 18:25:39 +00:00
|
|
|
Status s = Configurable::ConfigureOptions(config_options, opts_map, unused);
|
2020-09-14 23:59:00 +00:00
|
|
|
if (s.ok()) {
|
2021-05-05 20:59:21 +00:00
|
|
|
UpdateColumnFamilyOptions(mutable_, &cf_options_);
|
|
|
|
|
UpdateColumnFamilyOptions(immutable_, &cf_options_);
|
2020-09-14 23:59:00 +00:00
|
|
|
s = PrepareOptions(config_options);
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
}
|
|
|
|
|
|
2024-01-31 21:14:42 +00:00
|
|
|
const void* GetOptionsPtr(const std::string& name) const override {
|
2020-09-14 23:59:00 +00:00
|
|
|
if (name == OptionsHelper::kCFOptionsName) {
|
|
|
|
|
return &cf_options_;
|
2020-05-21 17:56:40 +00:00
|
|
|
} else {
|
2020-09-14 23:59:00 +00:00
|
|
|
return ConfigurableMutableCFOptions::GetOptionsPtr(name);
|
2020-04-03 17:48:46 +00:00
|
|
|
}
|
|
|
|
|
}
|
2020-09-14 23:59:00 +00:00
|
|
|
|
|
|
|
|
bool OptionsAreEqual(const ConfigOptions& config_options,
|
|
|
|
|
const OptionTypeInfo& opt_info,
|
|
|
|
|
const std::string& opt_name, const void* const this_ptr,
|
|
|
|
|
const void* const that_ptr,
|
|
|
|
|
std::string* mismatch) const override {
|
|
|
|
|
bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr,
|
|
|
|
|
that_ptr, mismatch);
|
|
|
|
|
if (!equals && opt_info.IsByName()) {
|
|
|
|
|
if (opt_map_ == nullptr) {
|
|
|
|
|
equals = true;
|
|
|
|
|
} else {
|
|
|
|
|
const auto& iter = opt_map_->find(opt_name);
|
|
|
|
|
if (iter == opt_map_->end()) {
|
|
|
|
|
equals = true;
|
|
|
|
|
} else {
|
|
|
|
|
equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr,
|
|
|
|
|
iter->second);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (equals) { // False alarm, clear mismatch
|
|
|
|
|
*mismatch = "";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) {
|
|
|
|
|
const auto* this_config = opt_info.AsRawPointer<Configurable>(this_ptr);
|
|
|
|
|
if (this_config == nullptr) {
|
|
|
|
|
const auto& iter = opt_map_->find(opt_name);
|
|
|
|
|
// If the name exists in the map and is not empty/null,
|
|
|
|
|
// then the this_config should be set.
|
|
|
|
|
if (iter != opt_map_->end() && !iter->second.empty() &&
|
|
|
|
|
iter->second != kNullptrString) {
|
|
|
|
|
*mismatch = opt_name;
|
|
|
|
|
equals = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return equals;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private:
|
2021-04-23 03:42:50 +00:00
|
|
|
ImmutableCFOptions immutable_;
|
2020-09-14 23:59:00 +00:00
|
|
|
ColumnFamilyOptions cf_options_;
|
|
|
|
|
const std::unordered_map<std::string, std::string>* opt_map_;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
|
|
|
|
|
const MutableCFOptions& opts) {
|
|
|
|
|
std::unique_ptr<Configurable> ptr(new ConfigurableMutableCFOptions(opts));
|
|
|
|
|
return ptr;
|
|
|
|
|
}
|
|
|
|
|
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
|
|
|
|
|
const ColumnFamilyOptions& opts,
|
|
|
|
|
const std::unordered_map<std::string, std::string>* opt_map) {
|
|
|
|
|
std::unique_ptr<Configurable> ptr(new ConfigurableCFOptions(opts, opt_map));
|
|
|
|
|
return ptr;
|
2020-04-03 17:48:46 +00:00
|
|
|
}
|
2014-10-01 23:19:16 +00:00
|
|
|
|
2021-04-23 03:42:50 +00:00
|
|
|
ImmutableCFOptions::ImmutableCFOptions() : ImmutableCFOptions(Options()) {}
|
|
|
|
|
|
2021-05-05 20:59:21 +00:00
|
|
|
ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options)
|
2021-04-26 19:43:02 +00:00
|
|
|
: compaction_style(cf_options.compaction_style),
|
2016-09-23 23:34:04 +00:00
|
|
|
compaction_pri(cf_options.compaction_pri),
|
2016-10-21 18:31:42 +00:00
|
|
|
user_comparator(cf_options.comparator),
|
2016-11-21 20:07:09 +00:00
|
|
|
internal_comparator(InternalKeyComparator(cf_options.comparator)),
|
2021-04-23 03:42:50 +00:00
|
|
|
merge_operator(cf_options.merge_operator),
|
2016-09-23 23:34:04 +00:00
|
|
|
compaction_filter(cf_options.compaction_filter),
|
2021-04-23 03:42:50 +00:00
|
|
|
compaction_filter_factory(cf_options.compaction_filter_factory),
|
2016-09-23 23:34:04 +00:00
|
|
|
min_write_buffer_number_to_merge(
|
|
|
|
|
cf_options.min_write_buffer_number_to_merge),
|
|
|
|
|
max_write_buffer_number_to_maintain(
|
|
|
|
|
cf_options.max_write_buffer_number_to_maintain),
|
Refactor trimming logic for immutable memtables (#5022)
Summary:
MyRocks currently sets `max_write_buffer_number_to_maintain` in order to maintain enough history for transaction conflict checking. The effectiveness of this approach depends on the size of memtables. When memtables are small, it may not keep enough history; when memtables are large, this may consume too much memory.
We are proposing a new way to configure memtable list history: by limiting the memory usage of immutable memtables. The new option is `max_write_buffer_size_to_maintain` and it will take precedence over the old `max_write_buffer_number_to_maintain` if they are both set to non-zero values. The new option accounts for the total memory usage of flushed immutable memtables and mutable memtable. When the total usage exceeds the limit, RocksDB may start dropping immutable memtables (which is also called trimming history), starting from the oldest one.
The semantics of the old option actually works both as an upper bound and lower bound. History trimming will start if number of immutable memtables exceeds the limit, but it will never go below (limit-1) due to history trimming.
In order the mimic the behavior with the new option, history trimming will stop if dropping the next immutable memtable causes the total memory usage go below the size limit. For example, assuming the size limit is set to 64MB, and there are 3 immutable memtables with sizes of 20, 30, 30. Although the total memory usage is 80MB > 64MB, dropping the oldest memtable will reduce the memory usage to 60MB < 64MB, so in this case no memtable will be dropped.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5022
Differential Revision: D14394062
Pulled By: miasantreble
fbshipit-source-id: 60457a509c6af89d0993f988c9b5c2aa9e45f5c5
2019-08-23 20:54:09 +00:00
|
|
|
max_write_buffer_size_to_maintain(
|
|
|
|
|
cf_options.max_write_buffer_size_to_maintain),
|
2016-09-23 23:34:04 +00:00
|
|
|
inplace_update_support(cf_options.inplace_update_support),
|
|
|
|
|
inplace_callback(cf_options.inplace_callback),
|
2021-04-23 03:42:50 +00:00
|
|
|
memtable_factory(cf_options.memtable_factory),
|
2016-09-02 21:16:31 +00:00
|
|
|
table_properties_collector_factories(
|
2016-09-23 23:34:04 +00:00
|
|
|
cf_options.table_properties_collector_factories),
|
|
|
|
|
bloom_locality(cf_options.bloom_locality),
|
2016-09-02 21:16:31 +00:00
|
|
|
level_compaction_dynamic_level_bytes(
|
2016-09-23 23:34:04 +00:00
|
|
|
cf_options.level_compaction_dynamic_level_bytes),
|
|
|
|
|
num_levels(cf_options.num_levels),
|
|
|
|
|
optimize_filters_for_hits(cf_options.optimize_filters_for_hits),
|
2016-10-08 00:21:45 +00:00
|
|
|
force_consistency_checks(cf_options.force_consistency_checks),
|
2023-08-18 00:06:57 +00:00
|
|
|
default_temperature(cf_options.default_temperature),
|
2022-07-15 04:49:34 +00:00
|
|
|
preclude_last_level_data_seconds(
|
|
|
|
|
cf_options.preclude_last_level_data_seconds),
|
2022-10-08 01:49:40 +00:00
|
|
|
preserve_internal_time_seconds(cf_options.preserve_internal_time_seconds),
|
2016-11-14 02:58:17 +00:00
|
|
|
memtable_insert_with_hint_prefix_extractor(
|
2021-04-23 03:42:50 +00:00
|
|
|
cf_options.memtable_insert_with_hint_prefix_extractor),
|
Concurrent task limiter for compaction thread control (#4332)
Summary:
The PR is targeting to resolve the issue of:
https://github.com/facebook/rocksdb/issues/3972#issue-330771918
We have a rocksdb created with leveled-compaction with multiple column families (CFs), some of CFs are using HDD to store big and less frequently accessed data and others are using SSD.
When there are continuously write traffics going on to all CFs, the compaction thread pool is mostly occupied by those slow HDD compactions, which blocks fully utilize SSD bandwidth.
Since atomic write and transaction is needed across CFs, so splitting it to multiple rocksdb instance is not an option for us.
With the compaction thread control, we got 30%+ HDD write throughput gain, and also a lot smooth SSD write since less write stall happening.
ConcurrentTaskLimiter can be shared with multi-CFs across rocksdb instances, so the feature does not only work for multi-CFs scenarios, but also for multi-rocksdbs scenarios, who need disk IO resource control per tenant.
The usage is straight forward:
e.g.:
//
// Enable compaction thread limiter thru ColumnFamilyOptions
//
std::shared_ptr<ConcurrentTaskLimiter> ctl(NewConcurrentTaskLimiter("foo_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = ctl;
...
//
// Compaction thread limiter can be tuned or disabled on-the-fly
//
ctl->SetMaxOutstandingTask(12); // enlarge to 12 tasks
...
ctl->ResetMaxOutstandingTask(); // disable (bypass) thread limiter
ctl->SetMaxOutstandingTask(-1); // Same as above
...
ctl->SetMaxOutstandingTask(0); // full throttle (0 task)
//
// Sharing compaction thread limiter among CFs (to resolve multiple storage perf issue)
//
std::shared_ptr<ConcurrentTaskLimiter> ctl_ssd(NewConcurrentTaskLimiter("ssd_limiter", 8));
std::shared_ptr<ConcurrentTaskLimiter> ctl_hdd(NewConcurrentTaskLimiter("hdd_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt_ssd1(options);
ColumnFamilyOptions cf_opt_ssd2(options);
ColumnFamilyOptions cf_opt_hdd1(options);
ColumnFamilyOptions cf_opt_hdd2(options);
ColumnFamilyOptions cf_opt_hdd3(options);
// SSD CFs
cf_opt_ssd1.compaction_thread_limiter = ctl_ssd;
cf_opt_ssd2.compaction_thread_limiter = ctl_ssd;
// HDD CFs
cf_opt_hdd1.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd2.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd3.compaction_thread_limiter = ctl_hdd;
...
//
// The limiter is disabled by default (or set to nullptr explicitly)
//
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = nullptr;
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4332
Differential Revision: D13226590
Pulled By: siying
fbshipit-source-id: 14307aec55b8bd59c8223d04aa6db3c03d1b0c1d
2018-12-13 21:16:04 +00:00
|
|
|
cf_paths(cf_options.cf_paths),
|
2020-02-10 23:42:46 +00:00
|
|
|
compaction_thread_limiter(cf_options.compaction_thread_limiter),
|
2022-06-14 21:19:26 +00:00
|
|
|
sst_partitioner_factory(cf_options.sst_partitioner_factory),
|
2023-04-12 00:50:34 +00:00
|
|
|
blob_cache(cf_options.blob_cache),
|
|
|
|
|
persist_user_defined_timestamps(
|
|
|
|
|
cf_options.persist_user_defined_timestamps) {}
|
2021-05-05 20:59:21 +00:00
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions() : ImmutableOptions(Options()) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const Options& options)
|
|
|
|
|
: ImmutableOptions(options, options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
|
|
|
|
|
const ColumnFamilyOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
|
|
|
|
|
const ImmutableCFOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
|
|
|
|
|
const ColumnFamilyOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
|
|
|
|
|
const ImmutableCFOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
2016-09-02 21:16:31 +00:00
|
|
|
|
2014-10-01 23:19:16 +00:00
|
|
|
// Multiple two operands. If they overflow, return op1.
|
2016-11-02 04:05:32 +00:00
|
|
|
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
|
|
|
|
|
if (op1 == 0 || op2 <= 0) {
|
2014-10-01 23:19:16 +00:00
|
|
|
return 0;
|
|
|
|
|
}
|
2022-05-05 20:08:21 +00:00
|
|
|
if (std::numeric_limits<uint64_t>::max() / op1 < op2) {
|
2014-10-01 23:19:16 +00:00
|
|
|
return op1;
|
|
|
|
|
}
|
2016-11-02 04:05:32 +00:00
|
|
|
return static_cast<uint64_t>(op1 * op2);
|
2014-10-01 23:19:16 +00:00
|
|
|
}
|
|
|
|
|
|
2018-05-03 23:35:46 +00:00
|
|
|
// when level_compaction_dynamic_level_bytes is true and leveled compaction
|
|
|
|
|
// is used, the base level is not always L1, so precomupted max_file_size can
|
|
|
|
|
// no longer be used. Recompute file_size_for_level from base level.
|
|
|
|
|
uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
|
|
|
|
|
int level, CompactionStyle compaction_style, int base_level,
|
|
|
|
|
bool level_compaction_dynamic_level_bytes) {
|
|
|
|
|
if (!level_compaction_dynamic_level_bytes || level < base_level ||
|
|
|
|
|
compaction_style != kCompactionStyleLevel) {
|
|
|
|
|
assert(level >= 0);
|
|
|
|
|
assert(level < (int)cf_options.max_file_size.size());
|
|
|
|
|
return cf_options.max_file_size[level];
|
|
|
|
|
} else {
|
|
|
|
|
assert(level >= 0 && base_level >= 0);
|
|
|
|
|
assert(level - base_level < (int)cf_options.max_file_size.size());
|
|
|
|
|
return cf_options.max_file_size[level - base_level];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-06-09 23:49:07 +00:00
|
|
|
size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options) {
|
|
|
|
|
// We do not want to pin meta-blocks that almost certainly came from intra-L0
|
|
|
|
|
// or a former larger `write_buffer_size` value to avoid surprising users with
|
|
|
|
|
// pinned memory usage. We use a factor of 1.5 to account for overhead
|
|
|
|
|
// introduced during flush in most cases.
|
2022-05-05 20:08:21 +00:00
|
|
|
if (std::numeric_limits<size_t>::max() / 3 <
|
|
|
|
|
cf_options.write_buffer_size / 2) {
|
|
|
|
|
return std::numeric_limits<size_t>::max();
|
2020-06-09 23:49:07 +00:00
|
|
|
}
|
|
|
|
|
return cf_options.write_buffer_size / 2 * 3;
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-14 04:11:59 +00:00
|
|
|
void MutableCFOptions::RefreshDerivedOptions(int num_levels,
|
|
|
|
|
CompactionStyle compaction_style) {
|
|
|
|
|
max_file_size.resize(num_levels);
|
|
|
|
|
for (int i = 0; i < num_levels; ++i) {
|
|
|
|
|
if (i == 0 && compaction_style == kCompactionStyleUniversal) {
|
2014-10-01 23:19:16 +00:00
|
|
|
max_file_size[i] = ULLONG_MAX;
|
|
|
|
|
} else if (i > 1) {
|
|
|
|
|
max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
|
|
|
|
|
target_file_size_multiplier);
|
|
|
|
|
} else {
|
|
|
|
|
max_file_size[i] = target_file_size_base;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-17 00:22:28 +00:00
|
|
|
void MutableCFOptions::Dump(Logger* log) const {
|
|
|
|
|
// Memtable related options
|
2017-03-16 02:22:52 +00:00
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" write_buffer_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
write_buffer_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_write_buffer_number: %d",
|
|
|
|
|
max_write_buffer_number);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" arena_block_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
arena_block_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f",
|
|
|
|
|
memtable_prefix_bloom_size_ratio);
|
2019-02-19 20:12:25 +00:00
|
|
|
ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d",
|
|
|
|
|
memtable_whole_key_filtering);
|
2017-03-16 02:22:52 +00:00
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" memtable_huge_page_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
memtable_huge_page_size);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" max_successive_merges: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_successive_merges);
|
2024-02-21 21:15:27 +00:00
|
|
|
ROCKS_LOG_INFO(log, " strict_max_successive_merges: %d",
|
|
|
|
|
strict_max_successive_merges);
|
2017-03-16 02:22:52 +00:00
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
|
|
|
|
inplace_update_num_locks);
|
2021-09-27 14:42:36 +00:00
|
|
|
ROCKS_LOG_INFO(log, " prefix_extractor: %s",
|
|
|
|
|
prefix_extractor == nullptr
|
|
|
|
|
? "nullptr"
|
|
|
|
|
: prefix_extractor->GetId().c_str());
|
2017-03-16 02:22:52 +00:00
|
|
|
ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
|
|
|
|
|
disable_auto_compactions);
|
|
|
|
|
ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
|
|
|
|
|
soft_pending_compaction_bytes_limit);
|
|
|
|
|
ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64,
|
|
|
|
|
hard_pending_compaction_bytes_limit);
|
|
|
|
|
ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d",
|
|
|
|
|
level0_file_num_compaction_trigger);
|
|
|
|
|
ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d",
|
|
|
|
|
level0_slowdown_writes_trigger);
|
|
|
|
|
ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d",
|
|
|
|
|
level0_stop_writes_trigger);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64,
|
|
|
|
|
max_compaction_bytes);
|
|
|
|
|
ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64,
|
|
|
|
|
target_file_size_base);
|
|
|
|
|
ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d",
|
|
|
|
|
target_file_size_multiplier);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64,
|
|
|
|
|
max_bytes_for_level_base);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f",
|
|
|
|
|
max_bytes_for_level_multiplier);
|
2018-07-16 21:24:33 +00:00
|
|
|
ROCKS_LOG_INFO(log, " ttl: %" PRIu64,
|
|
|
|
|
ttl);
|
Periodic Compactions (#5166)
Summary:
Introducing Periodic Compactions.
This feature allows all the files in a CF to be periodically compacted. It could help in catching any corruptions that could creep into the DB proactively as every file is constantly getting re-compacted. And also, of course, it helps to cleanup data older than certain threshold.
- Introduced a new option `periodic_compaction_time` to control how long a file can live without being compacted in a CF.
- This works across all levels.
- The files are put in the same level after going through the compaction. (Related files in the same level are picked up as `ExpandInputstoCleanCut` is used).
- Compaction filters, if any, are invoked as usual.
- A new table property, `file_creation_time`, is introduced to implement this feature. This property is set to the time at which the SST file was created (and that time is given by the underlying Env/OS).
This feature can be enabled on its own, or in conjunction with `ttl`. It is possible to set a different time threshold for the bottom level when used in conjunction with ttl. Since `ttl` works only on 0 to last but one levels, you could set `ttl` to, say, 1 day, and `periodic_compaction_time` to, say, 7 days. Since `ttl < periodic_compaction_time` all files in last but one levels keep getting picked up based on ttl, and almost never based on periodic_compaction_time. The files in the bottom level get picked up for compaction based on `periodic_compaction_time`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5166
Differential Revision: D14884441
Pulled By: sagar0
fbshipit-source-id: 408426cbacb409c06386a98632dcf90bfa1bda47
2019-04-11 02:24:25 +00:00
|
|
|
ROCKS_LOG_INFO(log, " periodic_compaction_seconds: %" PRIu64,
|
|
|
|
|
periodic_compaction_seconds);
|
2024-08-19 20:53:25 +00:00
|
|
|
ROCKS_LOG_INFO(log, " paranoid_memory_checks: %d",
|
|
|
|
|
paranoid_memory_checks);
|
2014-10-17 00:22:28 +00:00
|
|
|
std::string result;
|
|
|
|
|
char buf[10];
|
|
|
|
|
for (const auto m : max_bytes_for_level_multiplier_additional) {
|
|
|
|
|
snprintf(buf, sizeof(buf), "%d, ", m);
|
|
|
|
|
result += buf;
|
|
|
|
|
}
|
2016-10-08 00:16:13 +00:00
|
|
|
if (result.size() >= 2) {
|
|
|
|
|
result.resize(result.size() - 2);
|
|
|
|
|
} else {
|
|
|
|
|
result = "";
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-16 02:22:52 +00:00
|
|
|
ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s",
|
|
|
|
|
result.c_str());
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64,
|
|
|
|
|
max_sequential_skip_in_iterations);
|
|
|
|
|
ROCKS_LOG_INFO(log, " paranoid_file_checks: %d",
|
|
|
|
|
paranoid_file_checks);
|
|
|
|
|
ROCKS_LOG_INFO(log, " report_bg_io_stats: %d",
|
|
|
|
|
report_bg_io_stats);
|
|
|
|
|
ROCKS_LOG_INFO(log, " compression: %d",
|
|
|
|
|
static_cast<int>(compression));
|
Support pro-actively erasing obsolete block cache entries (#12694)
Summary:
Currently, when files become obsolete, the block cache entries associated with them just age out naturally. With pure LRU, this is not too bad, as once you "use" enough cache entries to (re-)fill the cache, you are guranteed to have purged the obsolete entries. However, HyperClockCache is a counting clock cache with a somewhat longer memory, so could be more negatively impacted by previously-hot cache entries becoming obsolete, and taking longer to age out than newer single-hit entries.
Part of the reason we still have this natural aging-out is that there's almost no connection between block cache entries and the file they are associated with. Everything is hashed into the same pool(s) of entries with nothing like a secondary index based on file. Keeping track of such an index could be expensive.
This change adds a new, mutable CF option `uncache_aggressiveness` for erasing obsolete block cache entries. The process can be speculative, lossy, or unproductive because not all potential block cache entries associated with files will be resident in memory, and attempting to remove them all could be wasted CPU time. Rather than a simple on/off switch, `uncache_aggressiveness` basically tells RocksDB how much CPU you're willing to burn trying to purge obsolete block cache entries. When such efforts are not sufficiently productive for a file, we stop and move on.
The option is in ColumnFamilyOptions so that it is dynamically changeable for already-open files, and customizeable by CF.
Note that this block cache removal happens as part of the process of purging obsolete files, which is often in a background thread (depending on `background_purge_on_iterator_cleanup` and `avoid_unnecessary_blocking_io` options) rather than along CPU critical paths.
Notable auxiliary code details:
* Possibly fixing some issues with trivial moves with `only_delete_metadata`: unnecessary TableCache::Evict in that case and missing from the ObsoleteFileInfo move operator. (Not able to reproduce an current failure.)
* Remove suspicious TableCache::Erase() from VersionSet::AddObsoleteBlobFile() (TODO follow-up item)
Marked EXPERIMENTAL until more thorough validation is complete.
Direct stats of this functionality are omitted because they could be misleading. Block cache hit rate is a better indicator of benefit, and CPU profiling a better indicator of cost.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12694
Test Plan:
* Unit tests added, including refactoring an existing test to make better use of parameterized tests.
* Added to crash test.
* Performance, sample command:
```
for I in `seq 1 10`; do for UA in 300; do for CT in lru_cache fixed_hyper_clock_cache auto_hyper_clock_cache; do rm -rf /dev/shm/test3; TEST_TMPDIR=/dev/shm/test3 /usr/bin/time ./db_bench -benchmarks=readwhilewriting -num=13000000 -read_random_exp_range=6 -write_buffer_size=10000000 -bloom_bits=10 -cache_type=$CT -cache_size=390000000 -cache_index_and_filter_blocks=1 -disable_wal=1 -duration=60 -statistics -uncache_aggressiveness=$UA 2>&1 | grep -E 'micros/op|rocksdb.block.cache.data.(hit|miss)|rocksdb.number.keys.(read|written)|maxresident' | awk '/rocksdb.block.cache.data.miss/ { miss = $4 } /rocksdb.block.cache.data.hit/ { hit = $4 } { print } END { print "hit rate = " ((hit * 1.0) / (miss + hit)) }' | tee -a results-$CT-$UA; done; done; done
```
Averaging 10 runs each case, block cache data block hit rates
```
lru_cache
UA=0 -> hit rate = 0.327, ops/s = 87668, user CPU sec = 139.0
UA=300 -> hit rate = 0.336, ops/s = 87960, user CPU sec = 139.0
fixed_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 100069, user CPU sec = 139.9
UA=300 -> hit rate = 0.343, ops/s = 100104, user CPU sec = 140.2
auto_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 97580, user CPU sec = 140.5
UA=300 -> hit rate = 0.345, ops/s = 97972, user CPU sec = 139.8
```
Conclusion: up to roughly 1 percentage point of improved block cache hit rate, likely leading to overall improved efficiency (because the foreground CPU cost of cache misses likely outweighs the background CPU cost of erasure, let alone I/O savings).
Reviewed By: ajkr
Differential Revision: D57932442
Pulled By: pdillinger
fbshipit-source-id: 84a243ca5f965f731f346a4853009780a904af6c
2024-06-07 15:57:11 +00:00
|
|
|
ROCKS_LOG_INFO(log, " experimental_mempurge_threshold: %f",
|
2022-06-23 16:42:18 +00:00
|
|
|
experimental_mempurge_threshold);
|
Delay bottommost level single file compactions (#11701)
Summary:
For leveled compaction, RocksDB has a special kind of compaction with reason "kBottommmostFiles" that compacts bottommost level files to clear data held by snapshots (more detail in https://github.com/facebook/rocksdb/issues/3009). Such compactions can happen soon after a relevant snapshot is released. For some use cases, a bottommost file may contain only a small amount of keys that can be cleared, so compacting such a file has a high write amp. In addition, these bottommost files may be compacted in compactions with reason other than "kBottommmostFiles" if we wait for some time (so that enough data is ingested to trigger such a compaction). This PR introduces an option `bottommost_file_compaction_delay` to specify the delay of these bottommost level single file compactions.
* The main change is in `VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction()` where we only add a file to `bottommost_files_marked_for_compaction_` if it oldest_snapshot is larger than its non-zero largest_seqno **and** the file is old enough. Note that if a file is not old enough but its largest_seqno is less than oldest_snapshot, we exclude it from the calculation of `bottommost_files_mark_threshold_`. This makes the change simpler, but such a file's eligibility for compaction will only be checked the next time `ComputeBottommostFilesMarkedForCompaction()` is called. This happens when a new Version is created (compaction, flush, SetOptions()...), a new enough snapshot is released (`VersionStorageInfo::UpdateOldestSnapshot()`) or when a compaction is picked and compaction score has to be re-calculated.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11701
Test Plan:
* Add two unit tests to test when bottommost_file_compaction_delay > 0.
* Ran crash test with the new option.
Reviewed By: jaykorean, ajkr
Differential Revision: D48331564
Pulled By: cbi42
fbshipit-source-id: c584f3dc5f6354fce3ed65f4c6366dc450b15ba8
2023-08-17 00:45:44 +00:00
|
|
|
ROCKS_LOG_INFO(log, " bottommost_file_compaction_delay: %" PRIu32,
|
|
|
|
|
bottommost_file_compaction_delay);
|
Support pro-actively erasing obsolete block cache entries (#12694)
Summary:
Currently, when files become obsolete, the block cache entries associated with them just age out naturally. With pure LRU, this is not too bad, as once you "use" enough cache entries to (re-)fill the cache, you are guranteed to have purged the obsolete entries. However, HyperClockCache is a counting clock cache with a somewhat longer memory, so could be more negatively impacted by previously-hot cache entries becoming obsolete, and taking longer to age out than newer single-hit entries.
Part of the reason we still have this natural aging-out is that there's almost no connection between block cache entries and the file they are associated with. Everything is hashed into the same pool(s) of entries with nothing like a secondary index based on file. Keeping track of such an index could be expensive.
This change adds a new, mutable CF option `uncache_aggressiveness` for erasing obsolete block cache entries. The process can be speculative, lossy, or unproductive because not all potential block cache entries associated with files will be resident in memory, and attempting to remove them all could be wasted CPU time. Rather than a simple on/off switch, `uncache_aggressiveness` basically tells RocksDB how much CPU you're willing to burn trying to purge obsolete block cache entries. When such efforts are not sufficiently productive for a file, we stop and move on.
The option is in ColumnFamilyOptions so that it is dynamically changeable for already-open files, and customizeable by CF.
Note that this block cache removal happens as part of the process of purging obsolete files, which is often in a background thread (depending on `background_purge_on_iterator_cleanup` and `avoid_unnecessary_blocking_io` options) rather than along CPU critical paths.
Notable auxiliary code details:
* Possibly fixing some issues with trivial moves with `only_delete_metadata`: unnecessary TableCache::Evict in that case and missing from the ObsoleteFileInfo move operator. (Not able to reproduce an current failure.)
* Remove suspicious TableCache::Erase() from VersionSet::AddObsoleteBlobFile() (TODO follow-up item)
Marked EXPERIMENTAL until more thorough validation is complete.
Direct stats of this functionality are omitted because they could be misleading. Block cache hit rate is a better indicator of benefit, and CPU profiling a better indicator of cost.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12694
Test Plan:
* Unit tests added, including refactoring an existing test to make better use of parameterized tests.
* Added to crash test.
* Performance, sample command:
```
for I in `seq 1 10`; do for UA in 300; do for CT in lru_cache fixed_hyper_clock_cache auto_hyper_clock_cache; do rm -rf /dev/shm/test3; TEST_TMPDIR=/dev/shm/test3 /usr/bin/time ./db_bench -benchmarks=readwhilewriting -num=13000000 -read_random_exp_range=6 -write_buffer_size=10000000 -bloom_bits=10 -cache_type=$CT -cache_size=390000000 -cache_index_and_filter_blocks=1 -disable_wal=1 -duration=60 -statistics -uncache_aggressiveness=$UA 2>&1 | grep -E 'micros/op|rocksdb.block.cache.data.(hit|miss)|rocksdb.number.keys.(read|written)|maxresident' | awk '/rocksdb.block.cache.data.miss/ { miss = $4 } /rocksdb.block.cache.data.hit/ { hit = $4 } { print } END { print "hit rate = " ((hit * 1.0) / (miss + hit)) }' | tee -a results-$CT-$UA; done; done; done
```
Averaging 10 runs each case, block cache data block hit rates
```
lru_cache
UA=0 -> hit rate = 0.327, ops/s = 87668, user CPU sec = 139.0
UA=300 -> hit rate = 0.336, ops/s = 87960, user CPU sec = 139.0
fixed_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 100069, user CPU sec = 139.9
UA=300 -> hit rate = 0.343, ops/s = 100104, user CPU sec = 140.2
auto_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 97580, user CPU sec = 140.5
UA=300 -> hit rate = 0.345, ops/s = 97972, user CPU sec = 139.8
```
Conclusion: up to roughly 1 percentage point of improved block cache hit rate, likely leading to overall improved efficiency (because the foreground CPU cost of cache misses likely outweighs the background CPU cost of erasure, let alone I/O savings).
Reviewed By: ajkr
Differential Revision: D57932442
Pulled By: pdillinger
fbshipit-source-id: 84a243ca5f965f731f346a4853009780a904af6c
2024-06-07 15:57:11 +00:00
|
|
|
ROCKS_LOG_INFO(log, " uncache_aggressiveness: %" PRIu32,
|
|
|
|
|
uncache_aggressiveness);
|
2018-07-17 05:12:28 +00:00
|
|
|
|
|
|
|
|
// Universal Compaction Options
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d",
|
|
|
|
|
compaction_options_universal.size_ratio);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d",
|
|
|
|
|
compaction_options_universal.min_merge_width);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d",
|
|
|
|
|
compaction_options_universal.max_merge_width);
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
log, "compaction_options_universal.max_size_amplification_percent : %d",
|
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
"compaction_options_universal.compression_size_percent : %d",
|
|
|
|
|
compaction_options_universal.compression_size_percent);
|
Improve universal compaction sorted-run trigger (#12477)
Summary:
Universal compaction currently uses `level0_file_num_compaction_trigger` for two purposes:
1. the trigger for checking if there is any compaction to do, and
2. the limit on the number of sorted runs. RocksDB will do compaction to keep the number of sorted runs no more than the value of this option.
This can make the option inflexible. A value that is too small causes higher write amp: more compactions to reduce the number of sorted runs. A value that is too big delays potential compaction work and causes worse read performance. This PR introduce an option `CompactionOptionsUniversal::max_read_amp` for only the second purpose: to specify
the hard limit on the number of sorted runs.
For backward compatibility, `max_read_amp = -1` by default, which means to fallback to the current behavior.
When `max_read_amp > 0`,`level0_file_num_compaction_trigger` will only serve as a trigger to find potential compaction.
When `max_read_amp = 0`, RocksDB will auto-tune the limit on the number of sorted runs. The estimation is based on DB size, write_buffer_size and size_ratio, so it is adaptive to the size change of the DB. See more in `UniversalCompactionBuilder::PickCompaction()`.
Alternatively, users now can configure `max_read_amp` to a very big value and keep `level0_file_num_compaction_trigger` small. This will allow `size_ratio` and `max_size_amplification_percent` to control the number of sorted runs. This essentially disables compactions with reason kUniversalSortedRunNum.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12477
Test Plan:
* new unit test
* existing unit test for default behavior
* updated crash test with the new option
* benchmark:
* Create a DB that is roughly 24GB in the last level. When `max_read_amp = 0`, we estimate that the DB needs 9 levels to avoid excessive compactions to reduce the number of sorted runs.
* We then run fillrandom to ingest another 24GB data to compare write amp.
* case 1: small level0 trigger: `level0_file_num_compaction_trigger=5, max_read_amp=-1`
* write-amp: 4.8
* case 2: auto-tune: `level0_file_num_compaction_trigger=5, max_read_amp=0`
* write-amp: 3.6
* case 3: auto-tune with minimal trigger: `level0_file_num_compaction_trigger=1, max_read_amp=0`
* write-amp: 3.8
* case 4: hard-code a good value for trigger: `level0_file_num_compaction_trigger=9`
* write-amp: 2.8
```
Case 1:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 0/0 0.00 KB 1.0 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 163.2 141.94 111.10 108 1.314 0 0 0.0 0.0
L45 8/0 1.81 GB 0.0 39.6 11.1 28.5 39.3 10.8 0.0 3.5 209.0 207.3 194.25 191.29 43 4.517 348M 2498K 0.0 0.0
L46 13/0 3.12 GB 0.0 15.3 9.5 5.8 15.0 9.3 0.0 1.6 203.1 199.3 77.13 75.88 16 4.821 134M 2362K 0.0 0.0
L47 19/0 4.68 GB 0.0 15.4 10.5 4.9 14.7 9.8 0.0 1.4 204.0 194.9 77.38 76.15 8 9.673 135M 5920K 0.0 0.0
L48 38/0 9.42 GB 0.0 19.6 11.7 7.9 17.3 9.4 0.0 1.5 206.5 182.3 97.15 95.02 4 24.287 172M 20M 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 169/0 41.74 GB 0.0 89.9 42.9 47.0 109.0 61.9 0.0 4.8 156.7 189.8 587.85 549.45 179 3.284 791M 31M 0.0 0.0
Case 2:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 1/0 214.47 MB 1.2 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 164.5 140.81 109.98 108 1.304 0 0 0.0 0.0
L44 0/0 0.00 KB 0.0 1.3 1.3 0.0 1.2 1.2 0.0 1.0 206.1 204.9 6.24 5.98 3 2.081 11M 51K 0.0 0.0
L45 4/0 844.36 MB 0.0 7.1 5.4 1.7 7.0 5.4 0.0 1.3 194.6 192.9 37.41 36.00 13 2.878 62M 489K 0.0 0.0
L46 11/0 2.57 GB 0.0 14.6 9.8 4.8 14.3 9.5 0.0 1.5 193.7 189.8 77.09 73.54 17 4.535 128M 2411K 0.0 0.0
L47 24/0 5.81 GB 0.0 19.8 12.0 7.8 18.8 11.0 0.0 1.6 191.4 181.1 106.19 101.21 9 11.799 174M 9166K 0.0 0.0
L48 38/0 9.42 GB 0.0 19.6 11.8 7.9 17.3 9.4 0.0 1.5 197.3 173.6 101.97 97.23 4 25.491 172M 20M 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 169/0 41.54 GB 0.0 62.4 40.3 22.1 81.3 59.2 0.0 3.6 136.1 177.2 469.71 423.94 154 3.050 549M 32M 0.0 0.0
Case 3:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 0/0 0.00 KB 5.0 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 163.8 141.43 111.13 108 1.310 0 0 0.0 0.0
L44 0/0 0.00 KB 0.0 0.8 0.8 0.0 0.8 0.8 0.0 1.0 201.4 200.2 4.26 4.19 2 2.130 7360K 33K 0.0 0.0
L45 4/0 844.38 MB 0.0 6.3 5.0 1.2 6.2 5.0 0.0 1.2 202.0 200.3 31.81 31.50 12 2.651 55M 403K 0.0 0.0
L46 7/0 1.62 GB 0.0 13.3 8.8 4.6 13.1 8.6 0.0 1.5 198.9 195.7 68.72 67.89 17 4.042 117M 1696K 0.0 0.0
L47 24/0 5.81 GB 0.0 21.7 12.9 8.8 20.6 11.8 0.0 1.6 198.5 188.6 112.04 109.97 12 9.336 191M 9352K 0.0 0.0
L48 41/0 10.14 GB 0.0 24.8 13.0 11.8 21.9 10.1 0.0 1.7 198.6 175.6 127.88 125.36 6 21.313 218M 25M 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 167/0 41.10 GB 0.0 67.0 40.5 26.4 85.4 58.9 0.0 3.8 141.1 179.8 486.13 450.04 157 3.096 589M 36M 0.0 0.0
Case 4:
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 0/0 0.00 KB 0.7 0.0 0.0 0.0 22.6 22.6 0.0 1.0 0.0 158.6 146.02 114.68 108 1.352 0 0 0.0 0.0
L42 0/0 0.00 KB 0.0 1.7 1.7 0.0 1.7 1.7 0.0 1.0 185.4 184.3 9.25 8.96 4 2.314 14M 67K 0.0 0.0
L43 0/0 0.00 KB 0.0 2.5 2.5 0.0 2.5 2.5 0.0 1.0 197.8 195.6 13.01 12.65 4 3.253 22M 202K 0.0 0.0
L44 4/0 844.40 MB 0.0 4.2 4.2 0.0 4.1 4.1 0.0 1.0 188.1 185.1 22.81 21.89 5 4.562 36M 503K 0.0 0.0
L45 13/0 3.12 GB 0.0 7.5 6.5 1.0 7.2 6.2 0.0 1.1 188.7 181.8 40.69 39.32 5 8.138 65M 2282K 0.0 0.0
L46 17/0 4.18 GB 0.0 8.3 7.1 1.2 7.9 6.6 0.0 1.1 192.2 181.8 44.23 43.06 4 11.058 73M 3846K 0.0 0.0
L47 22/0 5.34 GB 0.0 8.9 7.5 1.4 8.2 6.8 0.0 1.1 189.1 174.1 48.12 45.37 3 16.041 78M 6098K 0.0 0.0
L48 27/0 6.58 GB 0.0 9.2 7.6 1.6 8.2 6.6 0.0 1.1 195.2 172.9 48.52 47.11 2 24.262 81M 9217K 0.0 0.0
L49 91/0 22.70 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0
Sum 174/0 42.74 GB 0.0 42.3 37.0 5.3 62.4 57.1 0.0 2.8 116.3 171.3 372.66 333.04 135 2.760 372M 22M 0.0 0.0
setup:
./db_bench --benchmarks=fillseq,compactall,waitforcompaction --num=200000000 --compression_type=none --disable_wal=1 --compaction_style=1 --num_levels=50 --target_file_size_base=268435456 --max_compaction_bytes=6710886400 --level0_file_num_compaction_trigger=10 --write_buffer_size=268435456 --seed 1708494134896523
benchmark:
./db_bench --benchmarks=overwrite,waitforcompaction,stats --num=200000000 --compression_type=none --disable_wal=1 --compaction_style=1 --write_buffer_size=268435456 --level0_file_num_compaction_trigger=5 --target_file_size_base=268435456 --use_existing_db=1 --num_levels=50 --writes=200000000 --universal_max_read_amp=-1 --seed=1716488324800233
```
Reviewed By: ajkr
Differential Revision: D55370922
Pulled By: cbi42
fbshipit-source-id: 9be69979126b840d08e93e7059260e76a878bb2a
2024-05-24 17:10:31 +00:00
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.max_read_amp: %d",
|
|
|
|
|
compaction_options_universal.max_read_amp);
|
2018-07-17 05:12:28 +00:00
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d",
|
|
|
|
|
compaction_options_universal.stop_style);
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
log, "compaction_options_universal.allow_trivial_move : %d",
|
|
|
|
|
static_cast<int>(compaction_options_universal.allow_trivial_move));
|
Incremental Space Amp Compactions in Universal Style (#8655)
Summary:
This commit introduces incremental compaction in univeral style for space amplification. This follows the first improvement mentioned in https://rocksdb.org/blog/2021/04/12/universal-improvements.html . The implemention simply picks up files about size of max_compaction_bytes to compact and execute if the penalty is not too big. More optimizations can be done in the future, e.g. prioritizing between this compaction and other types. But for now, the feature is supposed to be functional and can often reduce frequency of full compactions, although it can introduce penalty.
In order to add cut files more efficiently so that more files from upper levels can be included, SST file cutting threshold (for current file + overlapping parent level files) is set to 1.5X of target file size. A 2MB target file size will generate files like this: https://gist.github.com/siying/29d2676fba417404f3c95e6c013c7de8 Number of files indeed increases but it is not out of control.
Two set of write benchmarks are run:
1. For ingestion rate limited scenario, we can see full compaction is mostly eliminated: https://gist.github.com/siying/959bc1186066906831cf4c808d6e0a19 . The write amp increased from 7.7 to 9.4, as expected. After applying file cutting, the number is improved to 8.9. In another benchmark, the write amp is even better with the incremental approach: https://gist.github.com/siying/d1c16c286d7c59c4d7bba718ca198163
2. For ingestion rate unlimited scenario, incremental compaction turns out to be too expensive most of the time and is not executed, as expected.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8655
Test Plan: Add unit tests to the functionality.
Reviewed By: ajkr
Differential Revision: D31787034
fbshipit-source-id: ce813e63b15a61d5a56e97bf8902a1b28e011beb
2021-10-20 17:03:03 +00:00
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.incremental : %d",
|
|
|
|
|
static_cast<int>(compaction_options_universal.incremental));
|
2018-07-17 05:12:28 +00:00
|
|
|
|
|
|
|
|
// FIFO Compaction Options
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64,
|
|
|
|
|
compaction_options_fifo.max_table_files_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d",
|
|
|
|
|
compaction_options_fifo.allow_compaction);
|
2020-08-19 01:31:31 +00:00
|
|
|
|
|
|
|
|
// Blob file related options
|
|
|
|
|
ROCKS_LOG_INFO(log, " enable_blob_files: %s",
|
|
|
|
|
enable_blob_files ? "true" : "false");
|
|
|
|
|
ROCKS_LOG_INFO(log, " min_blob_size: %" PRIu64,
|
|
|
|
|
min_blob_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_file_size: %" PRIu64,
|
|
|
|
|
blob_file_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_compression_type: %s",
|
|
|
|
|
CompressionTypeToString(blob_compression_type).c_str());
|
2020-11-13 02:57:20 +00:00
|
|
|
ROCKS_LOG_INFO(log, " enable_blob_garbage_collection: %s",
|
|
|
|
|
enable_blob_garbage_collection ? "true" : "false");
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_garbage_collection_age_cutoff: %f",
|
|
|
|
|
blob_garbage_collection_age_cutoff);
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
2021-10-12 01:00:44 +00:00
|
|
|
ROCKS_LOG_INFO(log, " blob_garbage_collection_force_threshold: %f",
|
|
|
|
|
blob_garbage_collection_force_threshold);
|
2021-11-20 01:52:42 +00:00
|
|
|
ROCKS_LOG_INFO(log, " blob_compaction_readahead_size: %" PRIu64,
|
|
|
|
|
blob_compaction_readahead_size);
|
2022-06-03 03:04:33 +00:00
|
|
|
ROCKS_LOG_INFO(log, " blob_file_starting_level: %d",
|
|
|
|
|
blob_file_starting_level);
|
2022-07-17 14:13:59 +00:00
|
|
|
ROCKS_LOG_INFO(log, " prepopulate_blob_cache: %s",
|
|
|
|
|
prepopulate_blob_cache == PrepopulateBlobCache::kFlushOnly
|
|
|
|
|
? "flush only"
|
|
|
|
|
: "disable");
|
2022-08-08 21:36:34 +00:00
|
|
|
ROCKS_LOG_INFO(log, " last_level_temperature: %d",
|
|
|
|
|
static_cast<int>(last_level_temperature));
|
2014-10-17 00:22:28 +00:00
|
|
|
}
|
|
|
|
|
|
2018-05-21 21:33:55 +00:00
|
|
|
MutableCFOptions::MutableCFOptions(const Options& options)
|
|
|
|
|
: MutableCFOptions(ColumnFamilyOptions(options)) {}
|
|
|
|
|
|
2021-05-11 23:14:33 +00:00
|
|
|
Status GetMutableOptionsFromStrings(
|
|
|
|
|
const MutableCFOptions& base_options,
|
|
|
|
|
const std::unordered_map<std::string, std::string>& options_map,
|
|
|
|
|
Logger* /*info_log*/, MutableCFOptions* new_options) {
|
|
|
|
|
assert(new_options);
|
|
|
|
|
*new_options = base_options;
|
|
|
|
|
ConfigOptions config_options;
|
|
|
|
|
Status s = OptionTypeInfo::ParseType(
|
|
|
|
|
config_options, options_map, cf_mutable_options_type_info, new_options);
|
|
|
|
|
if (!s.ok()) {
|
|
|
|
|
*new_options = base_options;
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
|
|
|
|
|
const MutableCFOptions& mutable_opts,
|
|
|
|
|
std::string* opt_string) {
|
|
|
|
|
assert(opt_string);
|
|
|
|
|
opt_string->clear();
|
|
|
|
|
return OptionTypeInfo::SerializeType(
|
|
|
|
|
config_options, cf_mutable_options_type_info, &mutable_opts, opt_string);
|
|
|
|
|
}
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|