2018-06-28 19:23:57 +00:00
|
|
|
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
|
//
|
|
|
|
|
#include "db/error_handler.h"
|
2021-03-18 05:36:42 +00:00
|
|
|
|
2019-05-31 18:52:59 +00:00
|
|
|
#include "db/db_impl/db_impl.h"
|
2019-05-31 22:21:36 +00:00
|
|
|
#include "db/event_helpers.h"
|
2019-05-30 03:44:08 +00:00
|
|
|
#include "file/sst_file_manager_impl.h"
|
2021-03-18 05:36:42 +00:00
|
|
|
#include "logging/logging.h"
|
2018-06-28 19:23:57 +00:00
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2018-06-28 19:23:57 +00:00
|
|
|
|
|
|
|
|
// Maps to help decide the severity of an error based on the
|
|
|
|
|
// BackgroundErrorReason, Code, SubCode and whether db_options.paranoid_checks
|
|
|
|
|
// is set or not. There are 3 maps, going from most specific to least specific
|
|
|
|
|
// (i.e from all 4 fields in a tuple to only the BackgroundErrorReason and
|
|
|
|
|
// paranoid_checks). The less specific map serves as a catch all in case we miss
|
|
|
|
|
// a specific error code or subcode.
|
|
|
|
|
std::map<std::tuple<BackgroundErrorReason, Status::Code, Status::SubCode, bool>,
|
|
|
|
|
Status::Severity>
|
|
|
|
|
ErrorSeverityMap = {
|
|
|
|
|
// Errors during BG compaction
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kSoftError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kSpaceLimit,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kHardError},
|
2020-09-14 23:03:32 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
// Errors during BG flush
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
Status::SubCode::kNoSpace, true),
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
Status::Severity::kHardError},
|
2018-06-28 19:23:57 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
Status::SubCode::kNoSpace, false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
Status::SubCode::kSpaceLimit, true),
|
|
|
|
|
Status::Severity::kHardError},
|
2020-09-14 23:03:32 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
Status::SubCode::kIOFenced, true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
Status::SubCode::kIOFenced, false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
// Errors during Write
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
true),
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
Status::Severity::kHardError},
|
2018-06-28 19:23:57 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
false),
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
Status::Severity::kHardError},
|
2020-09-14 23:03:32 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kFatalError},
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
// Errors during MANIFEST write
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWrite,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kHardError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWrite,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kHardError},
|
2020-09-14 23:03:32 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWrite,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWrite,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2020-09-18 03:22:35 +00:00
|
|
|
// Errors during BG flush with WAL disabled
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kHardError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kSpaceLimit,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kHardError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2020-12-03 02:22:05 +00:00
|
|
|
// Errors during MANIFEST write when WAL is disabled
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kHardError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kNoSpace,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kHardError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL,
|
|
|
|
|
Status::Code::kIOError, Status::SubCode::kIOFenced,
|
|
|
|
|
false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2020-09-18 03:22:35 +00:00
|
|
|
|
2018-06-28 19:23:57 +00:00
|
|
|
};
|
|
|
|
|
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
std::map<std::tuple<BackgroundErrorReason, Status::Code, bool>,
|
|
|
|
|
Status::Severity>
|
2018-06-28 19:23:57 +00:00
|
|
|
DefaultErrorSeverityMap = {
|
|
|
|
|
// Errors during BG compaction
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kCorruption, true),
|
|
|
|
|
Status::Severity::kUnrecoverableError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kCorruption, false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction,
|
|
|
|
|
Status::Code::kIOError, false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
// Errors during BG flush
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush,
|
|
|
|
|
Status::Code::kCorruption, true),
|
|
|
|
|
Status::Severity::kUnrecoverableError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush,
|
|
|
|
|
Status::Code::kCorruption, false),
|
|
|
|
|
Status::Severity::kNoError},
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
true),
|
2018-06-28 19:23:57 +00:00
|
|
|
Status::Severity::kFatalError},
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
|
|
|
|
|
false),
|
2018-06-28 19:23:57 +00:00
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
// Errors during Write
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kCorruption, true),
|
|
|
|
|
Status::Severity::kUnrecoverableError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kCorruption, false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kIOError, true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback,
|
|
|
|
|
Status::Code::kIOError, false),
|
|
|
|
|
Status::Severity::kNoError},
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWrite,
|
|
|
|
|
Status::Code::kIOError, true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWrite,
|
|
|
|
|
Status::Code::kIOError, false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2020-09-18 03:22:35 +00:00
|
|
|
// Errors during BG flush with WAL disabled
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kCorruption, true),
|
|
|
|
|
Status::Severity::kUnrecoverableError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kCorruption, false),
|
|
|
|
|
Status::Severity::kNoError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlushNoWAL,
|
|
|
|
|
Status::Code::kIOError, false),
|
|
|
|
|
Status::Severity::kNoError},
|
2020-12-03 02:22:05 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL,
|
|
|
|
|
Status::Code::kIOError, true),
|
|
|
|
|
Status::Severity::kFatalError},
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL,
|
|
|
|
|
Status::Code::kIOError, false),
|
|
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
std::map<std::tuple<BackgroundErrorReason, bool>, Status::Severity>
|
|
|
|
|
DefaultReasonMap = {
|
|
|
|
|
// Errors during BG compaction
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction, true),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kCompaction, false),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kNoError},
|
2018-06-28 19:23:57 +00:00
|
|
|
// Errors during BG flush
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, true),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kFlush, false),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kNoError},
|
2018-06-28 19:23:57 +00:00
|
|
|
// Errors during Write
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback, true),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kWriteCallback, false),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
// Errors during Memtable update
|
|
|
|
|
{std::make_tuple(BackgroundErrorReason::kMemTable, true),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
{std::make_tuple(BackgroundErrorReason::kMemTable, false),
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
Status::Severity::kFatalError},
|
2018-06-28 19:23:57 +00:00
|
|
|
};
|
|
|
|
|
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
void ErrorHandler::CancelErrorRecovery() {
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
db_mutex_->AssertHeld();
|
|
|
|
|
|
|
|
|
|
// We'll release the lock before calling sfm, so make sure no new
|
|
|
|
|
// recovery gets scheduled at that point
|
|
|
|
|
auto_recovery_ = false;
|
|
|
|
|
SstFileManagerImpl* sfm = reinterpret_cast<SstFileManagerImpl*>(
|
|
|
|
|
db_options_.sst_file_manager.get());
|
|
|
|
|
if (sfm) {
|
|
|
|
|
// This may or may not cancel a pending recovery
|
|
|
|
|
db_mutex_->Unlock();
|
|
|
|
|
bool cancelled = sfm->CancelErrorRecovery(this);
|
|
|
|
|
db_mutex_->Lock();
|
|
|
|
|
if (cancelled) {
|
|
|
|
|
recovery_in_prog_ = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
|
|
|
|
|
// If auto recovery is also runing to resume from the retryable error,
|
|
|
|
|
// we should wait and end the auto recovery.
|
|
|
|
|
EndAutoRecovery();
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// This is the main function for looking at an error during a background
|
|
|
|
|
// operation and deciding the severity, and error recovery strategy. The high
|
|
|
|
|
// level algorithm is as follows -
|
|
|
|
|
// 1. Classify the severity of the error based on the ErrorSeverityMap,
|
|
|
|
|
// DefaultErrorSeverityMap and DefaultReasonMap defined earlier
|
|
|
|
|
// 2. Call a Status code specific override function to adjust the severity
|
|
|
|
|
// if needed. The reason for this is our ability to recover may depend on
|
|
|
|
|
// the exact options enabled in DBOptions
|
|
|
|
|
// 3. Determine if auto recovery is possible. A listener notification callback
|
|
|
|
|
// is called, which can disable the auto recovery even if we decide its
|
|
|
|
|
// feasible
|
|
|
|
|
// 4. For Status::NoSpace() errors, rely on SstFileManagerImpl to control
|
|
|
|
|
// the actual recovery. If no sst file manager is specified in DBOptions,
|
|
|
|
|
// a default one is allocated during DB::Open(), so there will always be
|
|
|
|
|
// one.
|
|
|
|
|
// This can also get called as part of a recovery operation. In that case, we
|
2018-10-05 03:44:43 +00:00
|
|
|
// also track the error separately in recovery_error_ so we can tell in the
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
// end whether recovery succeeded or not
|
2020-12-08 04:09:55 +00:00
|
|
|
const Status& ErrorHandler::SetBGError(const Status& bg_err,
|
|
|
|
|
BackgroundErrorReason reason) {
|
2018-06-28 19:23:57 +00:00
|
|
|
db_mutex_->AssertHeld();
|
|
|
|
|
if (bg_err.ok()) {
|
2020-12-08 04:09:55 +00:00
|
|
|
return bg_err;
|
2018-06-28 19:23:57 +00:00
|
|
|
}
|
|
|
|
|
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT);
|
|
|
|
|
}
|
|
|
|
|
ROCKS_LOG_INFO(db_options_.info_log,
|
|
|
|
|
"ErrorHandler: Set regular background error\n");
|
|
|
|
|
|
2018-06-28 19:23:57 +00:00
|
|
|
bool paranoid = db_options_.paranoid_checks;
|
|
|
|
|
Status::Severity sev = Status::Severity::kFatalError;
|
|
|
|
|
Status new_bg_err;
|
2020-09-18 03:22:35 +00:00
|
|
|
DBRecoverContext context;
|
2018-06-28 19:23:57 +00:00
|
|
|
bool found = false;
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
auto entry = ErrorSeverityMap.find(std::make_tuple(reason, bg_err.code(),
|
|
|
|
|
bg_err.subcode(), paranoid));
|
|
|
|
|
if (entry != ErrorSeverityMap.end()) {
|
|
|
|
|
sev = entry->second;
|
|
|
|
|
found = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!found) {
|
|
|
|
|
auto entry = DefaultErrorSeverityMap.find(std::make_tuple(reason,
|
|
|
|
|
bg_err.code(), paranoid));
|
|
|
|
|
if (entry != DefaultErrorSeverityMap.end()) {
|
|
|
|
|
sev = entry->second;
|
|
|
|
|
found = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!found) {
|
|
|
|
|
auto entry = DefaultReasonMap.find(std::make_tuple(reason, paranoid));
|
|
|
|
|
if (entry != DefaultReasonMap.end()) {
|
|
|
|
|
sev = entry->second;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
new_bg_err = Status(bg_err, sev);
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
|
2020-01-30 18:53:46 +00:00
|
|
|
// Check if recovery is currently in progress. If it is, we will save this
|
|
|
|
|
// error so we can check it at the end to see if recovery succeeded or not
|
|
|
|
|
if (recovery_in_prog_ && recovery_error_.ok()) {
|
|
|
|
|
recovery_error_ = new_bg_err;
|
|
|
|
|
}
|
|
|
|
|
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
bool auto_recovery = auto_recovery_;
|
|
|
|
|
if (new_bg_err.severity() >= Status::Severity::kFatalError && auto_recovery) {
|
|
|
|
|
auto_recovery = false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Allow some error specific overrides
|
|
|
|
|
if (new_bg_err == Status::NoSpace()) {
|
|
|
|
|
new_bg_err = OverrideNoSpaceError(new_bg_err, &auto_recovery);
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-28 19:23:57 +00:00
|
|
|
if (!new_bg_err.ok()) {
|
|
|
|
|
Status s = new_bg_err;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, &s,
|
|
|
|
|
db_mutex_, &auto_recovery);
|
2018-06-28 19:23:57 +00:00
|
|
|
if (!s.ok() && (s.severity() > bg_error_.severity())) {
|
|
|
|
|
bg_error_ = s;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
} else {
|
|
|
|
|
// This error is less severe than previously encountered error. Don't
|
|
|
|
|
// take any further action
|
|
|
|
|
return bg_error_;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-18 03:22:35 +00:00
|
|
|
recover_context_ = context;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
if (auto_recovery) {
|
|
|
|
|
recovery_in_prog_ = true;
|
|
|
|
|
|
|
|
|
|
// Kick-off error specific recovery
|
|
|
|
|
if (bg_error_ == Status::NoSpace()) {
|
|
|
|
|
RecoverFromNoSpace();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return bg_error_;
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-03 02:22:05 +00:00
|
|
|
// This is the main function for looking at IO related error during the
|
|
|
|
|
// background operations. The main logic is:
|
2021-01-08 00:30:06 +00:00
|
|
|
// 1) File scope IO error is treated as retryable IO error in the write
|
|
|
|
|
// path. In RocksDB, If a file has write IO error and it is at file scope,
|
|
|
|
|
// RocksDB never write to the same file again. RocksDB will create a new
|
|
|
|
|
// file and rewrite the whole content. Thus, it is retryable.
|
2020-12-03 02:22:05 +00:00
|
|
|
// 1) if the error is caused by data loss, the error is mapped to
|
|
|
|
|
// unrecoverable error. Application/user must take action to handle
|
2021-01-08 00:30:06 +00:00
|
|
|
// this situation (File scope case is excluded).
|
|
|
|
|
// 2) if the error is a Retryable IO error (i.e., it is a file scope IO error,
|
|
|
|
|
// or its retryable flag is set and not a data loss error), auto resume
|
|
|
|
|
// will be called and the auto resume can be controlled by resume count
|
|
|
|
|
// and resume interval options. There are three sub-cases:
|
2020-12-03 02:22:05 +00:00
|
|
|
// a) if the error happens during compaction, it is mapped to a soft error.
|
|
|
|
|
// the compaction thread will reschedule a new compaction.
|
|
|
|
|
// b) if the error happens during flush and also WAL is empty, it is mapped
|
|
|
|
|
// to a soft error. Note that, it includes the case that IO error happens
|
|
|
|
|
// in SST or manifest write during flush.
|
|
|
|
|
// c) all other errors are mapped to hard error.
|
|
|
|
|
// 3) for other cases, SetBGError(const Status& bg_err, BackgroundErrorReason
|
|
|
|
|
// reason) will be called to handle other error cases.
|
2020-12-08 04:09:55 +00:00
|
|
|
const Status& ErrorHandler::SetBGError(const IOStatus& bg_io_err,
|
|
|
|
|
BackgroundErrorReason reason) {
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
db_mutex_->AssertHeld();
|
|
|
|
|
if (bg_io_err.ok()) {
|
2020-12-08 04:09:55 +00:00
|
|
|
return bg_io_err;
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
ROCKS_LOG_WARN(db_options_.info_log, "Background IO error %s",
|
|
|
|
|
bg_io_err.ToString().c_str());
|
|
|
|
|
|
|
|
|
|
if (recovery_in_prog_ && recovery_io_error_.ok()) {
|
|
|
|
|
recovery_io_error_ = bg_io_err;
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
}
|
2020-12-03 02:22:05 +00:00
|
|
|
if (BackgroundErrorReason::kManifestWrite == reason ||
|
|
|
|
|
BackgroundErrorReason::kManifestWriteNoWAL == reason) {
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
// Always returns ok
|
2020-09-29 16:47:33 +00:00
|
|
|
db_->DisableFileDeletionsWithLock().PermitUncheckedError();
|
First step towards handling MANIFEST write error (#6949)
Summary:
This PR provides preliminary support for handling IO error during MANIFEST write.
File write/sync is not guaranteed to be atomic. If we encounter an IOError while writing/syncing to the MANIFEST file, we cannot be sure about the state of the MANIFEST file. The version edits may or may not have reached the file. During cleanup, if we delete the newly-generated SST files referenced by the pending version edit(s), but the version edit(s) actually are persistent in the MANIFEST, then next recovery attempt will process the version edits(s) and then fail since the SST files have already been deleted.
One approach is to truncate the MANIFEST after write/sync error, so that it is safe to delete the SST files. However, file truncation may not be supported on certain file systems. Therefore, we take the following approach.
If an IOError is detected during MANIFEST write/sync, we disable file deletions for the faulty database. Depending on whether the IOError is retryable (set by underlying file system), either RocksDB or application can call `DB::Resume()`, or simply shutdown and restart. During `Resume()`, RocksDB will try to switch to a new MANIFEST and write all existing in-memory version storage in the new file. If this succeeds, then RocksDB may proceed. If all recovery is completed, then file deletions will be re-enabled.
Note that multiple threads can call `LogAndApply()` at the same time, though only one of them will be going through the process MANIFEST write, possibly batching the version edits of other threads. When the leading MANIFEST writer finishes, all of the MANIFEST writing threads in this batch will have the same IOError. They will all call `ErrorHandler::SetBGError()` in which file deletion will be disabled.
Possible future directions:
- Add an `ErrorContext` structure so that it is easier to pass more info to `ErrorHandler`. Currently, as in this example, a new `BackgroundErrorReason` has to be added.
Test plan (dev server):
make check
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6949
Reviewed By: anand1976
Differential Revision: D22026020
Pulled By: riversand963
fbshipit-source-id: f3c68a2ef45d9b505d0d625c7c5e0c88495b91c8
2020-06-25 02:05:47 +00:00
|
|
|
}
|
2020-09-18 03:22:35 +00:00
|
|
|
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
Status new_bg_io_err = bg_io_err;
|
2020-09-18 03:22:35 +00:00
|
|
|
DBRecoverContext context;
|
2021-01-08 00:30:06 +00:00
|
|
|
if (bg_io_err.GetScope() != IOStatus::IOErrorScope::kIOErrorScopeFile &&
|
|
|
|
|
bg_io_err.GetDataLoss()) {
|
|
|
|
|
// First, data loss (non file scope) is treated as unrecoverable error. So
|
|
|
|
|
// it can directly overwrite any existing bg_error_.
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
bool auto_recovery = false;
|
|
|
|
|
Status bg_err(new_bg_io_err, Status::Severity::kUnrecoverableError);
|
|
|
|
|
bg_error_ = bg_err;
|
2020-07-15 18:02:44 +00:00
|
|
|
if (recovery_in_prog_ && recovery_error_.ok()) {
|
|
|
|
|
recovery_error_ = bg_err;
|
|
|
|
|
}
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT);
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT);
|
|
|
|
|
}
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
db_options_.info_log,
|
|
|
|
|
"ErrorHandler: Set background IO error as unrecoverable error\n");
|
2020-12-08 04:09:55 +00:00
|
|
|
EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason,
|
|
|
|
|
&bg_err, db_mutex_, &auto_recovery);
|
2020-09-18 03:22:35 +00:00
|
|
|
recover_context_ = context;
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
return bg_error_;
|
All the NoSpace() errors will be handled by regular SetBGError and RecoverFromNoSpace() (#8376)
Summary:
In the current logic, any IO Error with retryable flag == true will be handled by the special logic and in most cases, StartRecoverFromRetryableBGIOError will be called to do the auto resume. If the NoSpace error with retryable flag is set during WAL write, it is mapped as a hard error, which will trigger the auto recovery. During the recover process, if write continues and append to the WAL, the write process sees that bg_error is set to HardError and it calls WriteStatusCheck(), which calls SetBGError() with Status (not IOStatus). This will redirect to the regular SetBGError interface, in which recovery_error_ will be set to the corresponding error. With the recovery_error_ set, the auto resume thread created in StartRecoverFromRetryableBGIOError will keep failing as long as user keeps trying to write.
To fix this issue. All the NoSpace error (no matter retryable flag is set or not) will be redirect to the regular SetBGError, and RecoverFromNoSpace() will do the recovery job which calls SstFileManager::StartErrorRecovery().
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8376
Test Plan: make check and added the new testing case
Reviewed By: anand1976
Differential Revision: D29071828
Pulled By: zhichao-cao
fbshipit-source-id: 7171d7e14cc4620fdab49b7eff7a2fe9a89942c2
2021-06-11 21:47:40 +00:00
|
|
|
} else if (bg_io_err.subcode() != IOStatus::SubCode::kNoSpace &&
|
|
|
|
|
(bg_io_err.GetScope() ==
|
|
|
|
|
IOStatus::IOErrorScope::kIOErrorScopeFile ||
|
|
|
|
|
bg_io_err.GetRetryable())) {
|
2021-01-08 00:30:06 +00:00
|
|
|
// Second, check if the error is a retryable IO error (file scope IO error
|
|
|
|
|
// is also treated as retryable IO error in RocksDB write path). if it is
|
|
|
|
|
// retryable error and its severity is higher than bg_error_, overwrite the
|
|
|
|
|
// bg_error_ with new error. In current stage, for retryable IO error of
|
|
|
|
|
// compaction, treat it as soft error. In other cases, treat the retryable
|
All the NoSpace() errors will be handled by regular SetBGError and RecoverFromNoSpace() (#8376)
Summary:
In the current logic, any IO Error with retryable flag == true will be handled by the special logic and in most cases, StartRecoverFromRetryableBGIOError will be called to do the auto resume. If the NoSpace error with retryable flag is set during WAL write, it is mapped as a hard error, which will trigger the auto recovery. During the recover process, if write continues and append to the WAL, the write process sees that bg_error is set to HardError and it calls WriteStatusCheck(), which calls SetBGError() with Status (not IOStatus). This will redirect to the regular SetBGError interface, in which recovery_error_ will be set to the corresponding error. With the recovery_error_ set, the auto resume thread created in StartRecoverFromRetryableBGIOError will keep failing as long as user keeps trying to write.
To fix this issue. All the NoSpace error (no matter retryable flag is set or not) will be redirect to the regular SetBGError, and RecoverFromNoSpace() will do the recovery job which calls SstFileManager::StartErrorRecovery().
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8376
Test Plan: make check and added the new testing case
Reviewed By: anand1976
Differential Revision: D29071828
Pulled By: zhichao-cao
fbshipit-source-id: 7171d7e14cc4620fdab49b7eff7a2fe9a89942c2
2021-06-11 21:47:40 +00:00
|
|
|
// IO error as hard error. Note that, all the NoSpace error should be
|
|
|
|
|
// handled by the SstFileManager::StartErrorRecovery(). Therefore, no matter
|
|
|
|
|
// it is retryable or file scope, this logic will be bypassed.
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
bool auto_recovery = false;
|
2020-12-08 04:09:55 +00:00
|
|
|
EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason,
|
|
|
|
|
&new_bg_io_err, db_mutex_,
|
|
|
|
|
&auto_recovery);
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT);
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT);
|
|
|
|
|
RecordTick(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT);
|
|
|
|
|
}
|
|
|
|
|
ROCKS_LOG_INFO(db_options_.info_log,
|
|
|
|
|
"ErrorHandler: Set background retryable IO error\n");
|
2020-07-15 18:02:44 +00:00
|
|
|
if (BackgroundErrorReason::kCompaction == reason) {
|
2021-01-28 01:56:17 +00:00
|
|
|
// We map the retryable IO error during compaction to soft error. Since
|
|
|
|
|
// compaction can reschedule by itself. We will not set the BG error in
|
|
|
|
|
// this case
|
|
|
|
|
// TODO: a better way to set or clean the retryable IO error which
|
|
|
|
|
// happens during compaction SST file write.
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_COUNT);
|
|
|
|
|
}
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
db_options_.info_log,
|
|
|
|
|
"ErrorHandler: Compaction will schedule by itself to resume\n");
|
2020-07-15 18:02:44 +00:00
|
|
|
return bg_error_;
|
2020-12-03 02:22:05 +00:00
|
|
|
} else if (BackgroundErrorReason::kFlushNoWAL == reason ||
|
|
|
|
|
BackgroundErrorReason::kManifestWriteNoWAL == reason) {
|
2020-09-18 03:22:35 +00:00
|
|
|
// When the BG Retryable IO error reason is flush without WAL,
|
|
|
|
|
// We map it to a soft error. At the same time, all the background work
|
|
|
|
|
// should be stopped except the BG work from recovery. Therefore, we
|
|
|
|
|
// set the soft_error_no_bg_work_ to true. At the same time, since DB
|
|
|
|
|
// continues to receive writes when BG error is soft error, to avoid
|
|
|
|
|
// to many small memtable being generated during auto resume, the flush
|
|
|
|
|
// reason is set to kErrorRecoveryRetryFlush.
|
|
|
|
|
Status bg_err(new_bg_io_err, Status::Severity::kSoftError);
|
|
|
|
|
if (recovery_in_prog_ && recovery_error_.ok()) {
|
|
|
|
|
recovery_error_ = bg_err;
|
|
|
|
|
}
|
|
|
|
|
if (bg_err.severity() > bg_error_.severity()) {
|
|
|
|
|
bg_error_ = bg_err;
|
|
|
|
|
}
|
|
|
|
|
soft_error_no_bg_work_ = true;
|
|
|
|
|
context.flush_reason = FlushReason::kErrorRecoveryRetryFlush;
|
|
|
|
|
recover_context_ = context;
|
|
|
|
|
return StartRecoverFromRetryableBGIOError(bg_io_err);
|
2020-07-15 18:02:44 +00:00
|
|
|
} else {
|
|
|
|
|
Status bg_err(new_bg_io_err, Status::Severity::kHardError);
|
|
|
|
|
if (recovery_in_prog_ && recovery_error_.ok()) {
|
|
|
|
|
recovery_error_ = bg_err;
|
|
|
|
|
}
|
|
|
|
|
if (bg_err.severity() > bg_error_.severity()) {
|
|
|
|
|
bg_error_ = bg_err;
|
|
|
|
|
}
|
2020-09-18 03:22:35 +00:00
|
|
|
recover_context_ = context;
|
2020-07-15 18:02:44 +00:00
|
|
|
return StartRecoverFromRetryableBGIOError(bg_io_err);
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
}
|
|
|
|
|
} else {
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT);
|
|
|
|
|
}
|
2020-12-08 04:09:55 +00:00
|
|
|
return SetBGError(new_bg_io_err, reason);
|
Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487)
Summary:
In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status.
The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487
Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check
Reviewed By: anand1976
Differential Revision: D20685017
Pulled By: zhichao-cao
fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0
2020-03-27 23:03:05 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-08 04:09:55 +00:00
|
|
|
Status ErrorHandler::OverrideNoSpaceError(const Status& bg_error,
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
bool* auto_recovery) {
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
if (bg_error.severity() >= Status::Severity::kFatalError) {
|
|
|
|
|
return bg_error;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (db_options_.sst_file_manager.get() == nullptr) {
|
|
|
|
|
// We rely on SFM to poll for enough disk space and recover
|
|
|
|
|
*auto_recovery = false;
|
|
|
|
|
return bg_error;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (db_options_.allow_2pc &&
|
|
|
|
|
(bg_error.severity() <= Status::Severity::kSoftError)) {
|
|
|
|
|
// Don't know how to recover, as the contents of the current WAL file may
|
|
|
|
|
// be inconsistent, and it may be needed for 2PC. If 2PC is not enabled,
|
|
|
|
|
// we can just flush the memtable and discard the log
|
|
|
|
|
*auto_recovery = false;
|
|
|
|
|
return Status(bg_error, Status::Severity::kFatalError);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
uint64_t free_space;
|
|
|
|
|
if (db_options_.env->GetFreeSpace(db_options_.db_paths[0].path,
|
|
|
|
|
&free_space) == Status::NotSupported()) {
|
|
|
|
|
*auto_recovery = false;
|
2018-06-28 19:23:57 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
return bg_error;
|
|
|
|
|
#else
|
|
|
|
|
(void)auto_recovery;
|
|
|
|
|
return Status(bg_error, Status::Severity::kFatalError);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ErrorHandler::RecoverFromNoSpace() {
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
SstFileManagerImpl* sfm =
|
|
|
|
|
reinterpret_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
|
|
|
|
|
|
|
|
|
|
// Inform SFM of the error, so it can kick-off the recovery
|
|
|
|
|
if (sfm) {
|
|
|
|
|
sfm->StartErrorRecovery(this, bg_error_);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Status ErrorHandler::ClearBGError() {
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
db_mutex_->AssertHeld();
|
|
|
|
|
|
|
|
|
|
// Signal that recovery succeeded
|
|
|
|
|
if (recovery_error_.ok()) {
|
|
|
|
|
Status old_bg_error = bg_error_;
|
2020-12-08 04:09:55 +00:00
|
|
|
// Clear and check the recovery IO and BG error
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
bg_error_ = Status::OK();
|
2020-12-08 04:09:55 +00:00
|
|
|
recovery_io_error_ = IOStatus::OK();
|
|
|
|
|
bg_error_.PermitUncheckedError();
|
|
|
|
|
recovery_io_error_.PermitUncheckedError();
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
recovery_in_prog_ = false;
|
2020-09-18 03:22:35 +00:00
|
|
|
soft_error_no_bg_work_ = false;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
EventHelpers::NotifyOnErrorRecoveryCompleted(db_options_.listeners,
|
|
|
|
|
old_bg_error, db_mutex_);
|
|
|
|
|
}
|
|
|
|
|
return recovery_error_;
|
|
|
|
|
#else
|
2018-06-28 19:23:57 +00:00
|
|
|
return bg_error_;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
#endif
|
2018-06-28 19:23:57 +00:00
|
|
|
}
|
|
|
|
|
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
Status ErrorHandler::RecoverFromBGError(bool is_manual) {
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
InstrumentedMutexLock l(db_mutex_);
|
2020-09-18 03:22:35 +00:00
|
|
|
bool no_bg_work_original_flag = soft_error_no_bg_work_;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
if (is_manual) {
|
|
|
|
|
// If its a manual recovery and there's a background recovery in progress
|
|
|
|
|
// return busy status
|
|
|
|
|
if (recovery_in_prog_) {
|
|
|
|
|
return Status::Busy();
|
|
|
|
|
}
|
|
|
|
|
recovery_in_prog_ = true;
|
2020-09-18 03:22:35 +00:00
|
|
|
|
|
|
|
|
// In manual resume, we allow the bg work to run. If it is a auto resume,
|
|
|
|
|
// the bg work should follow this tag.
|
|
|
|
|
soft_error_no_bg_work_ = false;
|
|
|
|
|
|
|
|
|
|
// In manual resume, if the bg error is a soft error and also requires
|
|
|
|
|
// no bg work, the error must be recovered by call the flush with
|
|
|
|
|
// flush reason: kErrorRecoveryRetryFlush. In other case, the flush
|
|
|
|
|
// reason is set to kErrorRecovery.
|
|
|
|
|
if (no_bg_work_original_flag) {
|
|
|
|
|
recover_context_.flush_reason = FlushReason::kErrorRecoveryRetryFlush;
|
|
|
|
|
} else {
|
|
|
|
|
recover_context_.flush_reason = FlushReason::kErrorRecovery;
|
|
|
|
|
}
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
}
|
|
|
|
|
|
2020-09-18 03:22:35 +00:00
|
|
|
if (bg_error_.severity() == Status::Severity::kSoftError &&
|
|
|
|
|
recover_context_.flush_reason == FlushReason::kErrorRecovery) {
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
// Simply clear the background error and return
|
|
|
|
|
recovery_error_ = Status::OK();
|
|
|
|
|
return ClearBGError();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Reset recovery_error_. We will use this to record any errors that happen
|
|
|
|
|
// during the recovery process. While recovering, the only operations that
|
|
|
|
|
// can generate background errors should be the flush operations
|
|
|
|
|
recovery_error_ = Status::OK();
|
2020-12-08 04:09:55 +00:00
|
|
|
recovery_error_.PermitUncheckedError();
|
2020-09-18 03:22:35 +00:00
|
|
|
Status s = db_->ResumeImpl(recover_context_);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
soft_error_no_bg_work_ = false;
|
|
|
|
|
} else {
|
|
|
|
|
soft_error_no_bg_work_ = no_bg_work_original_flag;
|
|
|
|
|
}
|
|
|
|
|
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
// For manual recover, shutdown, and fatal error cases, set
|
|
|
|
|
// recovery_in_prog_ to false. For automatic background recovery, leave it
|
|
|
|
|
// as is regardless of success or failure as it will be retried
|
|
|
|
|
if (is_manual || s.IsShutdownInProgress() ||
|
|
|
|
|
bg_error_.severity() >= Status::Severity::kFatalError) {
|
|
|
|
|
recovery_in_prog_ = false;
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
#else
|
|
|
|
|
(void)is_manual;
|
|
|
|
|
return bg_error_;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
|
2020-12-08 04:09:55 +00:00
|
|
|
const Status& ErrorHandler::StartRecoverFromRetryableBGIOError(
|
|
|
|
|
const IOStatus& io_error) {
|
2020-07-15 18:02:44 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
db_mutex_->AssertHeld();
|
2020-12-08 04:09:55 +00:00
|
|
|
if (bg_error_.ok()) {
|
|
|
|
|
return bg_error_;
|
|
|
|
|
} else if (io_error.ok()) {
|
|
|
|
|
return io_error;
|
|
|
|
|
} else if (db_options_.max_bgerror_resume_count <= 0 || recovery_in_prog_) {
|
2020-07-15 18:02:44 +00:00
|
|
|
// Auto resume BG error is not enabled, directly return bg_error_.
|
|
|
|
|
return bg_error_;
|
2020-12-04 22:57:29 +00:00
|
|
|
}
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_COUNT);
|
|
|
|
|
}
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
db_options_.info_log,
|
|
|
|
|
"ErrorHandler: Call StartRecoverFromRetryableBGIOError to resume\n");
|
2020-12-04 22:57:29 +00:00
|
|
|
if (recovery_thread_) {
|
|
|
|
|
// In this case, if recovery_in_prog_ is false, current thread should
|
|
|
|
|
// wait the previous recover thread to finish and create a new thread
|
|
|
|
|
// to recover from the bg error.
|
|
|
|
|
db_mutex_->Unlock();
|
|
|
|
|
recovery_thread_->join();
|
|
|
|
|
db_mutex_->Lock();
|
2020-07-15 18:02:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
recovery_in_prog_ = true;
|
|
|
|
|
recovery_thread_.reset(
|
|
|
|
|
new port::Thread(&ErrorHandler::RecoverFromRetryableBGIOError, this));
|
|
|
|
|
|
|
|
|
|
if (recovery_io_error_.ok() && recovery_error_.ok()) {
|
2020-12-08 04:09:55 +00:00
|
|
|
return recovery_error_;
|
2020-07-15 18:02:44 +00:00
|
|
|
} else {
|
|
|
|
|
TEST_SYNC_POINT("StartRecoverRetryableBGIOError:RecoverFail");
|
|
|
|
|
return bg_error_;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
(void)io_error;
|
|
|
|
|
return bg_error_;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Automatic recover from Retryable BG IO error. Must be called after db
|
|
|
|
|
// mutex is released.
|
|
|
|
|
void ErrorHandler::RecoverFromRetryableBGIOError() {
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart");
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart1");
|
|
|
|
|
InstrumentedMutexLock l(db_mutex_);
|
|
|
|
|
if (end_recovery_) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
2020-09-18 03:22:35 +00:00
|
|
|
DBRecoverContext context = recover_context_;
|
2020-07-15 18:02:44 +00:00
|
|
|
int resume_count = db_options_.max_bgerror_resume_count;
|
|
|
|
|
uint64_t wait_interval = db_options_.bgerror_resume_retry_interval;
|
2021-03-18 05:36:42 +00:00
|
|
|
uint64_t retry_count = 0;
|
2020-07-15 18:02:44 +00:00
|
|
|
// Recover from the retryable error. Create a separate thread to do it.
|
|
|
|
|
while (resume_count > 0) {
|
|
|
|
|
if (end_recovery_) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume0");
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume1");
|
|
|
|
|
recovery_io_error_ = IOStatus::OK();
|
|
|
|
|
recovery_error_ = Status::OK();
|
2021-03-18 05:36:42 +00:00
|
|
|
retry_count++;
|
2020-09-18 03:22:35 +00:00
|
|
|
Status s = db_->ResumeImpl(context);
|
2020-07-15 18:02:44 +00:00
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:AfterResume0");
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:AfterResume1");
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT);
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
if (s.IsShutdownInProgress() ||
|
|
|
|
|
bg_error_.severity() >= Status::Severity::kFatalError) {
|
|
|
|
|
// If DB shutdown in progress or the error severity is higher than
|
|
|
|
|
// Hard Error, stop auto resume and returns.
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverFail0");
|
|
|
|
|
recovery_in_prog_ = false;
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordInHistogram(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count);
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (!recovery_io_error_.ok() &&
|
|
|
|
|
recovery_error_.severity() <= Status::Severity::kHardError &&
|
|
|
|
|
recovery_io_error_.GetRetryable()) {
|
|
|
|
|
// If new BG IO error happens during auto recovery and it is retryable
|
|
|
|
|
// and its severity is Hard Error or lower, the auto resmue sleep for
|
|
|
|
|
// a period of time and redo auto resume if it is allowed.
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait0");
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait1");
|
2021-03-15 11:32:24 +00:00
|
|
|
int64_t wait_until = db_options_.clock->NowMicros() + wait_interval;
|
2020-07-15 18:02:44 +00:00
|
|
|
cv_.TimedWait(wait_until);
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:AfterWait0");
|
|
|
|
|
} else {
|
|
|
|
|
// There are three possibility: 1) recover_io_error is set during resume
|
|
|
|
|
// and the error is not retryable, 2) recover is successful, 3) other
|
|
|
|
|
// error happens during resume and cannot be resumed here.
|
|
|
|
|
if (recovery_io_error_.ok() && recovery_error_.ok() && s.ok()) {
|
|
|
|
|
// recover from the retryable IO error and no other BG errors. Clean
|
|
|
|
|
// the bg_error and notify user.
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverSuccess");
|
|
|
|
|
Status old_bg_error = bg_error_;
|
|
|
|
|
bg_error_ = Status::OK();
|
2020-12-08 04:09:55 +00:00
|
|
|
bg_error_.PermitUncheckedError();
|
2020-07-15 18:02:44 +00:00
|
|
|
EventHelpers::NotifyOnErrorRecoveryCompleted(db_options_.listeners,
|
|
|
|
|
old_bg_error, db_mutex_);
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordTick(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT);
|
|
|
|
|
RecordInHistogram(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count);
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
recovery_in_prog_ = false;
|
2020-09-18 03:22:35 +00:00
|
|
|
if (soft_error_no_bg_work_) {
|
|
|
|
|
soft_error_no_bg_work_ = false;
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
return;
|
|
|
|
|
} else {
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverFail1");
|
|
|
|
|
// In this case: 1) recovery_io_error is more serious or not retryable
|
|
|
|
|
// 2) other Non IO recovery_error happens. The auto recovery stops.
|
|
|
|
|
recovery_in_prog_ = false;
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordInHistogram(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count);
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
resume_count--;
|
|
|
|
|
}
|
|
|
|
|
recovery_in_prog_ = false;
|
|
|
|
|
TEST_SYNC_POINT("RecoverFromRetryableBGIOError:LoopOut");
|
2021-03-18 05:36:42 +00:00
|
|
|
if (bg_error_stats_ != nullptr) {
|
|
|
|
|
RecordInHistogram(bg_error_stats_.get(),
|
|
|
|
|
ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count);
|
|
|
|
|
}
|
2020-07-15 18:02:44 +00:00
|
|
|
return;
|
|
|
|
|
#else
|
|
|
|
|
return;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ErrorHandler::EndAutoRecovery() {
|
|
|
|
|
db_mutex_->AssertHeld();
|
|
|
|
|
if (!end_recovery_) {
|
|
|
|
|
end_recovery_ = true;
|
|
|
|
|
}
|
|
|
|
|
cv_.SignalAll();
|
|
|
|
|
db_mutex_->Unlock();
|
|
|
|
|
if (recovery_thread_) {
|
|
|
|
|
recovery_thread_->join();
|
|
|
|
|
}
|
|
|
|
|
db_mutex_->Lock();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|