2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-12 22:53:55 +00:00
|
|
|
|
2015-05-28 20:37:47 +00:00
|
|
|
#include "db/event_helpers.h"
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-12 22:53:55 +00:00
|
|
|
|
2021-07-27 14:46:09 +00:00
|
|
|
#include "rocksdb/convenience.h"
|
|
|
|
|
#include "rocksdb/listener.h"
|
|
|
|
|
#include "rocksdb/utilities/customizable_util.h"
|
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2021-07-27 14:46:09 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
Status EventListener::CreateFromString(const ConfigOptions& config_options,
|
|
|
|
|
const std::string& id,
|
|
|
|
|
std::shared_ptr<EventListener>* result) {
|
|
|
|
|
return LoadSharedObject<EventListener>(config_options, id, nullptr, result);
|
|
|
|
|
}
|
|
|
|
|
#endif // ROCKSDB_LITE
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-12 22:53:55 +00:00
|
|
|
|
|
|
|
|
namespace {
|
2018-04-13 00:55:14 +00:00
|
|
|
template <class T>
|
2015-11-19 19:47:12 +00:00
|
|
|
inline T SafeDivide(T a, T b) {
|
|
|
|
|
return b == 0 ? 0 : a / b;
|
|
|
|
|
}
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-12 22:53:55 +00:00
|
|
|
} // namespace
|
|
|
|
|
|
2015-05-28 20:37:47 +00:00
|
|
|
void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
|
2015-05-21 22:39:30 +00:00
|
|
|
*jwriter << "time_micros"
|
|
|
|
|
<< std::chrono::duration_cast<std::chrono::microseconds>(
|
2018-04-13 00:55:14 +00:00
|
|
|
std::chrono::system_clock::now().time_since_epoch())
|
|
|
|
|
.count();
|
2015-05-21 22:39:30 +00:00
|
|
|
}
|
|
|
|
|
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
void EventHelpers::NotifyTableFileCreationStarted(
|
|
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
|
|
|
|
const std::string& db_name, const std::string& cf_name,
|
|
|
|
|
const std::string& file_path, int job_id, TableFileCreationReason reason) {
|
2021-09-17 00:17:40 +00:00
|
|
|
if (listeners.empty()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
TableFileCreationBriefInfo info;
|
|
|
|
|
info.db_name = db_name;
|
|
|
|
|
info.cf_name = cf_name;
|
|
|
|
|
info.file_path = file_path;
|
|
|
|
|
info.job_id = job_id;
|
|
|
|
|
info.reason = reason;
|
|
|
|
|
for (auto& listener : listeners) {
|
|
|
|
|
listener->OnTableFileCreationStarted(info);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
2017-06-23 02:30:39 +00:00
|
|
|
void EventHelpers::NotifyOnBackgroundError(
|
|
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
BackgroundErrorReason reason, Status* bg_error, InstrumentedMutex* db_mutex,
|
|
|
|
|
bool* auto_recovery) {
|
2017-06-23 02:30:39 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2021-09-17 00:17:40 +00:00
|
|
|
if (listeners.empty()) {
|
2017-06-23 02:30:39 +00:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
db_mutex->AssertHeld();
|
|
|
|
|
// release lock while notifying events
|
|
|
|
|
db_mutex->Unlock();
|
|
|
|
|
for (auto& listener : listeners) {
|
|
|
|
|
listener->OnBackgroundError(reason, bg_error);
|
2020-10-02 23:39:17 +00:00
|
|
|
bg_error->PermitUncheckedError();
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
if (*auto_recovery) {
|
|
|
|
|
listener->OnErrorRecoveryBegin(reason, *bg_error, auto_recovery);
|
|
|
|
|
}
|
2017-06-23 02:30:39 +00:00
|
|
|
}
|
|
|
|
|
db_mutex->Lock();
|
2018-04-13 00:55:14 +00:00
|
|
|
#else
|
|
|
|
|
(void)listeners;
|
|
|
|
|
(void)reason;
|
|
|
|
|
(void)bg_error;
|
|
|
|
|
(void)db_mutex;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
(void)auto_recovery;
|
2017-06-23 02:30:39 +00:00
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
}
|
|
|
|
|
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
void EventHelpers::LogAndNotifyTableFileCreationFinished(
|
2015-06-02 21:12:23 +00:00
|
|
|
EventLogger* event_logger,
|
|
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
const std::string& db_name, const std::string& cf_name,
|
|
|
|
|
const std::string& file_path, int job_id, const FileDescriptor& fd,
|
2019-10-14 22:19:31 +00:00
|
|
|
uint64_t oldest_blob_file_number, const TableProperties& table_properties,
|
2020-08-25 17:44:39 +00:00
|
|
|
TableFileCreationReason reason, const Status& s,
|
|
|
|
|
const std::string& file_checksum,
|
|
|
|
|
const std::string& file_checksum_func_name) {
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
if (s.ok() && event_logger) {
|
|
|
|
|
JSONWriter jwriter;
|
|
|
|
|
AppendCurrentTime(&jwriter);
|
|
|
|
|
jwriter << "cf_name" << cf_name << "job" << job_id << "event"
|
|
|
|
|
<< "table_file_creation"
|
|
|
|
|
<< "file_number" << fd.GetNumber() << "file_size"
|
Fix a race condition in WAL tracking causing DB open failure (#9715)
Summary:
There is a race condition if WAL tracking in the MANIFEST is enabled in a database that disables 2PC.
The race condition is between two background flush threads trying to install flush results to the MANIFEST.
Consider an example database with two column families: "default" (cfd0) and "cf1" (cfd1). Initially,
both column families have one mutable (active) memtable whose data backed by 6.log.
1. Trigger a manual flush for "cf1", creating a 7.log
2. Insert another key to "default", and trigger flush for "default", creating 8.log
3. BgFlushThread1 finishes writing 9.sst
4. BgFlushThread2 finishes writing 10.sst
```
Time BgFlushThread1 BgFlushThread2
| mutex_.Lock()
| precompute min_wal_to_keep as 6
| mutex_.Unlock()
| mutex_.Lock()
| precompute min_wal_to_keep as 6
| join MANIFEST write queue and mutex_.Unlock()
| write to MANIFEST
| mutex_.Lock()
| cfd1->log_number = 7
| Signal bg_flush_2 and mutex_.Unlock()
| wake up and mutex_.Lock()
| cfd0->log_number = 8
| FindObsoleteFiles() with job_context->log_number == 7
| mutex_.Unlock()
| PurgeObsoleteFiles() deletes 6.log
V
```
As shown in the above, BgFlushThread2 thinks that the min wal to keep is 6.log because "cf1" has unflushed data in 6.log (cf1.log_number=6).
Similarly, BgThread1 thinks that min wal to keep is also 6.log because "default" has unflushed data (default.log_number=6).
No WAL deletion will be written to MANIFEST because 6 is equal to `versions_->wals_.min_wal_number_to_keep`,
due to https://github.com/facebook/rocksdb/blob/7.1.fb/db/memtable_list.cc#L513:L514.
The bg flush thread that finishes last will perform file purging. `job_context.log_number` will be evaluated as 7, i.e.
the min wal that contains unflushed data, causing 6.log to be deleted. However, MANIFEST thinks 6.log should still exist.
If you close the db at this point, you won't be able to re-open it if `track_and_verify_wal_in_manifest` is true.
We must handle the case of multiple bg flush threads, and it is difficult for one bg flush thread to know
the correct min wal number until the other bg flush threads have finished committing to the manifest and updated
the `cfd::log_number`.
To fix this issue, we rename an existing variable `min_log_number_to_keep_2pc` to `min_log_number_to_keep`,
and use it to track WAL file deletion in non-2pc mode as well.
This variable is updated only 1) during recovery with mutex held, or 2) in the MANIFEST write thread.
`min_log_number_to_keep` means RocksDB will delete WALs below it, although there may be WALs
above it which are also obsolete. Formally, we will have [min_wal_to_keep, max_obsolete_wal]. During recovery, we
make sure that only WALs above max_obsolete_wal are checked and added back to `alive_log_files_`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9715
Test Plan:
```
make check
```
Also ran stress test below (with asan) to make sure it completes successfully.
```
TEST_TMPDIR=/dev/shm/rocksdb OPT=-g ASAN_OPTIONS=disable_coredump=0 \
CRASH_TEST_EXT_ARGS=--compression_type=zstd SKIP_FORMAT_BUCK_CHECKS=1 \
make J=52 -j52 blackbox_asan_crash_test
```
Reviewed By: ltamasi
Differential Revision: D34984412
Pulled By: riversand963
fbshipit-source-id: c7b21a8d84751bb55ea79c9f387103d21b231005
2022-03-24 02:41:31 +00:00
|
|
|
<< fd.GetFileSize() << "file_checksum"
|
|
|
|
|
<< Slice(file_checksum).ToString(true) << "file_checksum_func_name"
|
2022-08-12 20:08:50 +00:00
|
|
|
<< file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno
|
|
|
|
|
<< "largest_seqno" << fd.largest_seqno;
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
|
|
|
|
|
// table_properties
|
|
|
|
|
{
|
|
|
|
|
jwriter << "table_properties";
|
|
|
|
|
jwriter.StartObject();
|
|
|
|
|
|
|
|
|
|
// basic properties:
|
|
|
|
|
jwriter << "data_size" << table_properties.data_size << "index_size"
|
2019-04-11 21:28:08 +00:00
|
|
|
<< table_properties.index_size << "index_partitions"
|
|
|
|
|
<< table_properties.index_partitions << "top_level_index_size"
|
|
|
|
|
<< table_properties.top_level_index_size
|
|
|
|
|
<< "index_key_is_user_key"
|
|
|
|
|
<< table_properties.index_key_is_user_key
|
|
|
|
|
<< "index_value_is_delta_encoded"
|
|
|
|
|
<< table_properties.index_value_is_delta_encoded << "filter_size"
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
<< table_properties.filter_size << "raw_key_size"
|
|
|
|
|
<< table_properties.raw_key_size << "raw_average_key_size"
|
|
|
|
|
<< SafeDivide(table_properties.raw_key_size,
|
|
|
|
|
table_properties.num_entries)
|
|
|
|
|
<< "raw_value_size" << table_properties.raw_value_size
|
|
|
|
|
<< "raw_average_value_size"
|
|
|
|
|
<< SafeDivide(table_properties.raw_value_size,
|
|
|
|
|
table_properties.num_entries)
|
|
|
|
|
<< "num_data_blocks" << table_properties.num_data_blocks
|
|
|
|
|
<< "num_entries" << table_properties.num_entries
|
2021-05-22 00:10:29 +00:00
|
|
|
<< "num_filter_entries" << table_properties.num_filter_entries
|
2019-04-11 21:28:08 +00:00
|
|
|
<< "num_deletions" << table_properties.num_deletions
|
|
|
|
|
<< "num_merge_operands" << table_properties.num_merge_operands
|
Fix wrong info log printing for num_range_deletions (#5617)
Summary:
num_range_deletions printing is wrong in this log line:
2019/07/18-12:59:15.309271 7f869f9ff700 EVENT_LOG_v1 {"time_micros": 1563479955309228, "cf_name": "5", "job": 955, "event": "table_file_creation", "file_number": 34579, "file_size": 2239842, "table_properties": {"data_size": 1988792, "index_size": 3067, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 1, "filter_size": 170821, "raw_key_size": 1951792, "raw_average_key_size": 16, "raw_value_size": 1731720, "raw_average_value_size": 14, "num_data_blocks": 199, "num_entries": 121987, "num_deletions": 15184, "num_merge_operands": 86512, "num_range_deletions": 86512, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "5", "column_family_id": 5, "comparator": "leveldb.BytewiseComparator", "merge_operator": "PutOperator", "prefix_extractor_name": "rocksdb.FixedPrefix.7", "property_collectors": "[]", "compression": "ZSTD", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1563479951, "oldest_key_time": 0, "file_creation_time": 1563479954}}
It actually prints "num_merge_operands" number. Fix it.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5617
Test Plan: Just build.
Differential Revision: D16453110
fbshipit-source-id: fc1024b3cd5650312ed47a1379f0d2cf8b2d8a8f
2019-07-24 02:34:56 +00:00
|
|
|
<< "num_range_deletions" << table_properties.num_range_deletions
|
2019-04-11 21:28:08 +00:00
|
|
|
<< "format_version" << table_properties.format_version
|
|
|
|
|
<< "fixed_key_len" << table_properties.fixed_key_len
|
|
|
|
|
<< "filter_policy" << table_properties.filter_policy_name
|
|
|
|
|
<< "column_family_name" << table_properties.column_family_name
|
|
|
|
|
<< "column_family_id" << table_properties.column_family_id
|
|
|
|
|
<< "comparator" << table_properties.comparator_name
|
|
|
|
|
<< "merge_operator" << table_properties.merge_operator_name
|
|
|
|
|
<< "prefix_extractor_name"
|
|
|
|
|
<< table_properties.prefix_extractor_name << "property_collectors"
|
|
|
|
|
<< table_properties.property_collectors_names << "compression"
|
|
|
|
|
<< table_properties.compression_name << "compression_options"
|
|
|
|
|
<< table_properties.compression_options << "creation_time"
|
|
|
|
|
<< table_properties.creation_time << "oldest_key_time"
|
2019-04-22 22:24:04 +00:00
|
|
|
<< table_properties.oldest_key_time << "file_creation_time"
|
2021-04-01 01:20:44 +00:00
|
|
|
<< table_properties.file_creation_time
|
|
|
|
|
<< "slow_compression_estimated_data_size"
|
|
|
|
|
<< table_properties.slow_compression_estimated_data_size
|
|
|
|
|
<< "fast_compression_estimated_data_size"
|
|
|
|
|
<< table_properties.fast_compression_estimated_data_size
|
|
|
|
|
<< "db_id" << table_properties.db_id << "db_session_id"
|
2021-08-21 03:39:52 +00:00
|
|
|
<< table_properties.db_session_id << "orig_file_number"
|
2022-07-16 02:01:30 +00:00
|
|
|
<< table_properties.orig_file_number << "seqno_to_time_mapping";
|
|
|
|
|
|
|
|
|
|
if (table_properties.seqno_to_time_mapping.empty()) {
|
|
|
|
|
jwriter << "N/A";
|
|
|
|
|
} else {
|
|
|
|
|
SeqnoToTimeMapping tmp;
|
|
|
|
|
Status status = tmp.Add(table_properties.seqno_to_time_mapping);
|
|
|
|
|
if (status.ok()) {
|
|
|
|
|
jwriter << tmp.ToHumanString();
|
|
|
|
|
} else {
|
|
|
|
|
jwriter << "Invalid";
|
|
|
|
|
}
|
|
|
|
|
}
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
|
|
|
|
|
// user collected properties
|
|
|
|
|
for (const auto& prop : table_properties.readable_properties) {
|
|
|
|
|
jwriter << prop.first << prop.second;
|
|
|
|
|
}
|
|
|
|
|
jwriter.EndObject();
|
2015-05-21 22:39:30 +00:00
|
|
|
}
|
2019-10-14 22:19:31 +00:00
|
|
|
|
|
|
|
|
if (oldest_blob_file_number != kInvalidBlobFileNumber) {
|
|
|
|
|
jwriter << "oldest_blob_file_number" << oldest_blob_file_number;
|
|
|
|
|
}
|
|
|
|
|
|
2015-05-21 22:39:30 +00:00
|
|
|
jwriter.EndObject();
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-12 22:53:55 +00:00
|
|
|
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
event_logger->Log(jwriter);
|
|
|
|
|
}
|
2015-06-02 21:12:23 +00:00
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
2021-09-17 00:17:40 +00:00
|
|
|
if (listeners.empty()) {
|
2015-06-02 21:12:23 +00:00
|
|
|
return;
|
|
|
|
|
}
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
TableFileCreationInfo info;
|
|
|
|
|
info.db_name = db_name;
|
|
|
|
|
info.cf_name = cf_name;
|
|
|
|
|
info.file_path = file_path;
|
|
|
|
|
info.file_size = fd.file_size;
|
|
|
|
|
info.job_id = job_id;
|
|
|
|
|
info.table_properties = table_properties;
|
|
|
|
|
info.reason = reason;
|
|
|
|
|
info.status = s;
|
2020-08-25 17:44:39 +00:00
|
|
|
info.file_checksum = file_checksum;
|
|
|
|
|
info.file_checksum_func_name = file_checksum_func_name;
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 18:35:00 +00:00
|
|
|
for (auto& listener : listeners) {
|
2015-06-02 21:12:23 +00:00
|
|
|
listener->OnTableFileCreated(info);
|
|
|
|
|
}
|
2020-09-16 22:45:30 +00:00
|
|
|
info.status.PermitUncheckedError();
|
2018-04-13 00:55:14 +00:00
|
|
|
#else
|
|
|
|
|
(void)listeners;
|
|
|
|
|
(void)db_name;
|
|
|
|
|
(void)cf_name;
|
|
|
|
|
(void)file_path;
|
|
|
|
|
(void)reason;
|
2015-06-02 21:12:23 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-12 22:53:55 +00:00
|
|
|
}
|
|
|
|
|
|
2015-06-04 02:57:01 +00:00
|
|
|
void EventHelpers::LogAndNotifyTableFileDeletion(
|
2018-04-13 00:55:14 +00:00
|
|
|
EventLogger* event_logger, int job_id, uint64_t file_number,
|
|
|
|
|
const std::string& file_path, const Status& status,
|
|
|
|
|
const std::string& dbname,
|
2015-06-04 02:57:01 +00:00
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners) {
|
|
|
|
|
JSONWriter jwriter;
|
|
|
|
|
AppendCurrentTime(&jwriter);
|
|
|
|
|
|
2018-04-13 00:55:14 +00:00
|
|
|
jwriter << "job" << job_id << "event"
|
|
|
|
|
<< "table_file_deletion"
|
2015-06-04 02:57:01 +00:00
|
|
|
<< "file_number" << file_number;
|
|
|
|
|
if (!status.ok()) {
|
|
|
|
|
jwriter << "status" << status.ToString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
jwriter.EndObject();
|
|
|
|
|
|
|
|
|
|
event_logger->Log(jwriter);
|
|
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
2021-09-17 00:17:40 +00:00
|
|
|
if (listeners.empty()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
2015-06-04 02:57:01 +00:00
|
|
|
TableFileDeletionInfo info;
|
|
|
|
|
info.db_name = dbname;
|
|
|
|
|
info.job_id = job_id;
|
|
|
|
|
info.file_path = file_path;
|
|
|
|
|
info.status = status;
|
2017-01-11 18:38:07 +00:00
|
|
|
for (auto& listener : listeners) {
|
2015-06-04 02:57:01 +00:00
|
|
|
listener->OnTableFileDeleted(info);
|
|
|
|
|
}
|
2020-09-16 22:45:30 +00:00
|
|
|
info.status.PermitUncheckedError();
|
2018-04-13 00:55:14 +00:00
|
|
|
#else
|
|
|
|
|
(void)file_path;
|
|
|
|
|
(void)dbname;
|
|
|
|
|
(void)listeners;
|
2015-06-04 02:57:01 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
}
|
|
|
|
|
|
2021-12-08 22:22:41 +00:00
|
|
|
void EventHelpers::NotifyOnErrorRecoveryEnd(
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
2021-12-08 22:22:41 +00:00
|
|
|
const Status& old_bg_error, const Status& new_bg_error,
|
|
|
|
|
InstrumentedMutex* db_mutex) {
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2021-09-17 00:17:40 +00:00
|
|
|
if (!listeners.empty()) {
|
2021-01-06 22:14:01 +00:00
|
|
|
db_mutex->AssertHeld();
|
|
|
|
|
// release lock while notifying events
|
|
|
|
|
db_mutex->Unlock();
|
|
|
|
|
for (auto& listener : listeners) {
|
2021-12-08 22:22:41 +00:00
|
|
|
BackgroundErrorRecoveryInfo info;
|
|
|
|
|
info.old_bg_error = old_bg_error;
|
|
|
|
|
info.new_bg_error = new_bg_error;
|
2021-01-06 22:14:01 +00:00
|
|
|
listener->OnErrorRecoveryCompleted(old_bg_error);
|
2021-12-08 22:22:41 +00:00
|
|
|
listener->OnErrorRecoveryEnd(info);
|
|
|
|
|
info.old_bg_error.PermitUncheckedError();
|
|
|
|
|
info.new_bg_error.PermitUncheckedError();
|
2021-01-06 22:14:01 +00:00
|
|
|
}
|
|
|
|
|
db_mutex->Lock();
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
(void)listeners;
|
|
|
|
|
(void)old_bg_error;
|
2021-12-08 22:22:41 +00:00
|
|
|
(void)new_bg_error;
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 20:36:19 +00:00
|
|
|
(void)db_mutex;
|
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-17 00:17:40 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
void EventHelpers::NotifyBlobFileCreationStarted(
|
|
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
|
|
|
|
const std::string& db_name, const std::string& cf_name,
|
|
|
|
|
const std::string& file_path, int job_id,
|
|
|
|
|
BlobFileCreationReason creation_reason) {
|
|
|
|
|
if (listeners.empty()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
BlobFileCreationBriefInfo info(db_name, cf_name, file_path, job_id,
|
|
|
|
|
creation_reason);
|
|
|
|
|
for (const auto& listener : listeners) {
|
|
|
|
|
listener->OnBlobFileCreationStarted(info);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
|
|
void EventHelpers::LogAndNotifyBlobFileCreationFinished(
|
|
|
|
|
EventLogger* event_logger,
|
|
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
|
|
|
|
const std::string& db_name, const std::string& cf_name,
|
|
|
|
|
const std::string& file_path, int job_id, uint64_t file_number,
|
|
|
|
|
BlobFileCreationReason creation_reason, const Status& s,
|
|
|
|
|
const std::string& file_checksum,
|
|
|
|
|
const std::string& file_checksum_func_name, uint64_t total_blob_count,
|
|
|
|
|
uint64_t total_blob_bytes) {
|
|
|
|
|
if (s.ok() && event_logger) {
|
|
|
|
|
JSONWriter jwriter;
|
|
|
|
|
AppendCurrentTime(&jwriter);
|
|
|
|
|
jwriter << "cf_name" << cf_name << "job" << job_id << "event"
|
|
|
|
|
<< "blob_file_creation"
|
|
|
|
|
<< "file_number" << file_number << "total_blob_count"
|
|
|
|
|
<< total_blob_count << "total_blob_bytes" << total_blob_bytes
|
|
|
|
|
<< "file_checksum" << file_checksum << "file_checksum_func_name"
|
|
|
|
|
<< file_checksum_func_name << "status" << s.ToString();
|
|
|
|
|
|
|
|
|
|
jwriter.EndObject();
|
|
|
|
|
event_logger->Log(jwriter);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
if (listeners.empty()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
BlobFileCreationInfo info(db_name, cf_name, file_path, job_id,
|
|
|
|
|
creation_reason, total_blob_count, total_blob_bytes,
|
|
|
|
|
s, file_checksum, file_checksum_func_name);
|
|
|
|
|
for (const auto& listener : listeners) {
|
|
|
|
|
listener->OnBlobFileCreated(info);
|
|
|
|
|
}
|
|
|
|
|
info.status.PermitUncheckedError();
|
|
|
|
|
#else
|
|
|
|
|
(void)listeners;
|
|
|
|
|
(void)db_name;
|
|
|
|
|
(void)file_path;
|
|
|
|
|
(void)creation_reason;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void EventHelpers::LogAndNotifyBlobFileDeletion(
|
|
|
|
|
EventLogger* event_logger,
|
|
|
|
|
const std::vector<std::shared_ptr<EventListener>>& listeners, int job_id,
|
|
|
|
|
uint64_t file_number, const std::string& file_path, const Status& status,
|
|
|
|
|
const std::string& dbname) {
|
|
|
|
|
if (event_logger) {
|
|
|
|
|
JSONWriter jwriter;
|
|
|
|
|
AppendCurrentTime(&jwriter);
|
|
|
|
|
|
|
|
|
|
jwriter << "job" << job_id << "event"
|
|
|
|
|
<< "blob_file_deletion"
|
|
|
|
|
<< "file_number" << file_number;
|
|
|
|
|
if (!status.ok()) {
|
|
|
|
|
jwriter << "status" << status.ToString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
jwriter.EndObject();
|
|
|
|
|
event_logger->Log(jwriter);
|
|
|
|
|
}
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
if (listeners.empty()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
BlobFileDeletionInfo info(dbname, file_path, job_id, status);
|
|
|
|
|
for (const auto& listener : listeners) {
|
|
|
|
|
listener->OnBlobFileDeleted(info);
|
|
|
|
|
}
|
|
|
|
|
info.status.PermitUncheckedError();
|
|
|
|
|
#else
|
|
|
|
|
(void)listeners;
|
|
|
|
|
(void)dbname;
|
|
|
|
|
(void)file_path;
|
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
}
|
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|