2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2013-10-16 21:59:46 +00:00
|
|
|
//
|
2011-03-18 22:37:00 +00:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#include "db/version_edit.h"
|
|
|
|
|
2019-10-14 22:19:31 +00:00
|
|
|
#include "db/blob_index.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
#include "db/version_set.h"
|
2019-06-01 00:19:43 +00:00
|
|
|
#include "logging/event_logger.h"
|
2017-04-06 21:49:13 +00:00
|
|
|
#include "rocksdb/slice.h"
|
2019-05-31 00:39:43 +00:00
|
|
|
#include "test_util/sync_point.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
#include "util/coding.h"
|
2017-04-06 21:49:13 +00:00
|
|
|
#include "util/string_util.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
namespace rocksdb {
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2019-09-03 15:50:47 +00:00
|
|
|
// Mask for an identified tag from the future which can be safely ignored.
|
|
|
|
const uint32_t kTagSafeIgnoreMask = 1 << 13;
|
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
// Tag numbers for serialized VersionEdit. These numbers are written to
|
2019-09-03 15:50:47 +00:00
|
|
|
// disk and should not be changed. The number should be forward compatible so
|
|
|
|
// users can down-grade RocksDB safely. A future Tag is ignored by doing '&'
|
|
|
|
// between Tag and kTagSafeIgnoreMask field.
|
2018-05-23 22:59:16 +00:00
|
|
|
enum Tag : uint32_t {
|
2014-07-02 16:54:20 +00:00
|
|
|
kComparator = 1,
|
|
|
|
kLogNumber = 2,
|
|
|
|
kNextFileNumber = 3,
|
|
|
|
kLastSequence = 4,
|
|
|
|
kCompactPointer = 5,
|
|
|
|
kDeletedFile = 6,
|
|
|
|
kNewFile = 7,
|
2011-04-20 22:48:11 +00:00
|
|
|
// 8 was used for large value refs
|
2014-07-02 16:54:20 +00:00
|
|
|
kPrevLogNumber = 9,
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
kMinLogNumberToKeep = 10,
|
2019-09-03 15:50:47 +00:00
|
|
|
// Ignore-able field
|
|
|
|
kDbId = kTagSafeIgnoreMask + 1,
|
2013-06-14 05:09:08 +00:00
|
|
|
|
|
|
|
// these are new formats divergent from open source leveldb
|
2014-07-02 16:54:20 +00:00
|
|
|
kNewFile2 = 100,
|
|
|
|
kNewFile3 = 102,
|
2015-10-03 00:32:46 +00:00
|
|
|
kNewFile4 = 103, // 4th (the latest) format version of adding files
|
2014-07-02 16:54:20 +00:00
|
|
|
kColumnFamily = 200, // specify column family for version edit
|
|
|
|
kColumnFamilyAdd = 201,
|
|
|
|
kColumnFamilyDrop = 202,
|
|
|
|
kMaxColumnFamily = 203,
|
2018-08-20 21:54:03 +00:00
|
|
|
|
|
|
|
kInAtomicGroup = 300,
|
2011-03-18 22:37:00 +00:00
|
|
|
};
|
|
|
|
|
2018-05-23 22:59:16 +00:00
|
|
|
enum CustomTag : uint32_t {
|
2015-10-03 00:32:46 +00:00
|
|
|
kTerminate = 1, // The end of customized fields
|
|
|
|
kNeedCompaction = 2,
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
// Since Manifest is not entirely currently forward-compatible, and the only
|
2018-08-09 23:49:45 +00:00
|
|
|
// forward-compatible part is the CutsomtTag of kNewFile, we currently encode
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
// kMinLogNumberToKeep as part of a CustomTag as a hack. This should be
|
|
|
|
// removed when manifest becomes forward-comptabile.
|
|
|
|
kMinLogNumberToKeepHack = 3,
|
2019-10-14 22:19:31 +00:00
|
|
|
kOldestBlobFileNumber = 4,
|
2019-11-23 00:01:21 +00:00
|
|
|
kOldestAncesterTime = 5,
|
2019-11-27 05:38:38 +00:00
|
|
|
kFileCreationTime = 6,
|
2015-10-03 00:32:46 +00:00
|
|
|
kPathId = 65,
|
|
|
|
};
|
|
|
|
// If this bit for the custom tag is set, opening DB should fail if
|
|
|
|
// we don't know this field.
|
|
|
|
uint32_t kCustomTagNonSafeIgnoreMask = 1 << 6;
|
|
|
|
|
2014-07-02 16:54:20 +00:00
|
|
|
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
|
|
|
|
assert(number <= kFileNumberMask);
|
|
|
|
return number | (path_id * (kFileNumberMask + 1));
|
|
|
|
}
|
|
|
|
|
2019-10-14 22:19:31 +00:00
|
|
|
void FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value,
|
|
|
|
SequenceNumber seqno,
|
|
|
|
ValueType value_type) {
|
|
|
|
if (smallest.size() == 0) {
|
|
|
|
smallest.DecodeFrom(key);
|
|
|
|
}
|
|
|
|
largest.DecodeFrom(key);
|
|
|
|
fd.smallest_seqno = std::min(fd.smallest_seqno, seqno);
|
|
|
|
fd.largest_seqno = std::max(fd.largest_seqno, seqno);
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (value_type == kTypeBlobIndex) {
|
|
|
|
BlobIndex blob_index;
|
|
|
|
const Status s = blob_index.DecodeFrom(value);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (blob_index.IsInlined()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (blob_index.HasTTL()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Paranoid check: this should not happen because BlobDB numbers the blob
|
|
|
|
// files starting from 1.
|
|
|
|
if (blob_index.file_number() == kInvalidBlobFileNumber) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (oldest_blob_file_number == kInvalidBlobFileNumber ||
|
|
|
|
oldest_blob_file_number > blob_index.file_number()) {
|
|
|
|
oldest_blob_file_number = blob_index.file_number();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
(void)value;
|
|
|
|
(void)value_type;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
void VersionEdit::Clear() {
|
2019-09-03 15:50:47 +00:00
|
|
|
db_id_.clear();
|
2011-03-18 22:37:00 +00:00
|
|
|
comparator_.clear();
|
2014-01-14 23:27:09 +00:00
|
|
|
max_level_ = 0;
|
2011-03-18 22:37:00 +00:00
|
|
|
log_number_ = 0;
|
2011-04-12 19:38:58 +00:00
|
|
|
prev_log_number_ = 0;
|
2011-03-18 22:37:00 +00:00
|
|
|
last_sequence_ = 0;
|
|
|
|
next_file_number_ = 0;
|
2014-03-05 20:13:44 +00:00
|
|
|
max_column_family_ = 0;
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
min_log_number_to_keep_ = 0;
|
2019-09-03 15:50:47 +00:00
|
|
|
has_db_id_ = false;
|
2011-03-18 22:37:00 +00:00
|
|
|
has_comparator_ = false;
|
|
|
|
has_log_number_ = false;
|
2011-04-12 19:38:58 +00:00
|
|
|
has_prev_log_number_ = false;
|
2011-03-18 22:37:00 +00:00
|
|
|
has_next_file_number_ = false;
|
|
|
|
has_last_sequence_ = false;
|
2014-03-05 20:13:44 +00:00
|
|
|
has_max_column_family_ = false;
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
has_min_log_number_to_keep_ = false;
|
2011-03-18 22:37:00 +00:00
|
|
|
deleted_files_.clear();
|
|
|
|
new_files_.clear();
|
2013-12-12 01:46:26 +00:00
|
|
|
column_family_ = 0;
|
|
|
|
is_column_family_add_ = 0;
|
|
|
|
is_column_family_drop_ = 0;
|
|
|
|
column_family_name_.clear();
|
2018-08-20 21:54:03 +00:00
|
|
|
is_in_atomic_group_ = false;
|
|
|
|
remaining_entries_ = 0;
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
|
2014-10-28 21:27:26 +00:00
|
|
|
bool VersionEdit::EncodeTo(std::string* dst) const {
|
2019-09-03 15:50:47 +00:00
|
|
|
if (has_db_id_) {
|
|
|
|
PutVarint32(dst, kDbId);
|
|
|
|
PutLengthPrefixedSlice(dst, db_id_);
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
if (has_comparator_) {
|
|
|
|
PutVarint32(dst, kComparator);
|
|
|
|
PutLengthPrefixedSlice(dst, comparator_);
|
|
|
|
}
|
|
|
|
if (has_log_number_) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint64(dst, kLogNumber, log_number_);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
2011-04-12 19:38:58 +00:00
|
|
|
if (has_prev_log_number_) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint64(dst, kPrevLogNumber, prev_log_number_);
|
2011-04-12 19:38:58 +00:00
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
if (has_next_file_number_) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint64(dst, kNextFileNumber, next_file_number_);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
if (has_last_sequence_) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint64(dst, kLastSequence, last_sequence_);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
2014-03-05 20:13:44 +00:00
|
|
|
if (has_max_column_family_) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint32(dst, kMaxColumnFamily, max_column_family_);
|
2014-03-05 20:13:44 +00:00
|
|
|
}
|
2013-12-31 02:33:57 +00:00
|
|
|
for (const auto& deleted : deleted_files_) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint32Varint64(dst, kDeletedFile, deleted.first /* level */,
|
|
|
|
deleted.second /* file number */);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
bool min_log_num_written = false;
|
2011-04-20 22:48:11 +00:00
|
|
|
for (size_t i = 0; i < new_files_.size(); i++) {
|
2011-03-18 22:37:00 +00:00
|
|
|
const FileMetaData& f = new_files_[i].second;
|
2014-10-29 18:21:51 +00:00
|
|
|
if (!f.smallest.Valid() || !f.largest.Valid()) {
|
|
|
|
return false;
|
|
|
|
}
|
2019-11-23 00:01:21 +00:00
|
|
|
PutVarint32(dst, kNewFile4);
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint64(dst, new_files_[i].first /* level */, f.fd.GetNumber());
|
2014-06-13 22:54:19 +00:00
|
|
|
PutVarint64(dst, f.fd.GetFileSize());
|
2011-03-18 22:37:00 +00:00
|
|
|
PutLengthPrefixedSlice(dst, f.smallest.Encode());
|
|
|
|
PutLengthPrefixedSlice(dst, f.largest.Encode());
|
2018-07-27 23:00:26 +00:00
|
|
|
PutVarint64Varint64(dst, f.fd.smallest_seqno, f.fd.largest_seqno);
|
2019-11-23 00:01:21 +00:00
|
|
|
// Customized fields' format:
|
|
|
|
// +-----------------------------+
|
|
|
|
// | 1st field's tag (varint32) |
|
|
|
|
// +-----------------------------+
|
|
|
|
// | 1st field's size (varint32) |
|
|
|
|
// +-----------------------------+
|
|
|
|
// | bytes for 1st field |
|
|
|
|
// | (based on size decoded) |
|
|
|
|
// +-----------------------------+
|
|
|
|
// | |
|
|
|
|
// | ...... |
|
|
|
|
// | |
|
|
|
|
// +-----------------------------+
|
|
|
|
// | last field's size (varint32)|
|
|
|
|
// +-----------------------------+
|
|
|
|
// | bytes for last field |
|
|
|
|
// | (based on size decoded) |
|
|
|
|
// +-----------------------------+
|
|
|
|
// | terminating tag (varint32) |
|
|
|
|
// +-----------------------------+
|
|
|
|
//
|
|
|
|
// Customized encoding for fields:
|
|
|
|
// tag kPathId: 1 byte as path_id
|
|
|
|
// tag kNeedCompaction:
|
|
|
|
// now only can take one char value 1 indicating need-compaction
|
|
|
|
//
|
|
|
|
PutVarint32(dst, CustomTag::kOldestAncesterTime);
|
|
|
|
std::string varint_oldest_ancester_time;
|
|
|
|
PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time);
|
|
|
|
TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime",
|
|
|
|
&varint_oldest_ancester_time);
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time));
|
2019-11-27 05:38:38 +00:00
|
|
|
|
|
|
|
PutVarint32(dst, CustomTag::kFileCreationTime);
|
|
|
|
std::string varint_file_creation_time;
|
|
|
|
PutVarint64(&varint_file_creation_time, f.file_creation_time);
|
|
|
|
TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime",
|
|
|
|
&varint_file_creation_time);
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time));
|
|
|
|
|
2019-11-23 00:01:21 +00:00
|
|
|
if (f.fd.GetPathId() != 0) {
|
|
|
|
PutVarint32(dst, CustomTag::kPathId);
|
|
|
|
char p = static_cast<char>(f.fd.GetPathId());
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(&p, 1));
|
2015-10-03 00:32:46 +00:00
|
|
|
}
|
2019-11-23 00:01:21 +00:00
|
|
|
if (f.marked_for_compaction) {
|
|
|
|
PutVarint32(dst, CustomTag::kNeedCompaction);
|
|
|
|
char p = static_cast<char>(1);
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(&p, 1));
|
|
|
|
}
|
|
|
|
if (has_min_log_number_to_keep_ && !min_log_num_written) {
|
|
|
|
PutVarint32(dst, CustomTag::kMinLogNumberToKeepHack);
|
|
|
|
std::string varint_log_number;
|
|
|
|
PutFixed64(&varint_log_number, min_log_number_to_keep_);
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(varint_log_number));
|
|
|
|
min_log_num_written = true;
|
|
|
|
}
|
|
|
|
if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
|
|
|
|
PutVarint32(dst, CustomTag::kOldestBlobFileNumber);
|
|
|
|
std::string oldest_blob_file_number;
|
|
|
|
PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number);
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number));
|
|
|
|
}
|
|
|
|
TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
|
|
|
|
dst);
|
|
|
|
|
|
|
|
PutVarint32(dst, CustomTag::kTerminate);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
2013-12-12 01:46:26 +00:00
|
|
|
|
|
|
|
// 0 is default and does not need to be explicitly written
|
|
|
|
if (column_family_ != 0) {
|
2016-06-13 16:57:43 +00:00
|
|
|
PutVarint32Varint32(dst, kColumnFamily, column_family_);
|
2013-12-12 01:46:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (is_column_family_add_) {
|
|
|
|
PutVarint32(dst, kColumnFamilyAdd);
|
|
|
|
PutLengthPrefixedSlice(dst, Slice(column_family_name_));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (is_column_family_drop_) {
|
|
|
|
PutVarint32(dst, kColumnFamilyDrop);
|
|
|
|
}
|
2018-08-20 21:54:03 +00:00
|
|
|
|
|
|
|
if (is_in_atomic_group_) {
|
|
|
|
PutVarint32(dst, kInAtomicGroup);
|
|
|
|
PutVarint32(dst, remaining_entries_);
|
|
|
|
}
|
2014-10-28 21:27:26 +00:00
|
|
|
return true;
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool GetInternalKey(Slice* input, InternalKey* dst) {
|
|
|
|
Slice str;
|
|
|
|
if (GetLengthPrefixedSlice(input, &str)) {
|
|
|
|
dst->DecodeFrom(str);
|
[fix] SIGSEGV when VersionEdit in MANIFEST is corrupted
Summary:
This was reported by our customers in task #4295529.
Cause:
* MANIFEST file contains a VersionEdit, which contains file entries whose 'smallest' and 'largest' internal keys are empty. String with zero characters. Root cause of corruption was not investigated. We should report corruption when this happens. However, we currently SIGSEGV.
Here's what happens:
* VersionEdit encodes zero-strings happily and stores them in smallest and largest InternalKeys. InternalKey::Encode() does assert when `rep_.empty()`, but we don't assert in production environemnts. Also, we should never assert as a result of DB corruption.
* As part of our ConsistencyCheck, we call GetLiveFilesMetaData()
* GetLiveFilesMetadata() calls `file->largest.user_key().ToString()`
* user_key() function does: 1. assert(size > 8) (ooops, no assert), 2. returns `Slice(internal_key.data(), internal_key.size() - 8)`
* since `internal_key.size()` is unsigned int, this call translates to `Slice(whatever, 1298471928561892576182756)`. Bazinga.
Fix:
* VersionEdit checks if InternalKey is valid in `VersionEdit::GetInternalKey()`. If it's invalid, returns corruption.
Lessons learned:
* Always keep in mind that even if you `assert()`, production code will continue execution even if assert fails.
* Never `assert` based on DB corruption. Assert only if the code should guarantee that assert can't fail.
Test Plan: dumped offending manifest. Before: assert. Now: corruption
Reviewers: dhruba, haobo, sdong
Reviewed By: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18507
2014-05-07 23:52:12 +00:00
|
|
|
return dst->Valid();
|
2011-03-18 22:37:00 +00:00
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-05 21:08:17 +00:00
|
|
|
bool VersionEdit::GetLevel(Slice* input, int* level, const char** /*msg*/) {
|
2011-03-18 22:37:00 +00:00
|
|
|
uint32_t v;
|
2014-01-14 23:27:09 +00:00
|
|
|
if (GetVarint32(input, &v)) {
|
2011-03-18 22:37:00 +00:00
|
|
|
*level = v;
|
2014-01-14 23:27:09 +00:00
|
|
|
if (max_level_ < *level) {
|
|
|
|
max_level_ = *level;
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-03 00:32:46 +00:00
|
|
|
const char* VersionEdit::DecodeNewFile4From(Slice* input) {
|
|
|
|
const char* msg = nullptr;
|
|
|
|
int level;
|
|
|
|
FileMetaData f;
|
|
|
|
uint64_t number;
|
|
|
|
uint32_t path_id = 0;
|
|
|
|
uint64_t file_size;
|
2018-07-27 23:00:26 +00:00
|
|
|
SequenceNumber smallest_seqno;
|
|
|
|
SequenceNumber largest_seqno;
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
// Since this is the only forward-compatible part of the code, we hack new
|
|
|
|
// extension into this record. When we do, we set this boolean to distinguish
|
|
|
|
// the record from the normal NewFile records.
|
2015-10-03 00:32:46 +00:00
|
|
|
if (GetLevel(input, &level, &msg) && GetVarint64(input, &number) &&
|
|
|
|
GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) &&
|
|
|
|
GetInternalKey(input, &f.largest) &&
|
2018-07-27 23:00:26 +00:00
|
|
|
GetVarint64(input, &smallest_seqno) &&
|
|
|
|
GetVarint64(input, &largest_seqno)) {
|
2015-10-03 00:32:46 +00:00
|
|
|
// See comments in VersionEdit::EncodeTo() for format of customized fields
|
|
|
|
while (true) {
|
|
|
|
uint32_t custom_tag;
|
|
|
|
Slice field;
|
|
|
|
if (!GetVarint32(input, &custom_tag)) {
|
|
|
|
return "new-file4 custom field";
|
|
|
|
}
|
|
|
|
if (custom_tag == kTerminate) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!GetLengthPrefixedSlice(input, &field)) {
|
2019-01-02 19:15:01 +00:00
|
|
|
return "new-file4 custom field length prefixed slice error";
|
2015-10-03 00:32:46 +00:00
|
|
|
}
|
|
|
|
switch (custom_tag) {
|
|
|
|
case kPathId:
|
|
|
|
if (field.size() != 1) {
|
|
|
|
return "path_id field wrong size";
|
|
|
|
}
|
|
|
|
path_id = field[0];
|
|
|
|
if (path_id > 3) {
|
|
|
|
return "path_id wrong vaue";
|
|
|
|
}
|
|
|
|
break;
|
2019-11-23 00:01:21 +00:00
|
|
|
case kOldestAncesterTime:
|
|
|
|
if (!GetVarint64(&field, &f.oldest_ancester_time)) {
|
|
|
|
return "invalid oldest ancester time";
|
|
|
|
}
|
|
|
|
break;
|
2019-11-27 05:38:38 +00:00
|
|
|
case kFileCreationTime:
|
|
|
|
if (!GetVarint64(&field, &f.file_creation_time)) {
|
|
|
|
return "invalid file creation time";
|
|
|
|
}
|
|
|
|
break;
|
2015-10-03 00:32:46 +00:00
|
|
|
case kNeedCompaction:
|
|
|
|
if (field.size() != 1) {
|
|
|
|
return "need_compaction field wrong size";
|
|
|
|
}
|
|
|
|
f.marked_for_compaction = (field[0] == 1);
|
|
|
|
break;
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
case kMinLogNumberToKeepHack:
|
|
|
|
// This is a hack to encode kMinLogNumberToKeep in a
|
2018-08-09 23:49:45 +00:00
|
|
|
// forward-compatible fashion.
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
if (!GetFixed64(&field, &min_log_number_to_keep_)) {
|
|
|
|
return "deleted log number malformatted";
|
|
|
|
}
|
|
|
|
has_min_log_number_to_keep_ = true;
|
|
|
|
break;
|
2019-10-14 22:19:31 +00:00
|
|
|
case kOldestBlobFileNumber:
|
|
|
|
if (!GetVarint64(&field, &f.oldest_blob_file_number)) {
|
|
|
|
return "invalid oldest blob file number";
|
|
|
|
}
|
|
|
|
break;
|
2015-10-03 00:32:46 +00:00
|
|
|
default:
|
|
|
|
if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) {
|
|
|
|
// Should not proceed if cannot understand it
|
|
|
|
return "new-file4 custom field not supported";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return "new-file4 entry";
|
|
|
|
}
|
2018-07-27 23:00:26 +00:00
|
|
|
f.fd =
|
|
|
|
FileDescriptor(number, path_id, file_size, smallest_seqno, largest_seqno);
|
2015-10-03 00:32:46 +00:00
|
|
|
new_files_.push_back(std::make_pair(level, f));
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
Status VersionEdit::DecodeFrom(const Slice& src) {
|
|
|
|
Clear();
|
|
|
|
Slice input = src;
|
2013-03-01 02:04:58 +00:00
|
|
|
const char* msg = nullptr;
|
2011-03-18 22:37:00 +00:00
|
|
|
uint32_t tag;
|
|
|
|
|
|
|
|
// Temporary storage for parsing
|
|
|
|
int level;
|
|
|
|
FileMetaData f;
|
|
|
|
Slice str;
|
|
|
|
InternalKey key;
|
2013-03-01 02:04:58 +00:00
|
|
|
while (msg == nullptr && GetVarint32(&input, &tag)) {
|
2011-03-18 22:37:00 +00:00
|
|
|
switch (tag) {
|
2019-09-03 15:50:47 +00:00
|
|
|
case kDbId:
|
|
|
|
if (GetLengthPrefixedSlice(&input, &str)) {
|
|
|
|
db_id_ = str.ToString();
|
|
|
|
has_db_id_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "db id";
|
|
|
|
}
|
|
|
|
break;
|
2011-03-18 22:37:00 +00:00
|
|
|
case kComparator:
|
|
|
|
if (GetLengthPrefixedSlice(&input, &str)) {
|
|
|
|
comparator_ = str.ToString();
|
|
|
|
has_comparator_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "comparator name";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case kLogNumber:
|
|
|
|
if (GetVarint64(&input, &log_number_)) {
|
|
|
|
has_log_number_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "log number";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2011-04-12 19:38:58 +00:00
|
|
|
case kPrevLogNumber:
|
|
|
|
if (GetVarint64(&input, &prev_log_number_)) {
|
|
|
|
has_prev_log_number_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "previous log number";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
case kNextFileNumber:
|
|
|
|
if (GetVarint64(&input, &next_file_number_)) {
|
|
|
|
has_next_file_number_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "next file number";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case kLastSequence:
|
|
|
|
if (GetVarint64(&input, &last_sequence_)) {
|
|
|
|
has_last_sequence_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "last sequence number";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2014-03-05 20:13:44 +00:00
|
|
|
case kMaxColumnFamily:
|
|
|
|
if (GetVarint32(&input, &max_column_family_)) {
|
|
|
|
has_max_column_family_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "max column family";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
case kMinLogNumberToKeep:
|
|
|
|
if (GetVarint64(&input, &min_log_number_to_keep_)) {
|
|
|
|
has_min_log_number_to_keep_ = true;
|
|
|
|
} else {
|
|
|
|
msg = "min log number to kee";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
case kCompactPointer:
|
2013-01-24 18:54:26 +00:00
|
|
|
if (GetLevel(&input, &level, &msg) &&
|
2011-03-18 22:37:00 +00:00
|
|
|
GetInternalKey(&input, &key)) {
|
2014-01-16 22:06:53 +00:00
|
|
|
// we don't use compact pointers anymore,
|
|
|
|
// but we should not fail if they are still
|
|
|
|
// in manifest
|
2011-03-18 22:37:00 +00:00
|
|
|
} else {
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!msg) {
|
|
|
|
msg = "compaction pointer";
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2014-10-31 18:59:54 +00:00
|
|
|
case kDeletedFile: {
|
|
|
|
uint64_t number;
|
|
|
|
if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number)) {
|
2011-03-18 22:37:00 +00:00
|
|
|
deleted_files_.insert(std::make_pair(level, number));
|
|
|
|
} else {
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!msg) {
|
|
|
|
msg = "deleted file";
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
break;
|
2014-10-31 18:59:54 +00:00
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-06-13 22:54:19 +00:00
|
|
|
case kNewFile: {
|
|
|
|
uint64_t number;
|
|
|
|
uint64_t file_size;
|
|
|
|
if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
|
|
|
|
GetVarint64(&input, &file_size) &&
|
2011-03-18 22:37:00 +00:00
|
|
|
GetInternalKey(&input, &f.smallest) &&
|
|
|
|
GetInternalKey(&input, &f.largest)) {
|
2014-07-02 16:54:20 +00:00
|
|
|
f.fd = FileDescriptor(number, 0, file_size);
|
2011-03-18 22:37:00 +00:00
|
|
|
new_files_.push_back(std::make_pair(level, f));
|
|
|
|
} else {
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!msg) {
|
|
|
|
msg = "new-file entry";
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
break;
|
2014-06-13 22:54:19 +00:00
|
|
|
}
|
|
|
|
case kNewFile2: {
|
|
|
|
uint64_t number;
|
|
|
|
uint64_t file_size;
|
2018-07-27 23:00:26 +00:00
|
|
|
SequenceNumber smallest_seqno;
|
|
|
|
SequenceNumber largest_seqno;
|
2014-06-13 22:54:19 +00:00
|
|
|
if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
|
|
|
|
GetVarint64(&input, &file_size) &&
|
2013-06-14 05:09:08 +00:00
|
|
|
GetInternalKey(&input, &f.smallest) &&
|
|
|
|
GetInternalKey(&input, &f.largest) &&
|
2018-07-27 23:00:26 +00:00
|
|
|
GetVarint64(&input, &smallest_seqno) &&
|
|
|
|
GetVarint64(&input, &largest_seqno)) {
|
|
|
|
f.fd = FileDescriptor(number, 0, file_size, smallest_seqno,
|
|
|
|
largest_seqno);
|
2014-07-02 16:54:20 +00:00
|
|
|
new_files_.push_back(std::make_pair(level, f));
|
|
|
|
} else {
|
|
|
|
if (!msg) {
|
|
|
|
msg = "new-file2 entry";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
case kNewFile3: {
|
|
|
|
uint64_t number;
|
|
|
|
uint32_t path_id;
|
|
|
|
uint64_t file_size;
|
2018-07-27 23:00:26 +00:00
|
|
|
SequenceNumber smallest_seqno;
|
|
|
|
SequenceNumber largest_seqno;
|
2014-07-02 16:54:20 +00:00
|
|
|
if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
|
|
|
|
GetVarint32(&input, &path_id) && GetVarint64(&input, &file_size) &&
|
|
|
|
GetInternalKey(&input, &f.smallest) &&
|
|
|
|
GetInternalKey(&input, &f.largest) &&
|
2018-07-27 23:00:26 +00:00
|
|
|
GetVarint64(&input, &smallest_seqno) &&
|
|
|
|
GetVarint64(&input, &largest_seqno)) {
|
|
|
|
f.fd = FileDescriptor(number, path_id, file_size, smallest_seqno,
|
|
|
|
largest_seqno);
|
2013-06-14 05:09:08 +00:00
|
|
|
new_files_.push_back(std::make_pair(level, f));
|
|
|
|
} else {
|
|
|
|
if (!msg) {
|
2014-10-23 17:41:58 +00:00
|
|
|
msg = "new-file3 entry";
|
2013-06-14 05:09:08 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
2014-06-13 22:54:19 +00:00
|
|
|
}
|
2013-06-14 05:09:08 +00:00
|
|
|
|
2015-10-03 00:32:46 +00:00
|
|
|
case kNewFile4: {
|
|
|
|
msg = DecodeNewFile4From(&input);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2013-12-12 01:46:26 +00:00
|
|
|
case kColumnFamily:
|
|
|
|
if (!GetVarint32(&input, &column_family_)) {
|
|
|
|
if (!msg) {
|
|
|
|
msg = "set column family id";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case kColumnFamilyAdd:
|
|
|
|
if (GetLengthPrefixedSlice(&input, &str)) {
|
|
|
|
is_column_family_add_ = true;
|
|
|
|
column_family_name_ = str.ToString();
|
|
|
|
} else {
|
|
|
|
if (!msg) {
|
|
|
|
msg = "column family add";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case kColumnFamilyDrop:
|
|
|
|
is_column_family_drop_ = true;
|
|
|
|
break;
|
|
|
|
|
2018-08-20 21:54:03 +00:00
|
|
|
case kInAtomicGroup:
|
|
|
|
is_in_atomic_group_ = true;
|
|
|
|
if (!GetVarint32(&input, &remaining_entries_)) {
|
|
|
|
if (!msg) {
|
|
|
|
msg = "remaining entries";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
default:
|
2019-02-08 18:48:08 +00:00
|
|
|
if (tag & kTagSafeIgnoreMask) {
|
|
|
|
// Tag from future which can be safely ignored.
|
|
|
|
// The next field must be the length of the entry.
|
|
|
|
uint32_t field_len;
|
|
|
|
if (!GetVarint32(&input, &field_len) ||
|
|
|
|
static_cast<size_t>(field_len) > input.size()) {
|
|
|
|
if (!msg) {
|
|
|
|
msg = "safely ignoreable tag length error";
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
input.remove_prefix(static_cast<size_t>(field_len));
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
msg = "unknown tag";
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-03-01 02:04:58 +00:00
|
|
|
if (msg == nullptr && !input.empty()) {
|
2011-03-18 22:37:00 +00:00
|
|
|
msg = "invalid tag";
|
|
|
|
}
|
|
|
|
|
|
|
|
Status result;
|
2013-03-01 02:04:58 +00:00
|
|
|
if (msg != nullptr) {
|
2011-03-18 22:37:00 +00:00
|
|
|
result = Status::Corruption("VersionEdit", msg);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-08-08 22:51:16 +00:00
|
|
|
std::string VersionEdit::DebugString(bool hex_key) const {
|
2011-03-18 22:37:00 +00:00
|
|
|
std::string r;
|
|
|
|
r.append("VersionEdit {");
|
2019-09-03 15:50:47 +00:00
|
|
|
if (has_db_id_) {
|
|
|
|
r.append("\n DB ID: ");
|
|
|
|
r.append(db_id_);
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
if (has_comparator_) {
|
|
|
|
r.append("\n Comparator: ");
|
|
|
|
r.append(comparator_);
|
|
|
|
}
|
|
|
|
if (has_log_number_) {
|
|
|
|
r.append("\n LogNumber: ");
|
|
|
|
AppendNumberTo(&r, log_number_);
|
|
|
|
}
|
2011-04-12 19:38:58 +00:00
|
|
|
if (has_prev_log_number_) {
|
|
|
|
r.append("\n PrevLogNumber: ");
|
|
|
|
AppendNumberTo(&r, prev_log_number_);
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
if (has_next_file_number_) {
|
Added JSON manifest dump option to ldb command
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.
Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```
**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```
Reviewers: sdong, anthony, yhchiang, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D41727
2015-07-17 17:07:40 +00:00
|
|
|
r.append("\n NextFileNumber: ");
|
2011-03-18 22:37:00 +00:00
|
|
|
AppendNumberTo(&r, next_file_number_);
|
|
|
|
}
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
if (has_min_log_number_to_keep_) {
|
|
|
|
r.append("\n MinLogNumberToKeep: ");
|
|
|
|
AppendNumberTo(&r, min_log_number_to_keep_);
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
if (has_last_sequence_) {
|
|
|
|
r.append("\n LastSeq: ");
|
|
|
|
AppendNumberTo(&r, last_sequence_);
|
|
|
|
}
|
|
|
|
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
|
|
|
iter != deleted_files_.end();
|
|
|
|
++iter) {
|
|
|
|
r.append("\n DeleteFile: ");
|
|
|
|
AppendNumberTo(&r, iter->first);
|
|
|
|
r.append(" ");
|
|
|
|
AppendNumberTo(&r, iter->second);
|
|
|
|
}
|
2011-04-20 22:48:11 +00:00
|
|
|
for (size_t i = 0; i < new_files_.size(); i++) {
|
2011-03-18 22:37:00 +00:00
|
|
|
const FileMetaData& f = new_files_[i].second;
|
|
|
|
r.append("\n AddFile: ");
|
|
|
|
AppendNumberTo(&r, new_files_[i].first);
|
|
|
|
r.append(" ");
|
2014-06-13 22:54:19 +00:00
|
|
|
AppendNumberTo(&r, f.fd.GetNumber());
|
2011-03-18 22:37:00 +00:00
|
|
|
r.append(" ");
|
2014-06-13 22:54:19 +00:00
|
|
|
AppendNumberTo(&r, f.fd.GetFileSize());
|
2011-10-05 23:30:28 +00:00
|
|
|
r.append(" ");
|
2013-08-08 22:51:16 +00:00
|
|
|
r.append(f.smallest.DebugString(hex_key));
|
2011-10-05 23:30:28 +00:00
|
|
|
r.append(" .. ");
|
2013-08-08 22:51:16 +00:00
|
|
|
r.append(f.largest.DebugString(hex_key));
|
2019-10-14 22:19:31 +00:00
|
|
|
if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
|
|
|
|
r.append(" blob_file:");
|
|
|
|
AppendNumberTo(&r, f.oldest_blob_file_number);
|
|
|
|
}
|
2019-11-23 00:01:21 +00:00
|
|
|
r.append(" oldest_ancester_time:");
|
|
|
|
AppendNumberTo(&r, f.oldest_ancester_time);
|
2019-11-27 05:38:38 +00:00
|
|
|
r.append(" file_creation_time:");
|
|
|
|
AppendNumberTo(&r, f.file_creation_time);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
2013-12-12 01:46:26 +00:00
|
|
|
r.append("\n ColumnFamily: ");
|
|
|
|
AppendNumberTo(&r, column_family_);
|
|
|
|
if (is_column_family_add_) {
|
|
|
|
r.append("\n ColumnFamilyAdd: ");
|
|
|
|
r.append(column_family_name_);
|
|
|
|
}
|
|
|
|
if (is_column_family_drop_) {
|
|
|
|
r.append("\n ColumnFamilyDrop");
|
|
|
|
}
|
2014-03-11 02:59:44 +00:00
|
|
|
if (has_max_column_family_) {
|
|
|
|
r.append("\n MaxColumnFamily: ");
|
2014-03-11 19:14:09 +00:00
|
|
|
AppendNumberTo(&r, max_column_family_);
|
2014-03-11 02:59:44 +00:00
|
|
|
}
|
2018-08-20 21:54:03 +00:00
|
|
|
if (is_in_atomic_group_) {
|
2018-12-13 23:10:16 +00:00
|
|
|
r.append("\n AtomicGroup: ");
|
2018-08-20 21:54:03 +00:00
|
|
|
AppendNumberTo(&r, remaining_entries_);
|
|
|
|
r.append(" entries remains");
|
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
r.append("\n}\n");
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
Added JSON manifest dump option to ldb command
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.
Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```
**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```
Reviewers: sdong, anthony, yhchiang, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D41727
2015-07-17 17:07:40 +00:00
|
|
|
std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
|
|
|
|
JSONWriter jw;
|
|
|
|
jw << "EditNumber" << edit_num;
|
|
|
|
|
2019-09-03 15:50:47 +00:00
|
|
|
if (has_db_id_) {
|
|
|
|
jw << "DB ID" << db_id_;
|
|
|
|
}
|
Added JSON manifest dump option to ldb command
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.
Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```
**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```
Reviewers: sdong, anthony, yhchiang, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D41727
2015-07-17 17:07:40 +00:00
|
|
|
if (has_comparator_) {
|
|
|
|
jw << "Comparator" << comparator_;
|
|
|
|
}
|
|
|
|
if (has_log_number_) {
|
|
|
|
jw << "LogNumber" << log_number_;
|
|
|
|
}
|
|
|
|
if (has_prev_log_number_) {
|
|
|
|
jw << "PrevLogNumber" << prev_log_number_;
|
|
|
|
}
|
|
|
|
if (has_next_file_number_) {
|
|
|
|
jw << "NextFileNumber" << next_file_number_;
|
|
|
|
}
|
|
|
|
if (has_last_sequence_) {
|
|
|
|
jw << "LastSeq" << last_sequence_;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!deleted_files_.empty()) {
|
|
|
|
jw << "DeletedFiles";
|
|
|
|
jw.StartArray();
|
|
|
|
|
|
|
|
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
|
|
|
iter != deleted_files_.end();
|
|
|
|
++iter) {
|
|
|
|
jw.StartArrayedObject();
|
|
|
|
jw << "Level" << iter->first;
|
|
|
|
jw << "FileNumber" << iter->second;
|
|
|
|
jw.EndArrayedObject();
|
|
|
|
}
|
|
|
|
|
|
|
|
jw.EndArray();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!new_files_.empty()) {
|
|
|
|
jw << "AddedFiles";
|
|
|
|
jw.StartArray();
|
|
|
|
|
|
|
|
for (size_t i = 0; i < new_files_.size(); i++) {
|
|
|
|
jw.StartArrayedObject();
|
|
|
|
jw << "Level" << new_files_[i].first;
|
|
|
|
const FileMetaData& f = new_files_[i].second;
|
|
|
|
jw << "FileNumber" << f.fd.GetNumber();
|
|
|
|
jw << "FileSize" << f.fd.GetFileSize();
|
|
|
|
jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
|
|
|
|
jw << "LargestIKey" << f.largest.DebugString(hex_key);
|
2019-10-14 22:19:31 +00:00
|
|
|
if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
|
|
|
|
jw << "OldestBlobFile" << f.oldest_blob_file_number;
|
|
|
|
}
|
Added JSON manifest dump option to ldb command
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.
Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```
**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```
Reviewers: sdong, anthony, yhchiang, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D41727
2015-07-17 17:07:40 +00:00
|
|
|
jw.EndArrayedObject();
|
|
|
|
}
|
|
|
|
|
|
|
|
jw.EndArray();
|
|
|
|
}
|
|
|
|
|
|
|
|
jw << "ColumnFamily" << column_family_;
|
|
|
|
|
|
|
|
if (is_column_family_add_) {
|
|
|
|
jw << "ColumnFamilyAdd" << column_family_name_;
|
|
|
|
}
|
|
|
|
if (is_column_family_drop_) {
|
|
|
|
jw << "ColumnFamilyDrop" << column_family_name_;
|
|
|
|
}
|
|
|
|
if (has_max_column_family_) {
|
|
|
|
jw << "MaxColumnFamily" << max_column_family_;
|
|
|
|
}
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-03 22:35:11 +00:00
|
|
|
if (has_min_log_number_to_keep_) {
|
|
|
|
jw << "MinLogNumberToKeep" << min_log_number_to_keep_;
|
|
|
|
}
|
2018-08-20 21:54:03 +00:00
|
|
|
if (is_in_atomic_group_) {
|
|
|
|
jw << "AtomicGroup" << remaining_entries_;
|
|
|
|
}
|
Added JSON manifest dump option to ldb command
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.
Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```
**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```
Reviewers: sdong, anthony, yhchiang, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D41727
2015-07-17 17:07:40 +00:00
|
|
|
|
|
|
|
jw.EndObject();
|
|
|
|
|
|
|
|
return jw.Get();
|
|
|
|
}
|
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
} // namespace rocksdb
|