2016-03-18 22:18:42 +00:00
|
|
|
// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
|
#include <string>
|
|
|
|
|
#include <vector>
|
|
|
|
|
|
2019-05-31 18:52:59 +00:00
|
|
|
#include "db/db_impl/db_impl.h"
|
2016-03-18 22:18:42 +00:00
|
|
|
#include "db/db_test_util.h"
|
2023-05-31 02:32:00 +00:00
|
|
|
#include "db/db_with_timestamp_test_util.h"
|
2019-05-30 03:44:08 +00:00
|
|
|
#include "file/file_util.h"
|
2016-06-24 23:29:43 +00:00
|
|
|
#include "rocksdb/comparator.h"
|
2016-03-18 22:18:42 +00:00
|
|
|
#include "rocksdb/db.h"
|
2023-05-31 02:32:00 +00:00
|
|
|
#include "rocksdb/options.h"
|
2016-03-18 22:18:42 +00:00
|
|
|
#include "rocksdb/transaction_log.h"
|
2022-05-24 16:00:06 +00:00
|
|
|
#include "table/unique_id_impl.h"
|
2016-06-24 20:12:13 +00:00
|
|
|
#include "util/string_util.h"
|
2016-03-18 22:18:42 +00:00
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
class RepairTest : public DBTestBase {
|
|
|
|
|
public:
|
2021-07-23 15:37:27 +00:00
|
|
|
RepairTest() : DBTestBase("repair_test", /*env_do_fsync=*/true) {}
|
2016-03-18 22:18:42 +00:00
|
|
|
|
2020-09-29 23:28:42 +00:00
|
|
|
Status GetFirstSstPath(std::string* first_sst_path) {
|
|
|
|
|
assert(first_sst_path != nullptr);
|
|
|
|
|
first_sst_path->clear();
|
2016-03-18 22:18:42 +00:00
|
|
|
uint64_t manifest_size;
|
|
|
|
|
std::vector<std::string> files;
|
2020-09-29 23:28:42 +00:00
|
|
|
Status s = db_->GetLiveFiles(files, &manifest_size);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
auto sst_iter =
|
|
|
|
|
std::find_if(files.begin(), files.end(), [](const std::string& file) {
|
|
|
|
|
uint64_t number;
|
|
|
|
|
FileType type;
|
|
|
|
|
bool ok = ParseFileName(file, &number, &type);
|
|
|
|
|
return ok && type == kTableFile;
|
|
|
|
|
});
|
|
|
|
|
*first_sst_path = sst_iter == files.end() ? "" : dbname_ + *sst_iter;
|
|
|
|
|
}
|
|
|
|
|
return s;
|
2016-03-18 22:18:42 +00:00
|
|
|
}
|
2022-05-19 18:04:21 +00:00
|
|
|
|
|
|
|
|
void ReopenWithSstIdVerify() {
|
|
|
|
|
std::atomic_int verify_passed{0};
|
|
|
|
|
SyncPoint::GetInstance()->SetCallBack(
|
Always verify SST unique IDs on SST file open (#10532)
Summary:
Although we've been tracking SST unique IDs in the DB manifest
unconditionally, checking has been opt-in and with an extra pass at DB::Open
time. This changes the behavior of `verify_sst_unique_id_in_manifest` to
check unique ID against manifest every time an SST file is opened through
table cache (normal DB operations), replacing the explicit pass over files
at DB::Open time. This change also enables the option by default and
removes the "EXPERIMENTAL" designation.
One possible criticism is that the option no longer ensures the integrity
of a DB at Open time. This is far from an all-or-nothing issue. Verifying
the IDs of all SST files hardly ensures all the data in the DB is readable.
(VerifyChecksum is supposed to do that.) Also, with
max_open_files=-1 (default, extremely common), all SST files are
opened at DB::Open time anyway.
Implementation details:
* `VerifySstUniqueIdInManifest()` functions are the extra/explicit pass
that is now removed.
* Unit tests that manipulate/corrupt table properties have to opt out of
this check, because that corrupts the "actual" unique id. (And even for
testing we don't currently have a mechanism to set "no unique id"
in the in-memory file metadata for new files.)
* A lot of other unit test churn relates to (a) default checking on, and
(b) checking on SST open even without DB::Open (e.g. on flush)
* Use `FileMetaData` for more `TableCache` operations (in place of
`FileDescriptor`) so that we have access to the unique_id whenever
we might need to open an SST file. **There is the possibility of
performance impact because we can no longer use the more
localized `fd` part of an `FdWithKeyRange` but instead follow the
`file_metadata` pointer. However, this change (possible regression)
is only done for `GetMemoryUsageByTableReaders`.**
* Removed a completely unnecessary constructor overload of
`TableReaderOptions`
Possible follow-up:
* Verification only happens when opening through table cache. Are there
more places where this should happen?
* Improve error message when there is a file size mismatch vs. manifest
(FIXME added in the appropriate place).
* I'm not sure there's a justification for `FileDescriptor` to be distinct from
`FileMetaData`.
* I'm skeptical that `FdWithKeyRange` really still makes sense for
optimizing some data locality by duplicating some data in memory, but I
could be wrong.
* An unnecessary overload of NewTableReader was recently added, in
the public API nonetheless (though unusable there). It should be cleaned
up to put most things under `TableReaderOptions`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10532
Test Plan:
updated unit tests
Performance test showing no significant difference (just noise I think):
`./db_bench -benchmarks=readwhilewriting[-X10] -num=3000000 -disable_wal=1 -bloom_bits=8 -write_buffer_size=1000000 -target_file_size_base=1000000`
Before: readwhilewriting [AVG 10 runs] : 68702 (± 6932) ops/sec
After: readwhilewriting [AVG 10 runs] : 68239 (± 7198) ops/sec
Reviewed By: jay-zhuang
Differential Revision: D38765551
Pulled By: pdillinger
fbshipit-source-id: a827a708155f12344ab2a5c16e7701c7636da4c2
2022-09-08 05:52:42 +00:00
|
|
|
"BlockBasedTable::Open::PassedVerifyUniqueId", [&](void* arg) {
|
2022-05-19 18:04:21 +00:00
|
|
|
// override job status
|
2022-05-24 16:00:06 +00:00
|
|
|
auto id = static_cast<UniqueId64x2*>(arg);
|
|
|
|
|
assert(*id != kNullUniqueId64x2);
|
2022-05-19 18:04:21 +00:00
|
|
|
verify_passed++;
|
|
|
|
|
});
|
|
|
|
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
|
options.verify_sst_unique_id_in_manifest = true;
|
|
|
|
|
Reopen(options);
|
|
|
|
|
|
|
|
|
|
ASSERT_GT(verify_passed, 0);
|
Always verify SST unique IDs on SST file open (#10532)
Summary:
Although we've been tracking SST unique IDs in the DB manifest
unconditionally, checking has been opt-in and with an extra pass at DB::Open
time. This changes the behavior of `verify_sst_unique_id_in_manifest` to
check unique ID against manifest every time an SST file is opened through
table cache (normal DB operations), replacing the explicit pass over files
at DB::Open time. This change also enables the option by default and
removes the "EXPERIMENTAL" designation.
One possible criticism is that the option no longer ensures the integrity
of a DB at Open time. This is far from an all-or-nothing issue. Verifying
the IDs of all SST files hardly ensures all the data in the DB is readable.
(VerifyChecksum is supposed to do that.) Also, with
max_open_files=-1 (default, extremely common), all SST files are
opened at DB::Open time anyway.
Implementation details:
* `VerifySstUniqueIdInManifest()` functions are the extra/explicit pass
that is now removed.
* Unit tests that manipulate/corrupt table properties have to opt out of
this check, because that corrupts the "actual" unique id. (And even for
testing we don't currently have a mechanism to set "no unique id"
in the in-memory file metadata for new files.)
* A lot of other unit test churn relates to (a) default checking on, and
(b) checking on SST open even without DB::Open (e.g. on flush)
* Use `FileMetaData` for more `TableCache` operations (in place of
`FileDescriptor`) so that we have access to the unique_id whenever
we might need to open an SST file. **There is the possibility of
performance impact because we can no longer use the more
localized `fd` part of an `FdWithKeyRange` but instead follow the
`file_metadata` pointer. However, this change (possible regression)
is only done for `GetMemoryUsageByTableReaders`.**
* Removed a completely unnecessary constructor overload of
`TableReaderOptions`
Possible follow-up:
* Verification only happens when opening through table cache. Are there
more places where this should happen?
* Improve error message when there is a file size mismatch vs. manifest
(FIXME added in the appropriate place).
* I'm not sure there's a justification for `FileDescriptor` to be distinct from
`FileMetaData`.
* I'm skeptical that `FdWithKeyRange` really still makes sense for
optimizing some data locality by duplicating some data in memory, but I
could be wrong.
* An unnecessary overload of NewTableReader was recently added, in
the public API nonetheless (though unusable there). It should be cleaned
up to put most things under `TableReaderOptions`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10532
Test Plan:
updated unit tests
Performance test showing no significant difference (just noise I think):
`./db_bench -benchmarks=readwhilewriting[-X10] -num=3000000 -disable_wal=1 -bloom_bits=8 -write_buffer_size=1000000 -target_file_size_base=1000000`
Before: readwhilewriting [AVG 10 runs] : 68702 (± 6932) ops/sec
After: readwhilewriting [AVG 10 runs] : 68239 (± 7198) ops/sec
Reviewed By: jay-zhuang
Differential Revision: D38765551
Pulled By: pdillinger
fbshipit-source-id: a827a708155f12344ab2a5c16e7701c7636da4c2
2022-09-08 05:52:42 +00:00
|
|
|
SyncPoint::GetInstance()->DisableProcessing();
|
2022-05-19 18:04:21 +00:00
|
|
|
}
|
Sort L0 files by newly introduced epoch_num (#10922)
Summary:
**Context:**
Sorting L0 files by `largest_seqno` has at least two inconvenience:
- File ingestion and compaction involving ingested files can create files of overlapping seqno range with the existing files. `force_consistency_check=true` will catch such overlap seqno range even those harmless overlap.
- For example, consider the following sequence of events ("key@n" indicates key at seqno "n")
- insert k1@1 to memtable m1
- ingest file s1 with k2@2, ingest file s2 with k3@3
- insert k4@4 to m1
- compact files s1, s2 and result in new file s3 of seqno range [2, 3]
- flush m1 and result in new file s4 of seqno range [1, 4]. And `force_consistency_check=true` will think s4 and s3 has file reordering corruption that might cause retuning an old value of k1
- However such caught corruption is a false positive since s1, s2 will not have overlapped keys with k1 or whatever inserted into m1 before ingest file s1 by the requirement of file ingestion (otherwise the m1 will be flushed first before any of the file ingestion completes). Therefore there in fact isn't any file reordering corruption.
- Single delete can decrease a file's largest seqno and ordering by `largest_seqno` can introduce a wrong ordering hence file reordering corruption
- For example, consider the following sequence of events ("key@n" indicates key at seqno "n", Credit to ajkr for this example)
- an existing SST s1 contains only k1@1
- insert k1@2 to memtable m1
- ingest file s2 with k3@3, ingest file s3 with k4@4
- insert single delete k5@5 in m1
- flush m1 and result in new file s4 of seqno range [2, 5]
- compact s1, s2, s3 and result in new file s5 of seqno range [1, 4]
- compact s4 and result in new file s6 of seqno range [2] due to single delete
- By the last step, we have file ordering by largest seqno (">" means "newer") : s5 > s6 while s6 contains a newer version of the k1's value (i.e, k1@2) than s5, which is a real reordering corruption. While this can be caught by `force_consistency_check=true`, there isn't a good way to prevent this from happening if ordering by `largest_seqno`
Therefore, we are redesigning the sorting criteria of L0 files and avoid above inconvenience. Credit to ajkr , we now introduce `epoch_num` which describes the order of a file being flushed or ingested/imported (compaction output file will has the minimum `epoch_num` among input files'). This will avoid the above inconvenience in the following ways:
- In the first case above, there will no longer be overlap seqno range check in `force_consistency_check=true` but `epoch_number` ordering check. This will result in file ordering s1 < s2 < s4 (pre-compaction) and s3 < s4 (post-compaction) which won't trigger false positive corruption. See test class `DBCompactionTestL0FilesMisorderCorruption*` for more.
- In the second case above, this will result in file ordering s1 < s2 < s3 < s4 (pre-compacting s1, s2, s3), s5 < s4 (post-compacting s1, s2, s3), s5 < s6 (post-compacting s4), which are correct file ordering without causing any corruption.
**Summary:**
- Introduce `epoch_number` stored per `ColumnFamilyData` and sort CF's L0 files by their assigned `epoch_number` instead of `largest_seqno`.
- `epoch_number` is increased and assigned upon `VersionEdit::AddFile()` for flush (or similarly for WriteLevel0TableForRecovery) and file ingestion (except for allow_behind_true, which will always get assigned as the `kReservedEpochNumberForFileIngestedBehind`)
- Compaction output file is assigned with the minimum `epoch_number` among input files'
- Refit level: reuse refitted file's epoch_number
- Other paths needing `epoch_number` treatment:
- Import column families: reuse file's epoch_number if exists. If not, assign one based on `NewestFirstBySeqNo`
- Repair: reuse file's epoch_number if exists. If not, assign one based on `NewestFirstBySeqNo`.
- Assigning new epoch_number to a file and adding this file to LSM tree should be atomic. This is guaranteed by us assigning epoch_number right upon `VersionEdit::AddFile()` where this version edit will be apply to LSM tree shape right after by holding the db mutex (e.g, flush, file ingestion, import column family) or by there is only 1 ongoing edit per CF (e.g, WriteLevel0TableForRecovery, Repair).
- Assigning the minimum input epoch number to compaction output file won't misorder L0 files (even through later `Refit(target_level=0)`). It's due to for every key "k" in the input range, a legit compaction will cover a continuous epoch number range of that key. As long as we assign the key "k" the minimum input epoch number, it won't become newer or older than the versions of this key that aren't included in this compaction hence no misorder.
- Persist `epoch_number` of each file in manifest and recover `epoch_number` on db recovery
- Backward compatibility with old db without `epoch_number` support is guaranteed by assigning `epoch_number` to recovered files by `NewestFirstBySeqno` order. See `VersionStorageInfo::RecoverEpochNumbers()` for more
- Forward compatibility with manifest is guaranteed by flexibility of `NewFileCustomTag`
- Replace `force_consistent_check` on L0 with `epoch_number` and remove false positive check like case 1 with `largest_seqno` above
- Due to backward compatibility issue, we might encounter files with missing epoch number at the beginning of db recovery. We will still use old L0 sorting mechanism (`NewestFirstBySeqno`) to check/sort them till we infer their epoch number. See usages of `EpochNumberRequirement`.
- Remove fix https://github.com/facebook/rocksdb/pull/5958#issue-511150930 and their outdated tests to file reordering corruption because such fix can be replaced by this PR.
- Misc:
- update existing tests with `epoch_number` so make check will pass
- update https://github.com/facebook/rocksdb/pull/5958#issue-511150930 tests to verify corruption is fixed using `epoch_number` and cover universal/fifo compaction/CompactRange/CompactFile cases
- assert db_mutex is held for a few places before calling ColumnFamilyData::NewEpochNumber()
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10922
Test Plan:
- `make check`
- New unit tests under `db/db_compaction_test.cc`, `db/db_test2.cc`, `db/version_builder_test.cc`, `db/repair_test.cc`
- Updated tests (i.e, `DBCompactionTestL0FilesMisorderCorruption*`) under https://github.com/facebook/rocksdb/pull/5958#issue-511150930
- [Ongoing] Compatibility test: manually run https://github.com/ajkr/rocksdb/commit/36a5686ec012f35a4371e409aa85c404ca1c210d (with file ingestion off for running the `.orig` binary to prevent this bug affecting upgrade/downgrade formality checking) for 1 hour on `simple black/white box`, `cf_consistency/txn/enable_ts with whitebox + test_best_efforts_recovery with blackbox`
- [Ongoing] normal db stress test
- [Ongoing] db stress test with aggressive value https://github.com/facebook/rocksdb/pull/10761
Reviewed By: ajkr
Differential Revision: D41063187
Pulled By: hx235
fbshipit-source-id: 826cb23455de7beaabe2d16c57682a82733a32a9
2022-12-13 21:29:37 +00:00
|
|
|
|
|
|
|
|
std::vector<FileMetaData*> GetLevelFileMetadatas(int level, int cf = 0) {
|
|
|
|
|
VersionSet* const versions = dbfull()->GetVersionSet();
|
|
|
|
|
assert(versions);
|
|
|
|
|
ColumnFamilyData* const cfd =
|
|
|
|
|
versions->GetColumnFamilySet()->GetColumnFamily(cf);
|
|
|
|
|
assert(cfd);
|
|
|
|
|
Version* const current = cfd->current();
|
|
|
|
|
assert(current);
|
|
|
|
|
VersionStorageInfo* const storage_info = current->storage_info();
|
|
|
|
|
assert(storage_info);
|
|
|
|
|
return storage_info->LevelFiles(level);
|
|
|
|
|
}
|
2016-03-18 22:18:42 +00:00
|
|
|
};
|
|
|
|
|
|
Sort L0 files by newly introduced epoch_num (#10922)
Summary:
**Context:**
Sorting L0 files by `largest_seqno` has at least two inconvenience:
- File ingestion and compaction involving ingested files can create files of overlapping seqno range with the existing files. `force_consistency_check=true` will catch such overlap seqno range even those harmless overlap.
- For example, consider the following sequence of events ("key@n" indicates key at seqno "n")
- insert k1@1 to memtable m1
- ingest file s1 with k2@2, ingest file s2 with k3@3
- insert k4@4 to m1
- compact files s1, s2 and result in new file s3 of seqno range [2, 3]
- flush m1 and result in new file s4 of seqno range [1, 4]. And `force_consistency_check=true` will think s4 and s3 has file reordering corruption that might cause retuning an old value of k1
- However such caught corruption is a false positive since s1, s2 will not have overlapped keys with k1 or whatever inserted into m1 before ingest file s1 by the requirement of file ingestion (otherwise the m1 will be flushed first before any of the file ingestion completes). Therefore there in fact isn't any file reordering corruption.
- Single delete can decrease a file's largest seqno and ordering by `largest_seqno` can introduce a wrong ordering hence file reordering corruption
- For example, consider the following sequence of events ("key@n" indicates key at seqno "n", Credit to ajkr for this example)
- an existing SST s1 contains only k1@1
- insert k1@2 to memtable m1
- ingest file s2 with k3@3, ingest file s3 with k4@4
- insert single delete k5@5 in m1
- flush m1 and result in new file s4 of seqno range [2, 5]
- compact s1, s2, s3 and result in new file s5 of seqno range [1, 4]
- compact s4 and result in new file s6 of seqno range [2] due to single delete
- By the last step, we have file ordering by largest seqno (">" means "newer") : s5 > s6 while s6 contains a newer version of the k1's value (i.e, k1@2) than s5, which is a real reordering corruption. While this can be caught by `force_consistency_check=true`, there isn't a good way to prevent this from happening if ordering by `largest_seqno`
Therefore, we are redesigning the sorting criteria of L0 files and avoid above inconvenience. Credit to ajkr , we now introduce `epoch_num` which describes the order of a file being flushed or ingested/imported (compaction output file will has the minimum `epoch_num` among input files'). This will avoid the above inconvenience in the following ways:
- In the first case above, there will no longer be overlap seqno range check in `force_consistency_check=true` but `epoch_number` ordering check. This will result in file ordering s1 < s2 < s4 (pre-compaction) and s3 < s4 (post-compaction) which won't trigger false positive corruption. See test class `DBCompactionTestL0FilesMisorderCorruption*` for more.
- In the second case above, this will result in file ordering s1 < s2 < s3 < s4 (pre-compacting s1, s2, s3), s5 < s4 (post-compacting s1, s2, s3), s5 < s6 (post-compacting s4), which are correct file ordering without causing any corruption.
**Summary:**
- Introduce `epoch_number` stored per `ColumnFamilyData` and sort CF's L0 files by their assigned `epoch_number` instead of `largest_seqno`.
- `epoch_number` is increased and assigned upon `VersionEdit::AddFile()` for flush (or similarly for WriteLevel0TableForRecovery) and file ingestion (except for allow_behind_true, which will always get assigned as the `kReservedEpochNumberForFileIngestedBehind`)
- Compaction output file is assigned with the minimum `epoch_number` among input files'
- Refit level: reuse refitted file's epoch_number
- Other paths needing `epoch_number` treatment:
- Import column families: reuse file's epoch_number if exists. If not, assign one based on `NewestFirstBySeqNo`
- Repair: reuse file's epoch_number if exists. If not, assign one based on `NewestFirstBySeqNo`.
- Assigning new epoch_number to a file and adding this file to LSM tree should be atomic. This is guaranteed by us assigning epoch_number right upon `VersionEdit::AddFile()` where this version edit will be apply to LSM tree shape right after by holding the db mutex (e.g, flush, file ingestion, import column family) or by there is only 1 ongoing edit per CF (e.g, WriteLevel0TableForRecovery, Repair).
- Assigning the minimum input epoch number to compaction output file won't misorder L0 files (even through later `Refit(target_level=0)`). It's due to for every key "k" in the input range, a legit compaction will cover a continuous epoch number range of that key. As long as we assign the key "k" the minimum input epoch number, it won't become newer or older than the versions of this key that aren't included in this compaction hence no misorder.
- Persist `epoch_number` of each file in manifest and recover `epoch_number` on db recovery
- Backward compatibility with old db without `epoch_number` support is guaranteed by assigning `epoch_number` to recovered files by `NewestFirstBySeqno` order. See `VersionStorageInfo::RecoverEpochNumbers()` for more
- Forward compatibility with manifest is guaranteed by flexibility of `NewFileCustomTag`
- Replace `force_consistent_check` on L0 with `epoch_number` and remove false positive check like case 1 with `largest_seqno` above
- Due to backward compatibility issue, we might encounter files with missing epoch number at the beginning of db recovery. We will still use old L0 sorting mechanism (`NewestFirstBySeqno`) to check/sort them till we infer their epoch number. See usages of `EpochNumberRequirement`.
- Remove fix https://github.com/facebook/rocksdb/pull/5958#issue-511150930 and their outdated tests to file reordering corruption because such fix can be replaced by this PR.
- Misc:
- update existing tests with `epoch_number` so make check will pass
- update https://github.com/facebook/rocksdb/pull/5958#issue-511150930 tests to verify corruption is fixed using `epoch_number` and cover universal/fifo compaction/CompactRange/CompactFile cases
- assert db_mutex is held for a few places before calling ColumnFamilyData::NewEpochNumber()
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10922
Test Plan:
- `make check`
- New unit tests under `db/db_compaction_test.cc`, `db/db_test2.cc`, `db/version_builder_test.cc`, `db/repair_test.cc`
- Updated tests (i.e, `DBCompactionTestL0FilesMisorderCorruption*`) under https://github.com/facebook/rocksdb/pull/5958#issue-511150930
- [Ongoing] Compatibility test: manually run https://github.com/ajkr/rocksdb/commit/36a5686ec012f35a4371e409aa85c404ca1c210d (with file ingestion off for running the `.orig` binary to prevent this bug affecting upgrade/downgrade formality checking) for 1 hour on `simple black/white box`, `cf_consistency/txn/enable_ts with whitebox + test_best_efforts_recovery with blackbox`
- [Ongoing] normal db stress test
- [Ongoing] db stress test with aggressive value https://github.com/facebook/rocksdb/pull/10761
Reviewed By: ajkr
Differential Revision: D41063187
Pulled By: hx235
fbshipit-source-id: 826cb23455de7beaabe2d16c57682a82733a32a9
2022-12-13 21:29:37 +00:00
|
|
|
TEST_F(RepairTest, SortRepairedDBL0ByEpochNumber) {
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(Put("k1", "oldest"));
|
|
|
|
|
ASSERT_OK(Put("k1", "older"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
MoveFilesToLevel(1);
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(Put("k1", "old"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(Put("k1", "new"));
|
|
|
|
|
|
|
|
|
|
std::vector<FileMetaData*> level0_files = GetLevelFileMetadatas(0 /* level*/);
|
|
|
|
|
ASSERT_EQ(level0_files.size(), 1);
|
|
|
|
|
ASSERT_EQ(level0_files[0]->epoch_number, 2);
|
|
|
|
|
std::vector<FileMetaData*> level1_files = GetLevelFileMetadatas(1 /* level*/);
|
|
|
|
|
ASSERT_EQ(level1_files.size(), 1);
|
|
|
|
|
ASSERT_EQ(level1_files[0]->epoch_number, 1);
|
|
|
|
|
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
|
|
|
|
ReopenWithSstIdVerify();
|
|
|
|
|
|
|
|
|
|
EXPECT_EQ(Get("k1"), "new");
|
|
|
|
|
|
|
|
|
|
level0_files = GetLevelFileMetadatas(0 /* level*/);
|
|
|
|
|
ASSERT_EQ(level0_files.size(), 3);
|
|
|
|
|
EXPECT_EQ(level0_files[0]->epoch_number, 3);
|
|
|
|
|
EXPECT_EQ(level0_files[1]->epoch_number, 2);
|
|
|
|
|
EXPECT_EQ(level0_files[2]->epoch_number, 1);
|
|
|
|
|
level1_files = GetLevelFileMetadatas(1 /* level*/);
|
|
|
|
|
ASSERT_EQ(level1_files.size(), 0);
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-18 22:18:42 +00:00
|
|
|
TEST_F(RepairTest, LostManifest) {
|
|
|
|
|
// Add a couple SST files, delete the manifest, and verify RepairDB() saves
|
|
|
|
|
// the day.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Flush());
|
2016-03-18 22:18:42 +00:00
|
|
|
// Need to get path before Close() deletes db_, but delete it after Close() to
|
|
|
|
|
// ensure Close() didn't change the manifest.
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
2016-06-24 18:19:40 +00:00
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
ASSERT_EQ(Get("key2"), "val2");
|
|
|
|
|
}
|
|
|
|
|
|
2021-07-28 23:43:16 +00:00
|
|
|
TEST_F(RepairTest, LostManifestMoreDbFeatures) {
|
|
|
|
|
// Add a couple SST files, delete the manifest, and verify RepairDB() saves
|
|
|
|
|
// the day.
|
|
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Put("key3", "val3"));
|
|
|
|
|
ASSERT_OK(Put("key4", "val4"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
// Test an SST file containing only a range tombstone
|
|
|
|
|
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key2",
|
|
|
|
|
"key3z"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
// Need to get path before Close() deletes db_, but delete it after Close() to
|
|
|
|
|
// ensure Close() didn't change the manifest.
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
|
|
|
|
|
// repair from sst should work with unique_id verification
|
|
|
|
|
ReopenWithSstIdVerify();
|
2021-07-28 23:43:16 +00:00
|
|
|
|
|
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
ASSERT_EQ(Get("key2"), "NOT_FOUND");
|
|
|
|
|
ASSERT_EQ(Get("key3"), "NOT_FOUND");
|
|
|
|
|
ASSERT_EQ(Get("key4"), "val4");
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-18 22:18:42 +00:00
|
|
|
TEST_F(RepairTest, CorruptManifest) {
|
|
|
|
|
// Manifest is in an invalid format. Expect a full recovery.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Flush());
|
2016-03-18 22:18:42 +00:00
|
|
|
// Need to get path before Close() deletes db_, but overwrite it after Close()
|
|
|
|
|
// to ensure Close() didn't change the manifest.
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
2019-12-13 22:47:08 +00:00
|
|
|
|
2021-01-06 18:48:24 +00:00
|
|
|
ASSERT_OK(CreateFile(env_->GetFileSystem(), manifest_path, "blah",
|
|
|
|
|
false /* use_fsync */));
|
2016-06-24 18:19:40 +00:00
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
|
|
|
|
|
ReopenWithSstIdVerify();
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
ASSERT_EQ(Get("key2"), "val2");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST_F(RepairTest, IncompleteManifest) {
|
|
|
|
|
// In this case, the manifest is valid but does not reference all of the SST
|
|
|
|
|
// files. Expect a full recovery.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
2016-03-18 22:18:42 +00:00
|
|
|
std::string orig_manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
CopyFile(orig_manifest_path, orig_manifest_path + ".tmp");
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Flush());
|
2016-03-18 22:18:42 +00:00
|
|
|
// Need to get path before Close() deletes db_, but overwrite it after Close()
|
|
|
|
|
// to ensure Close() didn't change the manifest.
|
|
|
|
|
std::string new_manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(new_manifest_path));
|
|
|
|
|
// Replace the manifest with one that is only aware of the first SST file.
|
|
|
|
|
CopyFile(orig_manifest_path + ".tmp", new_manifest_path);
|
2016-06-24 18:19:40 +00:00
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
|
|
|
|
|
ReopenWithSstIdVerify();
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
ASSERT_EQ(Get("key2"), "val2");
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-10 20:07:00 +00:00
|
|
|
TEST_F(RepairTest, PostRepairSstFileNumbering) {
|
|
|
|
|
// Verify after a DB is repaired, new files will be assigned higher numbers
|
|
|
|
|
// than old files.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Flush());
|
2017-10-10 20:07:00 +00:00
|
|
|
uint64_t pre_repair_file_num = dbfull()->TEST_Current_Next_FileNo();
|
|
|
|
|
Close();
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
|
|
|
|
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
|
|
|
|
|
2017-10-10 20:07:00 +00:00
|
|
|
uint64_t post_repair_file_num = dbfull()->TEST_Current_Next_FileNo();
|
|
|
|
|
ASSERT_GE(post_repair_file_num, pre_repair_file_num);
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-18 22:18:42 +00:00
|
|
|
TEST_F(RepairTest, LostSst) {
|
|
|
|
|
// Delete one of the SST files but preserve the manifest that refers to it,
|
|
|
|
|
// then verify the DB is still usable for the intact SST.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
std::string sst_path;
|
|
|
|
|
ASSERT_OK(GetFirstSstPath(&sst_path));
|
2016-03-18 22:18:42 +00:00
|
|
|
ASSERT_FALSE(sst_path.empty());
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(sst_path));
|
|
|
|
|
|
|
|
|
|
Close();
|
2016-06-24 18:19:40 +00:00
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
// Exactly one of the key-value pairs should be in the DB now.
|
|
|
|
|
ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST_F(RepairTest, CorruptSst) {
|
|
|
|
|
// Corrupt one of the SST files but preserve the manifest that refers to it,
|
|
|
|
|
// then verify the DB is still usable for the intact SST.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
ASSERT_OK(Put("key2", "val2"));
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
|
std::string sst_path;
|
|
|
|
|
ASSERT_OK(GetFirstSstPath(&sst_path));
|
2016-03-18 22:18:42 +00:00
|
|
|
ASSERT_FALSE(sst_path.empty());
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
2019-12-13 22:47:08 +00:00
|
|
|
|
2021-01-06 18:48:24 +00:00
|
|
|
ASSERT_OK(CreateFile(env_->GetFileSystem(), sst_path, "blah",
|
|
|
|
|
false /* use_fsync */));
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
Close();
|
2016-06-24 18:19:40 +00:00
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
// Exactly one of the key-value pairs should be in the DB now.
|
|
|
|
|
ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST_F(RepairTest, UnflushedSst) {
|
|
|
|
|
// This test case invokes repair while some data is unflushed, then verifies
|
|
|
|
|
// that data is in the db.
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
2016-03-18 22:18:42 +00:00
|
|
|
VectorLogPtr wal_files;
|
|
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
|
ASSERT_EQ(wal_files.size(), 1);
|
2020-09-29 23:28:42 +00:00
|
|
|
{
|
|
|
|
|
uint64_t total_ssts_size;
|
|
|
|
|
std::unordered_map<std::string, uint64_t> sst_files;
|
2021-03-18 03:43:22 +00:00
|
|
|
ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_EQ(total_ssts_size, 0);
|
|
|
|
|
}
|
2016-03-18 22:18:42 +00:00
|
|
|
// Need to get path before Close() deletes db_, but delete it after Close() to
|
|
|
|
|
// ensure Close() didn't change the manifest.
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
2016-06-24 18:19:40 +00:00
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
|
ASSERT_EQ(wal_files.size(), 0);
|
2020-09-29 23:28:42 +00:00
|
|
|
{
|
|
|
|
|
uint64_t total_ssts_size;
|
|
|
|
|
std::unordered_map<std::string, uint64_t> sst_files;
|
2021-03-18 03:43:22 +00:00
|
|
|
ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_GT(total_ssts_size, 0);
|
|
|
|
|
}
|
2016-03-18 22:18:42 +00:00
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
}
|
|
|
|
|
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
// Test parameters:
|
|
|
|
|
// param 0): paranoid file check
|
|
|
|
|
// param 1): user-defined timestamp test mode
|
|
|
|
|
class RepairTestWithTimestamp
|
|
|
|
|
: public DBBasicTestWithTimestampBase,
|
|
|
|
|
public testing::WithParamInterface<
|
|
|
|
|
std::tuple<bool, test::UserDefinedTimestampTestMode>> {
|
2023-05-31 02:32:00 +00:00
|
|
|
public:
|
|
|
|
|
RepairTestWithTimestamp()
|
|
|
|
|
: DBBasicTestWithTimestampBase("repair_test_with_timestamp") {}
|
|
|
|
|
|
|
|
|
|
Status Put(const Slice& key, const Slice& ts, const Slice& value) {
|
|
|
|
|
WriteOptions write_opts;
|
|
|
|
|
return db_->Put(write_opts, handles_[0], key, ts, value);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void CheckGet(const ReadOptions& read_opts, const Slice& key,
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
const std::string& expected_value,
|
|
|
|
|
const std::string& expected_ts) {
|
2023-05-31 02:32:00 +00:00
|
|
|
std::string actual_value;
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
std::string actual_ts;
|
|
|
|
|
ASSERT_OK(db_->Get(read_opts, handles_[0], key, &actual_value, &actual_ts));
|
2023-05-31 02:32:00 +00:00
|
|
|
ASSERT_EQ(expected_value, actual_value);
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
ASSERT_EQ(expected_ts, actual_ts);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void CheckFileBoundaries(const Slice& smallest_user_key,
|
|
|
|
|
const Slice& largest_user_key) {
|
|
|
|
|
std::vector<std::vector<FileMetaData>> level_to_files;
|
|
|
|
|
dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
|
|
|
|
|
&level_to_files);
|
|
|
|
|
ASSERT_GT(level_to_files.size(), 1);
|
|
|
|
|
// L0 only has one SST file.
|
|
|
|
|
ASSERT_EQ(level_to_files[0].size(), 1);
|
|
|
|
|
auto file_meta = level_to_files[0][0];
|
|
|
|
|
ASSERT_EQ(smallest_user_key, file_meta.smallest.user_key());
|
|
|
|
|
ASSERT_EQ(largest_user_key, file_meta.largest.user_key());
|
2023-05-31 02:32:00 +00:00
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
TEST_P(RepairTestWithTimestamp, UnflushedSst) {
|
2023-05-31 02:32:00 +00:00
|
|
|
Destroy(last_options_);
|
|
|
|
|
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
bool paranoid_file_checks = std::get<0>(GetParam());
|
|
|
|
|
bool persist_udt = test::ShouldPersistUDT(std::get<1>(GetParam()));
|
|
|
|
|
std::string smallest_ukey_without_ts = "bar";
|
|
|
|
|
std::string largest_ukey_without_ts = "foo";
|
2023-05-31 02:32:00 +00:00
|
|
|
Options options = CurrentOptions();
|
|
|
|
|
options.env = env_;
|
|
|
|
|
options.create_if_missing = true;
|
2023-07-26 23:25:06 +00:00
|
|
|
std::string min_ts;
|
|
|
|
|
std::string write_ts;
|
|
|
|
|
PutFixed64(&min_ts, 0);
|
|
|
|
|
PutFixed64(&write_ts, 1);
|
|
|
|
|
options.comparator = test::BytewiseComparatorWithU64TsWrapper();
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
options.persist_user_defined_timestamps = persist_udt;
|
2023-07-26 23:25:06 +00:00
|
|
|
if (!persist_udt) {
|
|
|
|
|
options.allow_concurrent_memtable_write = false;
|
|
|
|
|
}
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
options.paranoid_file_checks = paranoid_file_checks;
|
2023-05-31 02:32:00 +00:00
|
|
|
|
|
|
|
|
ColumnFamilyOptions cf_options(options);
|
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
2023-12-04 19:17:32 +00:00
|
|
|
column_families.emplace_back(kDefaultColumnFamilyName, cf_options);
|
2023-05-31 02:32:00 +00:00
|
|
|
|
|
|
|
|
ASSERT_OK(DB::Open(options, dbname_, column_families, &handles_, &db_));
|
|
|
|
|
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
ASSERT_OK(Put(smallest_ukey_without_ts, write_ts,
|
|
|
|
|
smallest_ukey_without_ts + ":val"));
|
|
|
|
|
ASSERT_OK(
|
|
|
|
|
Put(largest_ukey_without_ts, write_ts, largest_ukey_without_ts + ":val"));
|
2023-05-31 02:32:00 +00:00
|
|
|
VectorLogPtr wal_files;
|
|
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
|
ASSERT_EQ(wal_files.size(), 1);
|
|
|
|
|
{
|
|
|
|
|
uint64_t total_ssts_size;
|
|
|
|
|
std::unordered_map<std::string, uint64_t> sst_files;
|
|
|
|
|
ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
|
|
|
|
|
ASSERT_EQ(total_ssts_size, 0);
|
|
|
|
|
}
|
|
|
|
|
// Need to get path before Close() deletes db_, but delete it after Close() to
|
|
|
|
|
// ensure Close() didn't change the manifest.
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, options));
|
|
|
|
|
ASSERT_OK(DB::Open(options, dbname_, column_families, &handles_, &db_));
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
|
ASSERT_EQ(wal_files.size(), 0);
|
|
|
|
|
{
|
|
|
|
|
uint64_t total_ssts_size;
|
|
|
|
|
std::unordered_map<std::string, uint64_t> sst_files;
|
|
|
|
|
ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
|
|
|
|
|
ASSERT_GT(total_ssts_size, 0);
|
|
|
|
|
}
|
|
|
|
|
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
// Check file boundaries are correct for different
|
|
|
|
|
// `persist_user_defined_timestamps` option values.
|
|
|
|
|
if (persist_udt) {
|
|
|
|
|
CheckFileBoundaries(smallest_ukey_without_ts + write_ts,
|
|
|
|
|
largest_ukey_without_ts + write_ts);
|
|
|
|
|
} else {
|
|
|
|
|
CheckFileBoundaries(smallest_ukey_without_ts + min_ts,
|
|
|
|
|
largest_ukey_without_ts + min_ts);
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-31 02:32:00 +00:00
|
|
|
ReadOptions read_opts;
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
Slice read_ts_slice = write_ts;
|
2023-05-31 02:32:00 +00:00
|
|
|
read_opts.timestamp = &read_ts_slice;
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
if (persist_udt) {
|
|
|
|
|
CheckGet(read_opts, smallest_ukey_without_ts,
|
|
|
|
|
smallest_ukey_without_ts + ":val", write_ts);
|
|
|
|
|
CheckGet(read_opts, largest_ukey_without_ts,
|
|
|
|
|
largest_ukey_without_ts + ":val", write_ts);
|
|
|
|
|
} else {
|
|
|
|
|
// TODO(yuzhangyu): currently when `persist_user_defined_timestamps` is
|
|
|
|
|
// false, ts is unconditionally stripped during flush.
|
|
|
|
|
// When `full_history_ts_low` is set and respected during flush.
|
|
|
|
|
// We should prohibit reading below `full_history_ts_low` all together.
|
|
|
|
|
CheckGet(read_opts, smallest_ukey_without_ts,
|
|
|
|
|
smallest_ukey_without_ts + ":val", min_ts);
|
|
|
|
|
CheckGet(read_opts, largest_ukey_without_ts,
|
|
|
|
|
largest_ukey_without_ts + ":val", min_ts);
|
|
|
|
|
}
|
2023-05-31 02:32:00 +00:00
|
|
|
}
|
|
|
|
|
|
Logically strip timestamp during flush (#11557)
Summary:
Logically strip the user-defined timestamp when L0 files are created during flush when `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` is false. Logically stripping timestamp here means replacing the original user-defined timestamp with a mininum timestamp, which for now is hard coded to be all zeros bytes.
While working on this, I caught a missing piece on the `BlockBuilder` level for this feature. The current quick path `std::min(buffer_size, last_key_size)` needs a bit tweaking to work for this feature. When user-defined timestamp is stripped during block building, on writing first entry or right after resetting, `buffer` is empty and `buffer_size` is zero as usual. However, in follow-up writes, depending on the size of the stripped user-defined timestamp, and the size of the value, what's in `buffer` can sometimes be smaller than `last_key_size`, leading `std::min(buffer_size, last_key_size)` to truncate the `last_key`. Previous test doesn't caught the bug because in those tests, the size of the stripped user-defined timestamps bytes is smaller than the length of the value. In order to avoid the conditional operation, this PR changed the original trivial `std::min` operation into an arithmetic operation. Since this is a change in a hot and performance critical path, I did the following benchmark to check no observable regression is introduced.
```TEST_TMPDIR=/dev/shm/rocksdb1 ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=50000000```
Compiled with DEBUG_LEVEL=0
Test vs. control runs simulaneous for better accuracy, units = ops/sec
PR vs base:
Round 1: 350652 vs 349055
Round 2: 365733 vs 364308
Round 3: 355681 vs 354475
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11557
Test Plan:
New timestamp specific test added or existing tests augmented, both are parameterized with `UserDefinedTimestampTestMode`:
`UserDefinedTimestampTestMode::kNormal` -> UDT feature enabled, write / read with min timestamp
`UserDefinedTimestampTestMode::kStripUserDefinedTimestamps` -> UDT feature enabled, write / read with min timestamp, set Options.persist_user_defined_timestamps to false.
```
make all check
./db_wal_test --gtest_filter="*WithTimestamp*"
./flush_job_test --gtest_filter="*WithTimestamp*"
./repair_test --gtest_filter="*WithTimestamp*"
./block_based_table_reader_test
```
Reviewed By: pdillinger
Differential Revision: D47027664
Pulled By: jowlyzhang
fbshipit-source-id: e729193b6334dfc63aaa736d684d907a022571f5
2023-06-29 22:50:50 +00:00
|
|
|
// Param 0: paranoid file check
|
|
|
|
|
// Param 1: test mode for the user-defined timestamp feature
|
|
|
|
|
INSTANTIATE_TEST_CASE_P(
|
|
|
|
|
UnflushedSst, RepairTestWithTimestamp,
|
|
|
|
|
::testing::Combine(
|
|
|
|
|
::testing::Bool(),
|
|
|
|
|
::testing::Values(
|
|
|
|
|
test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp,
|
|
|
|
|
test::UserDefinedTimestampTestMode::kNormal)));
|
|
|
|
|
|
2017-08-08 17:42:38 +00:00
|
|
|
TEST_F(RepairTest, SeparateWalDir) {
|
|
|
|
|
do {
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
|
DestroyAndReopen(options);
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Put("foo", "bar"));
|
2017-08-08 17:42:38 +00:00
|
|
|
VectorLogPtr wal_files;
|
|
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
|
ASSERT_EQ(wal_files.size(), 1);
|
2020-09-29 23:28:42 +00:00
|
|
|
{
|
|
|
|
|
uint64_t total_ssts_size;
|
|
|
|
|
std::unordered_map<std::string, uint64_t> sst_files;
|
2021-03-18 03:43:22 +00:00
|
|
|
ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_EQ(total_ssts_size, 0);
|
|
|
|
|
}
|
2017-08-08 17:42:38 +00:00
|
|
|
std::string manifest_path =
|
2022-11-02 21:34:24 +00:00
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
2017-08-08 17:42:38 +00:00
|
|
|
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, options));
|
|
|
|
|
|
|
|
|
|
// make sure that all WALs are converted to SSTables.
|
|
|
|
|
options.wal_dir = "";
|
|
|
|
|
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
2017-08-08 17:42:38 +00:00
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
|
ASSERT_EQ(wal_files.size(), 0);
|
2020-09-29 23:28:42 +00:00
|
|
|
{
|
|
|
|
|
uint64_t total_ssts_size;
|
|
|
|
|
std::unordered_map<std::string, uint64_t> sst_files;
|
2021-03-18 03:43:22 +00:00
|
|
|
ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_GT(total_ssts_size, 0);
|
|
|
|
|
}
|
2017-08-08 17:42:38 +00:00
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
ASSERT_EQ(Get("foo"), "bar");
|
|
|
|
|
|
2022-11-02 21:34:24 +00:00
|
|
|
} while (ChangeWalOptions());
|
2017-08-08 17:42:38 +00:00
|
|
|
}
|
|
|
|
|
|
2016-06-24 20:12:13 +00:00
|
|
|
TEST_F(RepairTest, RepairMultipleColumnFamilies) {
|
|
|
|
|
// Verify repair logic associates SST files with their original column
|
|
|
|
|
// families.
|
|
|
|
|
const int kNumCfs = 3;
|
|
|
|
|
const int kEntriesPerCf = 2;
|
|
|
|
|
DestroyAndReopen(CurrentOptions());
|
|
|
|
|
CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
|
|
|
|
|
for (int i = 0; i < kNumCfs; ++i) {
|
|
|
|
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
2022-05-06 20:03:58 +00:00
|
|
|
ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j)));
|
2016-06-24 20:12:13 +00:00
|
|
|
if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
|
|
|
|
|
// Leave one unflushed so we can verify WAL entries are properly
|
|
|
|
|
// associated with column families.
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Flush(i));
|
2016-06-24 20:12:13 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Need to get path before Close() deletes db_, but delete it after Close() to
|
|
|
|
|
// ensure Close() doesn't re-create the manifest.
|
|
|
|
|
std::string manifest_path =
|
|
|
|
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
|
|
|
|
Close();
|
|
|
|
|
ASSERT_OK(env_->FileExists(manifest_path));
|
|
|
|
|
ASSERT_OK(env_->DeleteFile(manifest_path));
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
|
|
|
|
|
|
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu1", "pikachu2"},
|
|
|
|
|
CurrentOptions());
|
|
|
|
|
for (int i = 0; i < kNumCfs; ++i) {
|
|
|
|
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
2022-05-06 20:03:58 +00:00
|
|
|
ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
|
2016-06-24 20:12:13 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-06-24 23:29:43 +00:00
|
|
|
TEST_F(RepairTest, RepairColumnFamilyOptions) {
|
|
|
|
|
// Verify repair logic uses correct ColumnFamilyOptions when repairing a
|
|
|
|
|
// database with different options for column families.
|
|
|
|
|
const int kNumCfs = 2;
|
|
|
|
|
const int kEntriesPerCf = 2;
|
|
|
|
|
|
|
|
|
|
Options opts(CurrentOptions()), rev_opts(CurrentOptions());
|
|
|
|
|
opts.comparator = BytewiseComparator();
|
|
|
|
|
rev_opts.comparator = ReverseBytewiseComparator();
|
|
|
|
|
|
|
|
|
|
DestroyAndReopen(opts);
|
|
|
|
|
CreateColumnFamilies({"reverse"}, rev_opts);
|
|
|
|
|
ReopenWithColumnFamilies({"default", "reverse"},
|
|
|
|
|
std::vector<Options>{opts, rev_opts});
|
|
|
|
|
for (int i = 0; i < kNumCfs; ++i) {
|
|
|
|
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
2022-05-06 20:03:58 +00:00
|
|
|
ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j)));
|
2016-06-24 23:29:43 +00:00
|
|
|
if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
|
|
|
|
|
// Leave one unflushed so we can verify RepairDB's flush logic
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Flush(i));
|
2016-06-24 23:29:43 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Close();
|
|
|
|
|
|
|
|
|
|
// RepairDB() records the comparator in the manifest, and DB::Open would fail
|
|
|
|
|
// if a different comparator were used.
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}, {"reverse", rev_opts}},
|
|
|
|
|
opts /* unknown_cf_opts */));
|
|
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"},
|
|
|
|
|
std::vector<Options>{opts, rev_opts}));
|
|
|
|
|
for (int i = 0; i < kNumCfs; ++i) {
|
|
|
|
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
2022-05-06 20:03:58 +00:00
|
|
|
ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
|
2016-06-24 23:29:43 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Examine table properties to verify RepairDB() used the right options when
|
|
|
|
|
// converting WAL->SST
|
|
|
|
|
TablePropertiesCollection fname_to_props;
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props));
|
2016-06-24 23:29:43 +00:00
|
|
|
ASSERT_EQ(fname_to_props.size(), 2U);
|
|
|
|
|
for (const auto& fname_and_props : fname_to_props) {
|
2022-07-12 20:30:35 +00:00
|
|
|
std::string comparator_name(rev_opts.comparator->Name());
|
2022-11-02 21:34:24 +00:00
|
|
|
ASSERT_EQ(comparator_name, fname_and_props.second->comparator_name);
|
2016-06-24 23:29:43 +00:00
|
|
|
}
|
2018-10-12 17:40:06 +00:00
|
|
|
Close();
|
2016-06-24 23:29:43 +00:00
|
|
|
|
|
|
|
|
// Also check comparator when it's provided via "unknown" CF options
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}},
|
|
|
|
|
rev_opts /* unknown_cf_opts */));
|
|
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"},
|
|
|
|
|
std::vector<Options>{opts, rev_opts}));
|
|
|
|
|
for (int i = 0; i < kNumCfs; ++i) {
|
|
|
|
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
2022-05-06 20:03:58 +00:00
|
|
|
ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
|
2016-06-24 23:29:43 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2017-02-16 18:25:06 +00:00
|
|
|
|
2017-09-22 19:37:59 +00:00
|
|
|
TEST_F(RepairTest, DbNameContainsTrailingSlash) {
|
|
|
|
|
{
|
|
|
|
|
bool tmp;
|
|
|
|
|
if (env_->AreFilesSame("", "", &tmp).IsNotSupported()) {
|
|
|
|
|
fprintf(stderr,
|
|
|
|
|
"skipping RepairTest.DbNameContainsTrailingSlash due to "
|
|
|
|
|
"unsupported Env::AreFilesSame\n");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 23:28:42 +00:00
|
|
|
ASSERT_OK(Put("key", "val"));
|
|
|
|
|
ASSERT_OK(Flush());
|
2017-09-22 19:37:59 +00:00
|
|
|
Close();
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions()));
|
2022-05-19 18:04:21 +00:00
|
|
|
ReopenWithSstIdVerify();
|
2017-09-22 19:37:59 +00:00
|
|
|
ASSERT_EQ(Get("key"), "val");
|
|
|
|
|
}
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
2016-03-18 22:18:42 +00:00
|
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
2022-10-18 07:35:35 +00:00
|
|
|
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
|
2016-03-18 22:18:42 +00:00
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
|
|
|
return RUN_ALL_TESTS();
|
|
|
|
|
}
|
2017-02-15 19:16:34 +00:00
|
|
|
|