2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2014-04-15 20:39:26 +00:00
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2015-10-13 00:35:43 +00:00
|
|
|
#ifndef NDEBUG
|
2014-04-15 20:39:26 +00:00
|
|
|
|
|
|
|
#include "db/db_impl.h"
|
2017-04-06 02:02:00 +00:00
|
|
|
#include "monitoring/thread_status_updater.h"
|
2014-04-15 20:39:26 +00:00
|
|
|
|
2014-04-15 20:51:07 +00:00
|
|
|
namespace rocksdb {
|
|
|
|
|
2014-04-15 20:39:26 +00:00
|
|
|
uint64_t DBImpl::TEST_GetLevel0TotalSize() {
|
2015-02-05 05:39:45 +00:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-10-31 15:48:19 +00:00
|
|
|
return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0);
|
2014-04-15 20:39:26 +00:00
|
|
|
}
|
|
|
|
|
2017-09-28 00:37:08 +00:00
|
|
|
void DBImpl::TEST_SwitchWAL() {
|
2017-04-04 17:19:33 +00:00
|
|
|
WriteContext write_context;
|
2017-01-19 23:21:07 +00:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2017-09-28 00:37:08 +00:00
|
|
|
SwitchWAL(&write_context);
|
2017-01-19 23:21:07 +00:00
|
|
|
}
|
|
|
|
|
2014-04-15 20:39:26 +00:00
|
|
|
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
|
|
|
|
ColumnFamilyHandle* column_family) {
|
|
|
|
ColumnFamilyData* cfd;
|
|
|
|
if (column_family == nullptr) {
|
|
|
|
cfd = default_cf_handle_->cfd();
|
|
|
|
} else {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
cfd = cfh->cfd();
|
|
|
|
}
|
2015-02-05 05:39:45 +00:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-10-31 15:48:19 +00:00
|
|
|
return cfd->current()->storage_info()->MaxNextLevelOverlappingBytes();
|
2014-04-15 20:39:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::TEST_GetFilesMetaData(
|
|
|
|
ColumnFamilyHandle* column_family,
|
|
|
|
std::vector<std::vector<FileMetaData>>* metadata) {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
2015-02-05 05:39:45 +00:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-04-15 20:39:26 +00:00
|
|
|
metadata->resize(NumberLevels());
|
|
|
|
for (int level = 0; level < NumberLevels(); level++) {
|
2014-10-27 22:49:46 +00:00
|
|
|
const std::vector<FileMetaData*>& files =
|
2014-10-31 15:48:19 +00:00
|
|
|
cfd->current()->storage_info()->LevelFiles(level);
|
2014-04-15 20:39:26 +00:00
|
|
|
|
|
|
|
(*metadata)[level].clear();
|
|
|
|
for (const auto& f : files) {
|
|
|
|
(*metadata)[level].push_back(*f);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
|
2014-11-04 01:45:55 +00:00
|
|
|
return versions_->manifest_file_number();
|
2014-04-15 20:39:26 +00:00
|
|
|
}
|
|
|
|
|
2017-10-10 20:07:00 +00:00
|
|
|
uint64_t DBImpl::TEST_Current_Next_FileNo() {
|
|
|
|
return versions_->current_next_file_number();
|
|
|
|
}
|
|
|
|
|
2014-04-15 20:39:26 +00:00
|
|
|
Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
|
|
|
|
const Slice* end,
|
Allowing L0 -> L1 trivial move on sorted data
Summary:
This diff updates the logic of how we do trivial move, now trivial move can run on any number of files in input level as long as they are not overlapping
The conditions for trivial move have been updated
Introduced conditions:
- Trivial move cannot happen if we have a compaction filter (except if the compaction is not manual)
- Input level files cannot be overlapping
Removed conditions:
- Trivial move only run when the compaction is not manual
- Input level should can contain only 1 file
More context on what tests failed because of Trivial move
```
DBTest.CompactionsGenerateMultipleFiles
This test is expecting compaction on a file in L0 to generate multiple files in L1, this test will fail with trivial move because we end up with one file in L1
```
```
DBTest.NoSpaceCompactRange
This test expect compaction to fail when we force environment to report running out of space, of course this is not valid in trivial move situation
because trivial move does not need any extra space, and did not check for that
```
```
DBTest.DropWrites
Similar to DBTest.NoSpaceCompactRange
```
```
DBTest.DeleteObsoleteFilesPendingOutputs
This test expect that a file in L2 is deleted after it's moved to L3, this is not valid with trivial move because although the file was moved it is now used by L3
```
```
CuckooTableDBTest.CompactionIntoMultipleFiles
Same as DBTest.CompactionsGenerateMultipleFiles
```
This diff is based on a work by @sdong https://reviews.facebook.net/D34149
Test Plan: make -j64 check
Reviewers: rven, sdong, igor
Reviewed By: igor
Subscribers: yhchiang, ott, march, dhruba, sdong
Differential Revision: https://reviews.facebook.net/D34797
2015-06-04 23:51:25 +00:00
|
|
|
ColumnFamilyHandle* column_family,
|
|
|
|
bool disallow_trivial_move) {
|
2014-04-15 20:39:26 +00:00
|
|
|
ColumnFamilyData* cfd;
|
|
|
|
if (column_family == nullptr) {
|
|
|
|
cfd = default_cf_handle_->cfd();
|
|
|
|
} else {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
cfd = cfh->cfd();
|
|
|
|
}
|
|
|
|
int output_level =
|
2014-11-18 18:20:10 +00:00
|
|
|
(cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
|
|
|
|
cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
|
2014-04-15 20:39:26 +00:00
|
|
|
? level
|
|
|
|
: level + 1;
|
Running manual compactions in parallel with other automatic or manual compactions in restricted cases
Summary:
This diff provides a framework for doing manual
compactions in parallel with other compactions. We now have a deque of manual compactions. We also pass manual compactions as an argument from RunManualCompactions down to
BackgroundCompactions, so that RunManualCompactions can be reentrant.
Parallelism is controlled by the two routines
ConflictingManualCompaction to allow/disallow new parallel/manual
compactions based on already existing ManualCompactions. In this diff, by default manual compactions still have to run exclusive of other compactions. However, by setting the compaction option, exclusive_manual_compaction to false, it is possible to run other compactions in parallel with a manual compaction. However, we are still restricted to one manual compaction per column family at a time. All of these restrictions will be relaxed in future diffs.
I will be adding more tests later.
Test Plan: Rocksdb regression + new tests + valgrind
Reviewers: igor, anthony, IslamAbdelRahman, kradhakrishnan, yhchiang, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D47973
2015-12-14 19:20:34 +00:00
|
|
|
return RunManualCompaction(cfd, level, output_level, 0, begin, end, true,
|
Allowing L0 -> L1 trivial move on sorted data
Summary:
This diff updates the logic of how we do trivial move, now trivial move can run on any number of files in input level as long as they are not overlapping
The conditions for trivial move have been updated
Introduced conditions:
- Trivial move cannot happen if we have a compaction filter (except if the compaction is not manual)
- Input level files cannot be overlapping
Removed conditions:
- Trivial move only run when the compaction is not manual
- Input level should can contain only 1 file
More context on what tests failed because of Trivial move
```
DBTest.CompactionsGenerateMultipleFiles
This test is expecting compaction on a file in L0 to generate multiple files in L1, this test will fail with trivial move because we end up with one file in L1
```
```
DBTest.NoSpaceCompactRange
This test expect compaction to fail when we force environment to report running out of space, of course this is not valid in trivial move situation
because trivial move does not need any extra space, and did not check for that
```
```
DBTest.DropWrites
Similar to DBTest.NoSpaceCompactRange
```
```
DBTest.DeleteObsoleteFilesPendingOutputs
This test expect that a file in L2 is deleted after it's moved to L3, this is not valid with trivial move because although the file was moved it is now used by L3
```
```
CuckooTableDBTest.CompactionIntoMultipleFiles
Same as DBTest.CompactionsGenerateMultipleFiles
```
This diff is based on a work by @sdong https://reviews.facebook.net/D34149
Test Plan: make -j64 check
Reviewers: rven, sdong, igor
Reviewed By: igor
Subscribers: yhchiang, ott, march, dhruba, sdong
Differential Revision: https://reviews.facebook.net/D34797
2015-06-04 23:51:25 +00:00
|
|
|
disallow_trivial_move);
|
2014-04-15 20:39:26 +00:00
|
|
|
}
|
|
|
|
|
2017-10-03 16:08:07 +00:00
|
|
|
Status DBImpl::TEST_SwitchMemtable(ColumnFamilyData* cfd) {
|
|
|
|
WriteContext write_context;
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
if (cfd == nullptr) {
|
|
|
|
cfd = default_cf_handle_->cfd();
|
|
|
|
}
|
|
|
|
return SwitchMemtable(cfd, &write_context);
|
|
|
|
}
|
|
|
|
|
2016-04-18 18:11:51 +00:00
|
|
|
Status DBImpl::TEST_FlushMemTable(bool wait, ColumnFamilyHandle* cfh) {
|
2014-04-15 20:39:26 +00:00
|
|
|
FlushOptions fo;
|
|
|
|
fo.wait = wait;
|
2016-04-18 18:11:51 +00:00
|
|
|
ColumnFamilyData* cfd;
|
|
|
|
if (cfh == nullptr) {
|
|
|
|
cfd = default_cf_handle_->cfd();
|
|
|
|
} else {
|
|
|
|
auto cfhi = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh);
|
|
|
|
cfd = cfhi->cfd();
|
|
|
|
}
|
|
|
|
return FlushMemTable(cfd, fo);
|
2014-04-15 20:39:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
|
|
|
|
ColumnFamilyData* cfd;
|
|
|
|
if (column_family == nullptr) {
|
|
|
|
cfd = default_cf_handle_->cfd();
|
|
|
|
} else {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
cfd = cfh->cfd();
|
|
|
|
}
|
|
|
|
return WaitForFlushMemTable(cfd);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::TEST_WaitForCompact() {
|
|
|
|
// Wait until the compaction completes
|
|
|
|
|
|
|
|
// TODO: a bug here. This function actually does not necessarily
|
|
|
|
// wait for compact. It actually waits for scheduled compaction
|
|
|
|
// OR flush to finish.
|
|
|
|
|
2015-02-05 05:39:45 +00:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2017-08-03 22:36:28 +00:00
|
|
|
while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
|
|
|
|
bg_flush_scheduled_) &&
|
|
|
|
bg_error_.ok()) {
|
2014-04-15 20:39:26 +00:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
return bg_error_;
|
|
|
|
}
|
2014-04-29 17:27:58 +00:00
|
|
|
|
2014-09-05 22:20:05 +00:00
|
|
|
void DBImpl::TEST_LockMutex() {
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::TEST_UnlockMutex() {
|
|
|
|
mutex_.Unlock();
|
|
|
|
}
|
|
|
|
|
|
|
|
void* DBImpl::TEST_BeginWrite() {
|
2015-08-05 23:56:28 +00:00
|
|
|
auto w = new WriteThread::Writer();
|
|
|
|
write_thread_.EnterUnbatched(w, &mutex_);
|
2014-09-05 22:20:05 +00:00
|
|
|
return reinterpret_cast<void*>(w);
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::TEST_EndWrite(void* w) {
|
2014-09-12 23:23:58 +00:00
|
|
|
auto writer = reinterpret_cast<WriteThread::Writer*>(w);
|
2015-08-05 23:56:28 +00:00
|
|
|
write_thread_.ExitUnbatched(writer);
|
2014-09-08 15:01:25 +00:00
|
|
|
delete writer;
|
2014-09-05 22:20:05 +00:00
|
|
|
}
|
|
|
|
|
2015-03-30 19:04:10 +00:00
|
|
|
size_t DBImpl::TEST_LogsToFreeSize() {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
return logs_to_free_.size();
|
|
|
|
}
|
|
|
|
|
2015-07-02 21:27:00 +00:00
|
|
|
uint64_t DBImpl::TEST_LogfileNumber() {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
return logfile_number_;
|
|
|
|
}
|
|
|
|
|
2015-11-04 01:52:17 +00:00
|
|
|
Status DBImpl::TEST_GetAllImmutableCFOptions(
|
|
|
|
std::unordered_map<std::string, const ImmutableCFOptions*>* iopts_map) {
|
|
|
|
std::vector<std::string> cf_names;
|
|
|
|
std::vector<const ImmutableCFOptions*> iopts;
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
|
|
|
cf_names.push_back(cfd->GetName());
|
|
|
|
iopts.push_back(cfd->ioptions());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
iopts_map->clear();
|
|
|
|
for (size_t i = 0; i < cf_names.size(); ++i) {
|
|
|
|
iopts_map->insert({cf_names[i], iopts[i]});
|
|
|
|
}
|
|
|
|
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2016-04-18 18:11:51 +00:00
|
|
|
uint64_t DBImpl::TEST_FindMinLogContainingOutstandingPrep() {
|
|
|
|
return FindMinLogContainingOutstandingPrep();
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() {
|
|
|
|
return FindMinPrepLogReferencedByMemTable();
|
|
|
|
}
|
2016-05-17 20:11:56 +00:00
|
|
|
|
|
|
|
Status DBImpl::TEST_GetLatestMutableCFOptions(
|
|
|
|
ColumnFamilyHandle* column_family, MutableCFOptions* mutable_cf_options) {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
*mutable_cf_options = *cfh->cfd()->GetLatestMutableCFOptions();
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2016-10-14 19:25:39 +00:00
|
|
|
int DBImpl::TEST_BGCompactionsAllowed() const {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2017-05-24 18:25:38 +00:00
|
|
|
return GetBGJobLimits().max_compactions;
|
|
|
|
}
|
|
|
|
|
|
|
|
int DBImpl::TEST_BGFlushesAllowed() const {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
return GetBGJobLimits().max_flushes;
|
2016-10-14 19:25:39 +00:00
|
|
|
}
|
|
|
|
|
2017-10-23 21:20:22 +00:00
|
|
|
SequenceNumber DBImpl::TEST_GetLatestVisibleSequenceNumber() const {
|
|
|
|
if (concurrent_prepare_ && seq_per_batch_) {
|
|
|
|
return versions_->LastToBeWrittenSequence();
|
|
|
|
} else {
|
|
|
|
return versions_->LastSequence();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-04-15 20:51:07 +00:00
|
|
|
} // namespace rocksdb
|
2015-10-13 00:35:43 +00:00
|
|
|
#endif // NDEBUG
|