dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More changes from upstream. 

git-svn-id: https://leveldb.googlecode.com/svn/trunk@12 62dab493-f737-651d-591e-8d6aee1b9529
This commit is contained in:
jorlow@chromium.org 2011-03-22 18:32:49 +00:00
parent 9df5aa892e
commit 13b72af77b
9 changed files with 405 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

264
db/db_bench.cc
View File

@ -17,11 +17,14 @@
// Comma-separated list of operations to run in the specified order
// Actual benchmarks:
// writeseq -- write N values in sequential key order
// writerandom -- write N values in random key order
// writebig -- write N/1000 100K valuesin random order
// readseq -- read N values sequentially
// readrandom -- read N values in random order
// fillseq -- write N values in sequential key order in async mode
// fillrandom -- write N values in random key order in async mode
// overwrite -- overwrite N values in random key order in async mode
// fillsync -- write N/100 values in random key order in sync mode
// fill100K -- write N/1000 100K values in random order in async mode
// readseq -- read N values sequentially
// readreverse -- read N values in reverse order
// readrandom -- read N values in random order
// Meta operations:
// compact -- Compact the entire DB
// heapprofile -- Dump a heap profile (if supported by this port)
@ -30,10 +33,10 @@
// tenth -- divide N by 10 (i.e., following benchmarks are smaller)
// normal -- reset N back to its normal value (1000000)
static const char* FLAGS_benchmarks =
"writeseq,"
"writeseq,"
"writerandom,"
"sync,tenth,tenth,writerandom,nosync,normal,"
"fillseq,"
"fillrandom,"
"overwrite,"
"fillsync,"
"readseq,"
"readreverse,"
"readrandom,"
@ -41,7 +44,7 @@ static const char* FLAGS_benchmarks =
"readseq,"
"readreverse,"
"readrandom,"
"writebig";
"fill100K";
// Number of key/values to place in database
static int FLAGS_num = 1000000;
@ -51,7 +54,7 @@ static int FLAGS_value_size = 100;
// Arrange to generate values that shrink to this fraction of
// their original size after compression
static double FLAGS_compression_ratio = 0.25;
static double FLAGS_compression_ratio = 0.5;
// Print histogram of operation timings
static bool FLAGS_histogram = false;
@ -93,6 +96,19 @@ class RandomGenerator {
return Slice(data_.data() + pos_ - len, len);
}
};
static Slice TrimSpace(Slice s) {
int start = 0;
while (start < s.size() && isspace(s[start])) {
start++;
}
int limit = s.size();
while (limit > start && isspace(s[limit-1])) {
limit--;
}
return Slice(s.data() + start, limit - start);
}
}
class Benchmark {
@ -100,7 +116,6 @@ class Benchmark {
Cache* cache_;
DB* db_;
int num_;
bool sync_;
int heap_counter_;
double start_;
double last_op_finish_;
@ -114,6 +129,70 @@ class Benchmark {
int done_;
int next_report_; // When to report next
void PrintHeader() {
const int kKeySize = 16;
PrintEnvironment();
fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
FLAGS_value_size,
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
fprintf(stdout, "Entries: %d\n", num_);
fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
(((kKeySize + FLAGS_value_size) * num_) / 1048576.0));
fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
/ 1048576.0));
PrintWarnings();
fprintf(stdout, "------------------------------------------------\n");
}
void PrintWarnings() {
#if defined(__GNUC__) && !defined(__OPTIMIZE__)
fprintf(stdout,
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
);
#endif
#ifndef NDEBUG
fprintf(stdout,
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
#endif
}
void PrintEnvironment() {
fprintf(stderr, "LevelDB: version %d.%d\n",
kMajorVersion, kMinorVersion);
#if defined(__linux)
time_t now = time(NULL);
fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
if (cpuinfo != NULL) {
char line[1000];
int num_cpus = 0;
std::string cpu_type;
std::string cache_size;
while (fgets(line, sizeof(line), cpuinfo) != NULL) {
const char* sep = strchr(line, ':');
if (sep == NULL) {
continue;
}
Slice key = TrimSpace(Slice(line, sep - 1 - line));
Slice val = TrimSpace(Slice(sep + 1));
if (key == "model name") {
++num_cpus;
cpu_type = val.ToString();
} else if (key == "cache size") {
cache_size = val.ToString();
}
}
fclose(cpuinfo);
fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
}
#endif
}
void Start() {
start_ = Env::Default()->NowMicros() * 1e-6;
bytes_ = 0;
@ -164,9 +243,10 @@ class Benchmark {
snprintf(rate, sizeof(rate), "%5.1f MB/s",
(bytes_ / 1048576.0) / (finish - start_));
if (!message_.empty()) {
message_.push_back(' ');
message_ = std::string(rate) + " " + message_;
} else {
message_ = rate;
}
message_.append(rate);
}
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
@ -183,14 +263,16 @@ class Benchmark {
public:
enum Order {
SEQUENTIAL,
REVERSE, // Currently only supported for reads
RANDOM
};
enum DBState {
FRESH,
EXISTING
};
Benchmark() : cache_(NewLRUCache(200<<20)),
db_(NULL),
num_(FLAGS_num),
sync_(false),
heap_counter_(0),
bytes_(0),
rand_(301) {
@ -210,19 +292,8 @@ class Benchmark {
}
void Run() {
Options options;
options.create_if_missing = true;
options.max_open_files = 10000;
options.block_cache = cache_;
options.write_buffer_size = FLAGS_write_buffer_size;
Start();
Status s = DB::Open(options, "/tmp/dbbench", &db_);
Stop("open");
if (!s.ok()) {
fprintf(stderr, "open error: %s\n", s.ToString().c_str());
exit(1);
}
PrintHeader();
Open();
const char* benchmarks = FLAGS_benchmarks;
while (benchmarks != NULL) {
@ -237,30 +308,30 @@ class Benchmark {
}
Start();
if (name == Slice("writeseq")) {
Write(SEQUENTIAL, num_, FLAGS_value_size);
} else if (name == Slice("writerandom")) {
Write(RANDOM, num_, FLAGS_value_size);
} else if (name == Slice("writebig")) {
Write(RANDOM, num_ / 1000, 100 * 1000);
WriteOptions write_options;
write_options.sync = false;
if (name == Slice("fillseq")) {
Write(write_options, SEQUENTIAL, FRESH, num_, FLAGS_value_size);
} else if (name == Slice("fillrandom")) {
Write(write_options, RANDOM, FRESH, num_, FLAGS_value_size);
} else if (name == Slice("overwrite")) {
Write(write_options, RANDOM, EXISTING, num_, FLAGS_value_size);
} else if (name == Slice("fillsync")) {
write_options.sync = true;
Write(write_options, RANDOM, FRESH, num_ / 100, FLAGS_value_size);
} else if (name == Slice("fill100K")) {
Write(write_options, RANDOM, FRESH, num_ / 1000, 100 * 1000);
} else if (name == Slice("readseq")) {
Read(SEQUENTIAL);
ReadSequential();
} else if (name == Slice("readreverse")) {
Read(REVERSE);
ReadReverse();
} else if (name == Slice("readrandom")) {
Read(RANDOM);
ReadRandom();
} else if (name == Slice("compact")) {
Compact();
} else if (name == Slice("heapprofile")) {
HeapProfile();
} else if (name == Slice("sync")) {
sync_ = true;
} else if (name == Slice("nosync")) {
sync_ = false;
} else if (name == Slice("tenth")) {
num_ = num_ / 10;
} else if (name == Slice("normal")) {
num_ = FLAGS_num;
} else {
fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
}
@ -268,16 +339,44 @@ class Benchmark {
}
}
void Write(Order order, int num_entries, int value_size) {
private:
void Open() {
assert(db_ == NULL);
Options options;
options.create_if_missing = true;
options.max_open_files = 10000;
options.block_cache = cache_;
options.write_buffer_size = FLAGS_write_buffer_size;
Status s = DB::Open(options, "/tmp/dbbench", &db_);
if (!s.ok()) {
fprintf(stderr, "open error: %s\n", s.ToString().c_str());
exit(1);
}
}
void Write(const WriteOptions& options, Order order, DBState state,
int num_entries, int value_size) {
if (state == FRESH) {
delete db_;
db_ = NULL;
DestroyDB("/tmp/dbbench", Options());
Open();
Start(); // Do not count time taken to destroy/open
}
if (num_entries != num_) {
char msg[100];
snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
message_ = msg;
}
WriteBatch batch;
Status s;
std::string val;
WriteOptions options;
options.sync = sync_;
for (int i = 0; i < num_entries; i++) {
const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % FLAGS_num);
char key[100];
snprintf(key, sizeof(key), "%012d", k);
snprintf(key, sizeof(key), "%016d", k);
batch.Clear();
batch.Put(key, gen_.Generate(value_size));
s = db_->Write(options, &batch);
@ -290,42 +389,37 @@ class Benchmark {
}
}
void Read(Order order) {
void ReadSequential() {
Iterator* iter = db_->NewIterator(ReadOptions());
int i = 0;
for (iter->SeekToFirst(); i < num_ && iter->Valid(); iter->Next()) {
bytes_ += iter->key().size() + iter->value().size();
FinishedSingleOp();
++i;
}
delete iter;
}
void ReadReverse() {
Iterator* iter = db_->NewIterator(ReadOptions());
int i = 0;
for (iter->SeekToLast(); i < num_ && iter->Valid(); iter->Prev()) {
bytes_ += iter->key().size() + iter->value().size();
FinishedSingleOp();
++i;
}
delete iter;
}
void ReadRandom() {
ReadOptions options;
switch (order) {
case SEQUENTIAL: {
Iterator* iter = db_->NewIterator(options);
int i = 0;
for (iter->SeekToFirst(); i < num_ && iter->Valid(); iter->Next()) {
bytes_ += iter->key().size() + iter->value().size();
FinishedSingleOp();
++i;
}
delete iter;
break;
}
case REVERSE: {
Iterator* iter = db_->NewIterator(options);
int i = 0;
for (iter->SeekToLast(); i < num_ && iter->Valid(); iter->Prev()) {
bytes_ += iter->key().size() + iter->value().size();
FinishedSingleOp();
++i;
}
delete iter;
break;
}
case RANDOM: {
std::string value;
for (int i = 0; i < num_; i++) {
char key[100];
const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % FLAGS_num);
snprintf(key, sizeof(key), "%012d", k);
db_->Get(options, key, &value);
FinishedSingleOp();
}
break;
}
std::string value;
for (int i = 0; i < num_; i++) {
char key[100];
const int k = rand_.Next() % FLAGS_num;
snprintf(key, sizeof(key), "%016d", k);
db_->Get(options, key, &value);
FinishedSingleOp();
}
}

20
db/db_impl.cc
View File

@ -532,8 +532,9 @@ void DBImpl::BackgroundCompaction() {
}
Status status;
if (c->num_input_files(0) == 1 && c->num_input_files(1) == 0) {
if (c->IsTrivialMove()) {
// Move file to next level
assert(c->num_input_files(0) == 1);
FileMetaData* f = c->input(0, 0);
c->edit()->DeleteFile(c->level(), f->number);
c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
@ -718,8 +719,18 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
bool has_current_user_key = false;
SequenceNumber last_sequence_for_key = kMaxSequenceNumber;
for (; input->Valid() && !shutting_down_.Acquire_Load(); ) {
// Handle key/value, add to state, etc.
Slice key = input->key();
InternalKey tmp_internal_key;
tmp_internal_key.DecodeFrom(key);
if (compact->compaction->ShouldStopBefore(tmp_internal_key) &&
compact->builder != NULL) {
status = FinishCompactionOutputFile(compact, input);
if (!status.ok()) {
break;
}
}
// Handle key/value, add to state, etc.
bool drop = false;
if (!ParseInternalKey(key, &ikey)) {
// Do not hide error keys
@ -855,6 +866,11 @@ Iterator* DBImpl::TEST_NewInternalIterator() {
return NewInternalIterator(ReadOptions(), &ignored);
}
int64 DBImpl::TEST_MaxNextLevelOverlappingBytes() {
MutexLock l(&mutex_);
return versions_->MaxNextLevelOverlappingBytes();
}
Status DBImpl::Get(const ReadOptions& options,
const Slice& key,
std::string* value) {

4
db/db_impl.h
View File

@ -55,6 +55,10 @@ class DBImpl : public DB {
// The returned iterator should be deleted when no longer needed.
Iterator* TEST_NewInternalIterator();
// Return the maximum overlapping data (in bytes) at next level for any
// file at a level >= 1.
int64 TEST_MaxNextLevelOverlappingBytes();
private:
friend class DB;

74
db/db_test.cc
View File

@ -72,15 +72,19 @@ class DBTest {
}
Status Put(const std::string& k, const std::string& v) {
WriteOptions options;
options.sync = false;
WriteBatch batch;
batch.Put(k, v);
return db_->Write(WriteOptions(), &batch);
return db_->Write(options, &batch);
}
Status Delete(const std::string& k) {
WriteOptions options;
options.sync = false;
WriteBatch batch;
batch.Delete(k);
return db_->Write(WriteOptions(), &batch);
return db_->Write(options, &batch);
}
std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
@ -176,6 +180,35 @@ class DBTest {
fprintf(stderr, "Found %d live large value files\n", (int)live.size());
return live;
}
void Compact(const Slice& start, const Slice& limit) {
dbfull()->TEST_CompactMemTable();
int max_level_with_files = 1;
for (int level = 1; level < config::kNumLevels; level++) {
uint64_t v;
char name[100];
snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level);
if (dbfull()->GetProperty(name, &v) && v > 0) {
max_level_with_files = level;
}
}
for (int level = 0; level < max_level_with_files; level++) {
dbfull()->TEST_CompactRange(level, "", "~");
}
}
void DumpFileCounts(const char* label) {
fprintf(stderr, "---\n%s:\n", label);
fprintf(stderr, "maxoverlap: %lld\n",
static_cast<long long>(
dbfull()->TEST_MaxNextLevelOverlappingBytes()));
for (int level = 0; level < config::kNumLevels; level++) {
int num = NumTableFilesAtLevel(level);
if (num > 0) {
fprintf(stderr, " level %3d : %d files\n", level, num);
}
}
}
};
TEST(DBTest, Empty) {
@ -315,6 +348,43 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) {
}
}
TEST(DBTest, SparseMerge) {
Options options;
options.compression = kNoCompression;
Reopen(&options);
// Suppose there is:
// small amount of data with prefix A
// large amount of data with prefix B
// small amount of data with prefix C
// and that recent updates have made small changes to all three prefixes.
// Check that we do not do a compaction that merges all of B in one shot.
const std::string value(1000, 'x');
Put("A", "va");
// Write approximately 100MB of "B" values
for (int i = 0; i < 100000; i++) {
char key[100];
snprintf(key, sizeof(key), "B%010d", i);
Put(key, value);
}
Put("C", "vc");
Compact("", "z");
// Make sparse update
Put("A", "va2");
Put("B100", "bvalue2");
Put("C", "vc2");
dbfull()->TEST_CompactMemTable();
// Compactions should not cause us to create a situation where
// a file overlaps too much data at the next level.
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
dbfull()->TEST_CompactRange(0, "", "z");
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
dbfull()->TEST_CompactRange(1, "", "z");
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
}
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
bool result = (val >= low) && (val <= high);
if (!result) {

122
db/version_set.cc
View File

@ -20,6 +20,10 @@
namespace leveldb {
// Maximum number of overlaps in grandparent (i.e., level+2) before we
// stop building a single file in a level->level+1 compaction.
static const int kMaxGrandParentFiles = 10;
static double MaxBytesForLevel(int level) {
if (level == 0) {
return 4 * 1048576.0;
@ -509,7 +513,7 @@ Status VersionSet::Finalize(Version* v) {
double best_score = -1;
Status s;
for (int level = 0; s.ok() && level < config::kNumLevels; level++) {
for (int level = 0; s.ok() && level < config::kNumLevels-1; level++) {
s = SortLevel(v, level);
// Compute the ratio of current size to size limit.
@ -751,6 +755,25 @@ void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
}
}
int64 VersionSet::MaxNextLevelOverlappingBytes() {
int64 result = 0;
std::vector<FileMetaData*> overlaps;
for (int level = 0; level < config::kNumLevels - 1; level++) {
for (int i = 0; i < current_->files_[level].size(); i++) {
const FileMetaData* f = current_->files_[level][i];
GetOverlappingInputs(level+1, f->smallest, f->largest, &overlaps);
int64 sum = 0;
for (int j = 0; j < overlaps.size(); j++) {
sum += overlaps[j]->file_size;
}
if (sum > result) {
result = sum;
}
}
}
return result;
}
// Store in "*inputs" all files in "level" that overlap [begin,end]
void VersionSet::GetOverlappingInputs(
int level,
@ -797,6 +820,18 @@ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
}
}
// Stores the minimal range that covers all entries in inputs1 and inputs2
// in *smallest, *largest.
// REQUIRES: inputs is not empty
void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2,
InternalKey* smallest,
InternalKey* largest) {
std::vector<FileMetaData*> all = inputs1;
all.insert(all.end(), inputs2.begin(), inputs2.end());
GetRange(all, smallest, largest);
}
Iterator* VersionSet::MakeInputIterator(Compaction* c) {
ReadOptions options;
options.verify_checksums = options_->paranoid_checks;
@ -836,6 +871,7 @@ Compaction* VersionSet::PickCompaction() {
}
const int level = current_->compaction_level_;
assert(level >= 0);
assert(level+1 < config::kNumLevels);
Compaction* c = new Compaction(level);
c->input_version_ = current_;
@ -855,31 +891,36 @@ Compaction* VersionSet::PickCompaction() {
c->inputs_[0].push_back(current_->files_[level][0]);
}
// Find the range we are compacting
InternalKey smallest, largest;
GetRange(c->inputs_[0], &smallest, &largest);
// Files in level 0 may overlap each other, so pick up all overlapping ones
if (level == 0) {
InternalKey smallest, largest;
GetRange(c->inputs_[0], &smallest, &largest);
// Note that the next call will discard the file we placed in
// c->inputs_[0] earlier and replace it with an overlapping set
// which will include the picked file.
GetOverlappingInputs(0, smallest, largest, &c->inputs_[0]);
assert(!c->inputs_[0].empty());
GetRange(c->inputs_[0], &smallest, &largest);
}
SetupOtherInputs(c);
return c;
}
void VersionSet::SetupOtherInputs(Compaction* c) {
const int level = c->level();
InternalKey smallest, largest;
GetRange(c->inputs_[0], &smallest, &largest);
GetOverlappingInputs(level+1, smallest, largest, &c->inputs_[1]);
// Get entire range covered by compaction
InternalKey all_start, all_limit;
GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
// See if we can grow the number of inputs in "level" without
// changing the number of "level+1" files we pick up.
if (!c->inputs_[1].empty()) {
// Get entire range covered by compaction
std::vector<FileMetaData*> all = c->inputs_[0];
all.insert(all.end(), c->inputs_[1].begin(), c->inputs_[1].end());
InternalKey all_start, all_limit;
GetRange(all, &all_start, &all_limit);
std::vector<FileMetaData*> expanded0;
GetOverlappingInputs(level, all_start, all_limit, &expanded0);
if (expanded0.size() > c->inputs_[0].size()) {
@ -899,10 +940,17 @@ Compaction* VersionSet::PickCompaction() {
largest = new_limit;
c->inputs_[0] = expanded0;
c->inputs_[1] = expanded1;
GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
}
}
}
// Compute the set of grandparent files that overlap this compaction
// (parent == level+1; grandparent == level+2)
if (level + 2 < config::kNumLevels) {
GetOverlappingInputs(level + 2, all_start, all_limit, &c->grandparents_);
}
if (false) {
Log(env_, options_->info_log, "Compacting %d '%s' .. '%s'",
level,
@ -916,8 +964,6 @@ Compaction* VersionSet::PickCompaction() {
// key range next time.
compact_pointer_[level] = largest.Encode().ToString();
c->edit_.SetCompactPointer(level, largest);
return c;
}
Compaction* VersionSet::CompactRange(
@ -934,25 +980,16 @@ Compaction* VersionSet::CompactRange(
c->input_version_ = current_;
c->input_version_->Ref();
c->inputs_[0] = inputs;
// Find the range we are compacting
InternalKey smallest, largest;
GetRange(c->inputs_[0], &smallest, &largest);
GetOverlappingInputs(level+1, smallest, largest, &c->inputs_[1]);
if (false) {
Log(env_, options_->info_log, "Compacting %d '%s' .. '%s'",
level,
EscapeString(smallest.Encode()).c_str(),
EscapeString(largest.Encode()).c_str());
}
SetupOtherInputs(c);
return c;
}
Compaction::Compaction(int level)
: level_(level),
max_output_file_size_(MaxFileSizeForLevel(level)),
input_version_(NULL) {
input_version_(NULL),
grandparent_index_(0),
output_start_(-1) {
for (int i = 0; i < config::kNumLevels; i++) {
level_ptrs_[i] = 0;
}
@ -964,6 +1001,15 @@ Compaction::~Compaction() {
}
}
bool Compaction::IsTrivialMove() const {
// Avoid a move if there are lots of overlapping grandparent files.
// Otherwise, the move could create a parent file that will require
// a very expensive merge later on.
return (num_input_files(0) == 1
&& num_input_files(1) == 0
&& grandparents_.size() <= kMaxGrandParentFiles);
}
void Compaction::AddInputDeletions(VersionEdit* edit) {
for (int which = 0; which < 2; which++) {
for (int i = 0; i < inputs_[which].size(); i++) {
@ -993,6 +1039,28 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
return true;
}
bool Compaction::ShouldStopBefore(const InternalKey& key) {
// Scan to find earliest grandparent file that contains key.
const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
while (grandparent_index_ < grandparents_.size() &&
icmp->Compare(key, grandparents_[grandparent_index_]->largest) > 0) {
grandparent_index_++;
}
// First call?
if (output_start_ < 0) {
output_start_ = grandparent_index_;
}
if (grandparent_index_ - output_start_ + 1 > kMaxGrandParentFiles) {
// Too many overlaps for current output; start new output
output_start_ = grandparent_index_;
return true;
} else {
return false;
}
}
void Compaction::ReleaseInputs() {
if (input_version_ != NULL) {
input_version_->Unref();

25
db/version_set.h
View File

@ -139,6 +139,10 @@ class VersionSet {
const InternalKey& begin,
const InternalKey& end);
// Return the maximum overlapping data (in bytes) at next level for any
// file at a level >= 1.
int64 MaxNextLevelOverlappingBytes();
// Create an iterator that reads over the compaction inputs for "*c".
// The caller should delete the iterator when no longer needed.
Iterator* MakeInputIterator(Compaction* c);
@ -195,6 +199,13 @@ class VersionSet {
InternalKey* smallest,
InternalKey* largest);
void GetRange2(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2,
InternalKey* smallest,
InternalKey* largest);
void SetupOtherInputs(Compaction* c);
Env* const env_;
const std::string dbname_;
const Options* const options_;
@ -250,6 +261,10 @@ class Compaction {
// Maximum size of files to build during this compaction.
uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
// Is this a trivial compaction that can be implemented by just
// moving a single input file to the next level (no merging or splitting)
bool IsTrivialMove() const;
// Add all inputs to this compaction as delete operations to *edit.
void AddInputDeletions(VersionEdit* edit);
@ -258,6 +273,10 @@ class Compaction {
// in levels greater than "level+1".
bool IsBaseLevelForKey(const Slice& user_key);
// Returns true iff we should stop building the current output
// before processing "key".
bool ShouldStopBefore(const InternalKey& key);
// Release the input version for the compaction, once the compaction
// is successful.
void ReleaseInputs();
@ -276,6 +295,12 @@ class Compaction {
// Each compaction reads inputs from "level_" and "level_+1"
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
// State used to check for number of of overlapping grandparent files
// (parent == level_ + 1, grandparent == level_ + 2)
std::vector<FileMetaData*> grandparents_;
int grandparent_index_; // Index in grandparent_starts_
int output_start_; // Index in grandparent_starts_ where output started
// State for implementing IsBaseLevelForKey
// level_ptrs_ holds indices into input_version_->levels_: our state

10
doc/impl.html
View File

@ -123,8 +123,14 @@ one level-0 file in case some of these files overlap each other.
A compaction merges the contents of the picked files to produce a
sequence of level-(L+1) files. We switch to producing a new
level-(L+1) file after the current output file has reached the target
file size (2MB). The old files are discarded and the new files are
added to the serving state.
file size (2MB). We also switch to a new output file when the key
range of the current output file has grown enough to overlap more then
ten level-(L+2) files. This last rule ensures that a later compaction
of a level-(L+1) file will not pick up too much data from level-(L+2).
<p>
The old files are discarded and the new files are added to the serving
state.
<p>
Compactions for a particular level rotate through the key space. In

3
include/db.h
View File

@ -12,6 +12,9 @@
namespace leveldb {
static const int kMajorVersion = 1;
static const int kMinorVersion = 0;
struct Options;
struct ReadOptions;
struct WriteOptions;

2
leveldb.gyp
View File

@ -4,7 +4,7 @@
{
'variables': {
'use_snappy%': 1,
'use_snappy%': 0,
},
'target_defaults': {
'defines': [