dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CF] Move InternalStats to ColumnFamilyData 

Summary: InternalStats is a messy thing, keeping both DB data and column family data. However, it's better off living in ColumnFamilyData than in DBImpl. For now, at least.

Test Plan: make check

Reviewers: dhruba, kailiu, haobo, sdong

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15879
This commit is contained in:
Igor Canadi 2014-02-04 17:45:19 -08:00
parent 73f62255c1
commit 7b9f134959
5 changed files with 40 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
db/column_family.cc
View File

@ -14,6 +14,7 @@
#include <algorithm> #include <algorithm>
#include "db/version_set.h" #include "db/version_set.h"
#include "db/internal_stats.h"
#include "db/compaction_picker.h" #include "db/compaction_picker.h"
#include "db/table_properties_collector.h" #include "db/table_properties_collector.h"
#include "util/hash_skiplist_rep.h" #include "util/hash_skiplist_rep.h"
@ -130,7 +131,7 @@ void SuperVersion::Init(MemTable* new_mem, MemTableListVersion* new_imm,
ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name, ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions, Version* dummy_versions,
const ColumnFamilyOptions& options, const ColumnFamilyOptions& options,
Logger* logger) const Options* db_options)
: id_(id), : id_(id),
name_(name), name_(name),
dummy_versions_(dummy_versions), dummy_versions_(dummy_versions),
@ -147,12 +148,16 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
prev_(nullptr), prev_(nullptr),
log_number_(0), log_number_(0),
need_slowdown_for_num_level0_files_(false) { need_slowdown_for_num_level0_files_(false) {
if (db_options != nullptr) {
internal_stats_.reset(new InternalStats(options.num_levels, db_options->env,
db_options->statistics.get()));
if (options_.compaction_style == kCompactionStyleUniversal) { if (options_.compaction_style == kCompactionStyleUniversal) {
compaction_picker_.reset(new UniversalCompactionPicker( compaction_picker_.reset(new UniversalCompactionPicker(
&options_, &internal_comparator_, logger)); &options_, &internal_comparator_, db_options->info_log.get()));
} else { } else {
compaction_picker_.reset( compaction_picker_.reset(new LevelCompactionPicker(
new LevelCompactionPicker(&options_, &internal_comparator_, logger)); &options_, &internal_comparator_, db_options->info_log.get()));
}
} }
} }
@ -180,6 +185,10 @@ ColumnFamilyData::~ColumnFamilyData() {
} }
} }
InternalStats* ColumnFamilyData::internal_stats() {
return internal_stats_.get();
}
void ColumnFamilyData::SetCurrent(Version* current) { void ColumnFamilyData::SetCurrent(Version* current) {
current_ = current; current_ = current;
need_slowdown_for_num_level0_files_ = need_slowdown_for_num_level0_files_ =
@ -221,11 +230,11 @@ SuperVersion* ColumnFamilyData::InstallSuperVersion(
return nullptr; return nullptr;
} }
ColumnFamilySet::ColumnFamilySet(Logger* logger) ColumnFamilySet::ColumnFamilySet(const Options* db_options)
: max_column_family_(0), : max_column_family_(0),
dummy_cfd_(new ColumnFamilyData(0, "", nullptr, ColumnFamilyOptions(), dummy_cfd_(new ColumnFamilyData(0, "", nullptr, ColumnFamilyOptions(),
nullptr)), nullptr)),
logger_(logger) { db_options_(db_options) {
// initialize linked list // initialize linked list
dummy_cfd_->prev_.store(dummy_cfd_); dummy_cfd_->prev_.store(dummy_cfd_);
dummy_cfd_->next_.store(dummy_cfd_); dummy_cfd_->next_.store(dummy_cfd_);
@ -281,7 +290,7 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
assert(column_families_.find(name) == column_families_.end()); assert(column_families_.find(name) == column_families_.end());
column_families_.insert({name, id}); column_families_.insert({name, id});
ColumnFamilyData* new_cfd = ColumnFamilyData* new_cfd =
new ColumnFamilyData(id, name, dummy_versions, options, logger_); new ColumnFamilyData(id, name, dummy_versions, options, db_options_);
column_family_data_.insert({id, new_cfd}); column_family_data_.insert({id, new_cfd});
max_column_family_ = std::max(max_column_family_, id); max_column_family_ = std::max(max_column_family_, id);
// add to linked list // add to linked list

11
db/column_family.h
View File

@ -27,6 +27,7 @@ class MemTableListVersion;
class CompactionPicker; class CompactionPicker;
class Compaction; class Compaction;
class InternalKey; class InternalKey;
class InternalStats;
// holds references to memtable, all immutable memtables and version // holds references to memtable, all immutable memtables and version
struct SuperVersion { struct SuperVersion {
@ -65,7 +66,7 @@ class ColumnFamilyData {
public: public:
ColumnFamilyData(uint32_t id, const std::string& name, ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions, const ColumnFamilyOptions& options, Version* dummy_versions, const ColumnFamilyOptions& options,
Logger* logger); const Options* db_options);
~ColumnFamilyData(); ~ColumnFamilyData();
uint32_t GetID() const { return id_; } uint32_t GetID() const { return id_; }
@ -77,6 +78,7 @@ class ColumnFamilyData {
uint64_t GetLogNumber() const { return log_number_; } uint64_t GetLogNumber() const { return log_number_; }
ColumnFamilyOptions* options() { return &options_; } ColumnFamilyOptions* options() { return &options_; }
InternalStats* internal_stats();
MemTableList* imm() { return &imm_; } MemTableList* imm() { return &imm_; }
MemTable* mem() { return mem_; } MemTable* mem() { return mem_; }
@ -133,6 +135,8 @@ class ColumnFamilyData {
ColumnFamilyOptions options_; ColumnFamilyOptions options_;
std::unique_ptr<InternalStats> internal_stats_;
MemTable* mem_; MemTable* mem_;
MemTableList imm_; MemTableList imm_;
SuperVersion* super_version_; SuperVersion* super_version_;
@ -182,7 +186,7 @@ class ColumnFamilySet {
ColumnFamilyData* current_; ColumnFamilyData* current_;
}; };
explicit ColumnFamilySet(Logger* logger); explicit ColumnFamilySet(const Options* db_options_);
~ColumnFamilySet(); ~ColumnFamilySet();
ColumnFamilyData* GetDefault() const; ColumnFamilyData* GetDefault() const;
@ -215,7 +219,8 @@ class ColumnFamilySet {
std::vector<ColumnFamilyData*> droppped_column_families_; std::vector<ColumnFamilyData*> droppped_column_families_;
uint32_t max_column_family_; uint32_t max_column_family_;
ColumnFamilyData* dummy_cfd_; ColumnFamilyData* dummy_cfd_;
Logger* logger_; // TODO(icanadi) change to DBOptions
const Options* const db_options_;
}; };
class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables { class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {

22
db/db_impl.cc
View File

@ -227,8 +227,6 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
last_stats_dump_time_microsec_(0), last_stats_dump_time_microsec_(0),
default_interval_to_delete_obsolete_WAL_(600), default_interval_to_delete_obsolete_WAL_(600),
flush_on_destroy_(false), flush_on_destroy_(false),
internal_stats_(options.num_levels, options.env,
options.statistics.get()),
delayed_writes_(0), delayed_writes_(0),
storage_options_(options), storage_options_(options),
bg_work_gate_closed_(false), bg_work_gate_closed_(false),
@ -1044,7 +1042,7 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
stats.micros = env_->NowMicros() - start_micros; stats.micros = env_->NowMicros() - start_micros;
stats.bytes_written = meta.file_size; stats.bytes_written = meta.file_size;
stats.files_out_levelnp1 = 1; stats.files_out_levelnp1 = 1;
internal_stats_.AddCompactionStats(level, stats); cfd->internal_stats()->AddCompactionStats(level, stats);
RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size); RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size);
return s; return s;
} }
@ -1132,7 +1130,7 @@ Status DBImpl::WriteLevel0Table(ColumnFamilyData* cfd,
InternalStats::CompactionStats stats; InternalStats::CompactionStats stats;
stats.micros = env_->NowMicros() - start_micros; stats.micros = env_->NowMicros() - start_micros;
stats.bytes_written = meta.file_size; stats.bytes_written = meta.file_size;
internal_stats_.AddCompactionStats(level, stats); cfd->internal_stats()->AddCompactionStats(level, stats);
RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size); RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size);
return s; return s;
} }
@ -2608,7 +2606,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact,
LogFlush(options_.info_log); LogFlush(options_.info_log);
mutex_.Lock(); mutex_.Lock();
internal_stats_.AddCompactionStats(compact->compaction->output_level(), cfd->internal_stats()->AddCompactionStats(compact->compaction->output_level(),
stats); stats);
// if there were any unused file number (mostly in case of // if there were any unused file number (mostly in case of
@ -3349,7 +3347,8 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) {
delayed = sw.ElapsedMicros(); delayed = sw.ElapsedMicros();
} }
RecordTick(options_.statistics.get(), STALL_L0_SLOWDOWN_MICROS, delayed); RecordTick(options_.statistics.get(), STALL_L0_SLOWDOWN_MICROS, delayed);
internal_stats_.RecordWriteStall(InternalStats::LEVEL0_SLOWDOWN, delayed); cfd->internal_stats()->RecordWriteStall(InternalStats::LEVEL0_SLOWDOWN,
delayed);
allow_delay = false; // Do not delay a single write more than once allow_delay = false; // Do not delay a single write more than once
mutex_.Lock(); mutex_.Lock();
delayed_writes_++; delayed_writes_++;
@ -3375,8 +3374,8 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) {
} }
RecordTick(options_.statistics.get(), RecordTick(options_.statistics.get(),
STALL_MEMTABLE_COMPACTION_MICROS, stall); STALL_MEMTABLE_COMPACTION_MICROS, stall);
internal_stats_.RecordWriteStall(InternalStats::MEMTABLE_COMPACTION, cfd->internal_stats()->RecordWriteStall(
stall); InternalStats::MEMTABLE_COMPACTION, stall);
} else if (cfd->current()->NumLevelFiles(0) >= } else if (cfd->current()->NumLevelFiles(0) >=
cfd->options()->level0_stop_writes_trigger) { cfd->options()->level0_stop_writes_trigger) {
// There are too many level-0 files. // There are too many level-0 files.
@ -3390,7 +3389,8 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) {
stall = sw.ElapsedMicros(); stall = sw.ElapsedMicros();
} }
RecordTick(options_.statistics.get(), STALL_L0_NUM_FILES_MICROS, stall); RecordTick(options_.statistics.get(), STALL_L0_NUM_FILES_MICROS, stall);
internal_stats_.RecordWriteStall(InternalStats::LEVEL0_NUM_FILES, stall); cfd->internal_stats()->RecordWriteStall(InternalStats::LEVEL0_NUM_FILES,
stall);
} else if (allow_hard_rate_limit_delay && options_.hard_rate_limit > 1.0 && } else if (allow_hard_rate_limit_delay && options_.hard_rate_limit > 1.0 &&
(score = cfd->current()->MaxCompactionScore()) > (score = cfd->current()->MaxCompactionScore()) >
cfd->options()->hard_rate_limit) { cfd->options()->hard_rate_limit) {
@ -3404,7 +3404,7 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) {
env_->SleepForMicroseconds(1000); env_->SleepForMicroseconds(1000);
delayed = sw.ElapsedMicros(); delayed = sw.ElapsedMicros();
} }
internal_stats_.RecordLevelNSlowdown(max_level, delayed); cfd->internal_stats()->RecordLevelNSlowdown(max_level, delayed);
// Make sure the following value doesn't round to zero. // Make sure the following value doesn't round to zero.
uint64_t rate_limit = std::max((delayed / 1000), (uint64_t) 1); uint64_t rate_limit = std::max((delayed / 1000), (uint64_t) 1);
rate_limit_delay_millis += rate_limit; rate_limit_delay_millis += rate_limit;
@ -3505,7 +3505,7 @@ bool DBImpl::GetProperty(const ColumnFamilyHandle& column_family,
MutexLock l(&mutex_); MutexLock l(&mutex_);
auto cfd = versions_->GetColumnFamilySet()->GetColumnFamily(column_family.id); auto cfd = versions_->GetColumnFamilySet()->GetColumnFamily(column_family.id);
assert(cfd != nullptr); assert(cfd != nullptr);
return internal_stats_.GetProperty(property, value, cfd); return cfd->internal_stats()->GetProperty(property, value, cfd);
} }
void DBImpl::GetApproximateSizes(const ColumnFamilyHandle& column_family, void DBImpl::GetApproximateSizes(const ColumnFamilyHandle& column_family,

2
db/db_impl.h
View File

@ -460,8 +460,6 @@ class DBImpl : public DB {
bool flush_on_destroy_; // Used when disableWAL is true. bool flush_on_destroy_; // Used when disableWAL is true.
InternalStats internal_stats_;
static const int KEEP_LOG_FILE_NUM = 1000; static const int KEEP_LOG_FILE_NUM = 1000;
std::string db_absolute_path_; std::string db_absolute_path_;

2
db/version_set.cc
View File

@ -1374,7 +1374,7 @@ class VersionSet::Builder {
VersionSet::VersionSet(const std::string& dbname, const Options* options, VersionSet::VersionSet(const std::string& dbname, const Options* options,
const EnvOptions& storage_options, const EnvOptions& storage_options,
TableCache* table_cache) TableCache* table_cache)
: column_family_set_(new ColumnFamilySet(options->info_log.get())), : column_family_set_(new ColumnFamilySet(options)),
env_(options->env), env_(options->env),
dbname_(dbname), dbname_(dbname),
options_(options), options_(options),