From d755e53b87a436d761f2f6a4654b34a7df506054 Mon Sep 17 00:00:00 2001 From: sdong Date: Wed, 22 Oct 2014 18:24:14 -0700 Subject: [PATCH] Printing number of keys in DB Stats Summary: It is useful to print out number of keys in DB Stats Test Plan: ./db_bench --benchmarks fillrandom --num 1000000 -threads 16 -batch_size=16 and watch the outputs in LOG files Reviewers: MarkCallaghan, ljin, yhchiang, igor Reviewed By: igor Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D24513 --- db/db_impl.cc | 4 +++- db/internal_stats.cc | 24 ++++++++++++++++++------ db/internal_stats.h | 6 ++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 90f50174a8..6a2daad7d2 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -4129,7 +4129,7 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) { const uint64_t batch_size = WriteBatchInternal::ByteSize(updates); // Record statistics RecordTick(stats_, NUMBER_KEYS_WRITTEN, my_batch_count); - RecordTick(stats_, BYTES_WRITTEN, WriteBatchInternal::ByteSize(updates)); + RecordTick(stats_, BYTES_WRITTEN, batch_size); if (write_options.disableWAL) { flush_on_destroy_ = true; } @@ -4179,6 +4179,8 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) { // internal stats default_cf_internal_stats_->AddDBStats( InternalStats::BYTES_WRITTEN, batch_size); + default_cf_internal_stats_->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, + my_batch_count); if (!write_options.disableWAL) { default_cf_internal_stats_->AddDBStats( InternalStats::WAL_FILE_SYNCED, 1); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index aa3b3c850e..cfeb9c00de 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -293,16 +293,25 @@ void InternalStats::DumpDBStats(std::string* value) { value->append(buf); // Cumulative uint64_t user_bytes_written = db_stats_[InternalStats::BYTES_WRITTEN]; + uint64_t num_keys_written = db_stats_[InternalStats::NUMBER_KEYS_WRITTEN]; uint64_t write_other = db_stats_[InternalStats::WRITE_DONE_BY_OTHER]; uint64_t write_self = db_stats_[InternalStats::WRITE_DONE_BY_SELF]; uint64_t wal_bytes = db_stats_[InternalStats::WAL_FILE_BYTES]; uint64_t wal_synced = db_stats_[InternalStats::WAL_FILE_SYNCED]; uint64_t write_with_wal = db_stats_[InternalStats::WRITE_WITH_WAL]; // Data + // writes: total number of write requests. + // keys: total number of key updates issued by all the write requests + // batches: number of group commits issued to the DB. Each group can contain + // one or more writes. + // so writes/keys is the average number of put in multi-put or put + // writes/batches is the average group commit size. + // + // The format is the same for interval stats. snprintf(buf, sizeof(buf), - "Cumulative writes: %" PRIu64 " writes, %" PRIu64 " batches, " - "%.1f writes per batch, %.2f GB user ingest\n", - write_other + write_self, write_self, + "Cumulative writes: %" PRIu64 " writes, %" PRIu64 " keys, %" PRIu64 + " batches, %.1f writes per batch, %.2f GB user ingest\n", + write_other + write_self, num_keys_written, write_self, (write_other + write_self) / static_cast(write_self + 1), user_bytes_written / kGB); value->append(buf); @@ -318,11 +327,13 @@ void InternalStats::DumpDBStats(std::string* value) { // Interval uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other; uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self; + uint64_t interval_num_keys_written = + num_keys_written - db_stats_snapshot_.num_keys_written; snprintf(buf, sizeof(buf), - "Interval writes: %" PRIu64 " writes, %" PRIu64 " batches, " - "%.1f writes per batch, %.1f MB user ingest\n", + "Interval writes: %" PRIu64 " writes, %" PRIu64 " keys, %" PRIu64 + " batches, %.1f writes per batch, %.1f MB user ingest\n", interval_write_other + interval_write_self, - interval_write_self, + interval_num_keys_written, interval_write_self, static_cast(interval_write_other + interval_write_self) / (interval_write_self + 1), (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB); @@ -347,6 +358,7 @@ void InternalStats::DumpDBStats(std::string* value) { db_stats_snapshot_.ingest_bytes = user_bytes_written; db_stats_snapshot_.write_other = write_other; db_stats_snapshot_.write_self = write_self; + db_stats_snapshot_.num_keys_written = num_keys_written; db_stats_snapshot_.wal_bytes = wal_bytes; db_stats_snapshot_.wal_synced = wal_synced; db_stats_snapshot_.write_with_wal = write_with_wal; diff --git a/db/internal_stats.h b/db/internal_stats.h index 4d12a25127..84fd102896 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -67,6 +67,7 @@ class InternalStats { WAL_FILE_BYTES, WAL_FILE_SYNCED, BYTES_WRITTEN, + NUMBER_KEYS_WRITTEN, WRITE_DONE_BY_OTHER, WRITE_DONE_BY_SELF, WRITE_WITH_WAL, @@ -264,6 +265,11 @@ class InternalStats { // another thread. uint64_t write_other; uint64_t write_self; + // Total number of keys written. write_self and write_other measure number + // of write requests written, Each of the write request can contain updates + // to multiple keys. num_keys_written is total number of keys updated by all + // those writes. + uint64_t num_keys_written; double seconds_up; DBStatsSnapshot()