2014-01-30 04:40:41 +00:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
2017-04-28 00:50:56 +00:00
|
|
|
// This source code is also licensed under the GPLv2 license found in the
|
|
|
|
// COPYING file in the root directory of this source tree.
|
2014-01-30 04:40:41 +00:00
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#include "db/internal_stats.h"
|
2014-09-05 06:14:37 +00:00
|
|
|
|
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-07-14 13:56:52 +00:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 06:14:37 +00:00
|
|
|
#endif
|
|
|
|
|
2014-07-14 13:56:52 +00:00
|
|
|
#include <inttypes.h>
|
2015-08-25 19:03:54 +00:00
|
|
|
#include <string>
|
2015-03-30 18:28:25 +00:00
|
|
|
#include <algorithm>
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
#include <utility>
|
2014-01-30 04:40:41 +00:00
|
|
|
#include <vector>
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
#include "db/column_family.h"
|
2014-11-25 20:01:27 +00:00
|
|
|
|
2014-08-26 23:26:29 +00:00
|
|
|
#include "db/db_impl.h"
|
2014-11-25 04:44:49 +00:00
|
|
|
#include "util/string_util.h"
|
2014-01-30 04:40:41 +00:00
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
2014-11-25 20:01:27 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2016-11-12 04:45:47 +00:00
|
|
|
|
|
|
|
const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
|
|
|
|
{
|
|
|
|
{LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
|
|
|
|
{LevelStatType::COMPACTED_FILES,
|
|
|
|
LevelStat{"CompactedFiles", "CompactedFiles"}},
|
2017-04-06 00:18:21 +00:00
|
|
|
{LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}},
|
2016-11-12 04:45:47 +00:00
|
|
|
{LevelStatType::SCORE, LevelStat{"Score", "Score"}},
|
2017-04-06 00:18:21 +00:00
|
|
|
{LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}},
|
|
|
|
{LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}},
|
|
|
|
{LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB)"}},
|
|
|
|
{LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB)"}},
|
|
|
|
{LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB)"}},
|
|
|
|
{LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB)"}},
|
2016-11-12 04:45:47 +00:00
|
|
|
{LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}},
|
2017-04-06 00:18:21 +00:00
|
|
|
{LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s)"}},
|
|
|
|
{LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s)"}},
|
|
|
|
{LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec)"}},
|
|
|
|
{LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt)"}},
|
|
|
|
{LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec)"}},
|
2016-11-12 04:45:47 +00:00
|
|
|
{LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}},
|
|
|
|
{LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}},
|
|
|
|
};
|
|
|
|
|
2014-07-11 22:03:30 +00:00
|
|
|
namespace {
|
|
|
|
const double kMB = 1048576.0;
|
|
|
|
const double kGB = kMB * 1024;
|
2015-09-02 20:58:22 +00:00
|
|
|
const double kMicrosInSec = 1000000.0;
|
2014-07-11 22:03:30 +00:00
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
|
2016-11-12 04:45:47 +00:00
|
|
|
int written_size =
|
|
|
|
snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str());
|
|
|
|
auto hdr = [](LevelStatType t) {
|
|
|
|
return InternalStats::compaction_level_stats.at(t).header_name.c_str();
|
|
|
|
};
|
|
|
|
int line_size = snprintf(
|
|
|
|
buf + written_size, len - written_size,
|
2017-04-06 00:18:21 +00:00
|
|
|
"Level %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
|
2016-11-12 04:45:47 +00:00
|
|
|
// Note that we skip COMPACTED_FILES and merge it with Files column
|
2017-04-06 00:18:21 +00:00
|
|
|
hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_BYTES),
|
2016-11-12 04:45:47 +00:00
|
|
|
hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB),
|
|
|
|
hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB),
|
|
|
|
hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB),
|
|
|
|
hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP),
|
|
|
|
hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS),
|
|
|
|
hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT),
|
|
|
|
hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN),
|
|
|
|
hdr(LevelStatType::KEY_DROP));
|
|
|
|
|
|
|
|
written_size += line_size;
|
|
|
|
snprintf(buf + written_size, len - written_size, "%s\n",
|
|
|
|
std::string(line_size, '-').c_str());
|
2014-07-11 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
void PrepareLevelStats(std::map<LevelStatType, double>* level_stats,
|
|
|
|
int num_files, int being_compacted,
|
|
|
|
double total_file_size, double score, double w_amp,
|
|
|
|
const InternalStats::CompactionStats& stats) {
|
2015-06-18 06:40:34 +00:00
|
|
|
uint64_t bytes_read =
|
|
|
|
stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
|
|
|
|
int64_t bytes_new =
|
|
|
|
stats.bytes_written - stats.bytes_read_output_level;
|
2015-09-02 20:58:22 +00:00
|
|
|
double elapsed = (stats.micros + 1) / kMicrosInSec;
|
2014-07-11 22:03:30 +00:00
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
(*level_stats)[LevelStatType::NUM_FILES] = num_files;
|
|
|
|
(*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted;
|
2017-04-06 00:18:21 +00:00
|
|
|
(*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size;
|
2016-11-12 04:45:47 +00:00
|
|
|
(*level_stats)[LevelStatType::SCORE] = score;
|
|
|
|
(*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB;
|
|
|
|
(*level_stats)[LevelStatType::RN_GB] =
|
|
|
|
stats.bytes_read_non_output_levels / kGB;
|
|
|
|
(*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB;
|
|
|
|
(*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB;
|
|
|
|
(*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB;
|
|
|
|
(*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB;
|
|
|
|
(*level_stats)[LevelStatType::WRITE_AMP] = w_amp;
|
|
|
|
(*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed;
|
|
|
|
(*level_stats)[LevelStatType::WRITE_MBPS] =
|
|
|
|
stats.bytes_written / kMB / elapsed;
|
|
|
|
(*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec;
|
|
|
|
(*level_stats)[LevelStatType::COMP_COUNT] = stats.count;
|
|
|
|
(*level_stats)[LevelStatType::AVG_SEC] =
|
|
|
|
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count;
|
|
|
|
(*level_stats)[LevelStatType::KEY_IN] =
|
|
|
|
static_cast<double>(stats.num_input_records);
|
|
|
|
(*level_stats)[LevelStatType::KEY_DROP] =
|
|
|
|
static_cast<double>(stats.num_dropped_records);
|
|
|
|
}
|
|
|
|
|
|
|
|
void PrintLevelStats(char* buf, size_t len, const std::string& name,
|
|
|
|
const std::map<LevelStatType, double>& stat_value) {
|
2014-07-11 22:03:30 +00:00
|
|
|
snprintf(buf, len,
|
2017-04-06 00:18:21 +00:00
|
|
|
"%4s " /* Level */
|
|
|
|
"%6d/%-3d " /* Files */
|
|
|
|
"%8s " /* Size */
|
|
|
|
"%5.1f " /* Score */
|
|
|
|
"%8.1f " /* Read(GB) */
|
|
|
|
"%7.1f " /* Rn(GB) */
|
|
|
|
"%8.1f " /* Rnp1(GB) */
|
|
|
|
"%9.1f " /* Write(GB) */
|
|
|
|
"%8.1f " /* Wnew(GB) */
|
|
|
|
"%9.1f " /* Moved(GB) */
|
|
|
|
"%5.1f " /* W-Amp */
|
|
|
|
"%8.1f " /* Rd(MB/s) */
|
|
|
|
"%8.1f " /* Wr(MB/s) */
|
|
|
|
"%9.0f " /* Comp(sec) */
|
|
|
|
"%9d " /* Comp(cnt) */
|
|
|
|
"%8.3f " /* Avg(sec) */
|
|
|
|
"%7s " /* KeyIn */
|
|
|
|
"%6s\n", /* KeyDrop */
|
2015-09-02 20:58:22 +00:00
|
|
|
name.c_str(),
|
2016-11-12 04:45:47 +00:00
|
|
|
static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)),
|
|
|
|
static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)),
|
2017-04-06 00:18:21 +00:00
|
|
|
BytesToHumanString(
|
|
|
|
static_cast<uint64_t>(stat_value.at(LevelStatType::SIZE_BYTES)))
|
|
|
|
.c_str(),
|
2016-11-12 04:45:47 +00:00
|
|
|
stat_value.at(LevelStatType::SCORE),
|
|
|
|
stat_value.at(LevelStatType::READ_GB),
|
|
|
|
stat_value.at(LevelStatType::RN_GB),
|
|
|
|
stat_value.at(LevelStatType::RNP1_GB),
|
|
|
|
stat_value.at(LevelStatType::WRITE_GB),
|
|
|
|
stat_value.at(LevelStatType::W_NEW_GB),
|
|
|
|
stat_value.at(LevelStatType::MOVED_GB),
|
|
|
|
stat_value.at(LevelStatType::WRITE_AMP),
|
|
|
|
stat_value.at(LevelStatType::READ_MBPS),
|
|
|
|
stat_value.at(LevelStatType::WRITE_MBPS),
|
|
|
|
stat_value.at(LevelStatType::COMP_SEC),
|
|
|
|
static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)),
|
|
|
|
stat_value.at(LevelStatType::AVG_SEC),
|
|
|
|
NumberToHumanString(
|
|
|
|
static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN)))
|
|
|
|
.c_str(),
|
|
|
|
NumberToHumanString(static_cast<std::int64_t>(
|
|
|
|
stat_value.at(LevelStatType::KEY_DROP)))
|
|
|
|
.c_str());
|
|
|
|
}
|
|
|
|
|
|
|
|
void PrintLevelStats(char* buf, size_t len, const std::string& name,
|
|
|
|
int num_files, int being_compacted, double total_file_size,
|
|
|
|
double score, double w_amp,
|
|
|
|
const InternalStats::CompactionStats& stats) {
|
|
|
|
std::map<LevelStatType, double> level_stats;
|
|
|
|
PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size,
|
|
|
|
score, w_amp, stats);
|
|
|
|
PrintLevelStats(buf, len, name, level_stats);
|
2014-07-11 22:03:30 +00:00
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
|
|
|
|
// Assumes that trailing numbers represent an optional argument. This requires
|
|
|
|
// property names to not end with numbers.
|
|
|
|
std::pair<Slice, Slice> GetPropertyNameAndArg(const Slice& property) {
|
|
|
|
Slice name = property, arg = property;
|
|
|
|
size_t sfx_len = 0;
|
|
|
|
while (sfx_len < property.size() &&
|
|
|
|
isdigit(property[property.size() - sfx_len - 1])) {
|
|
|
|
++sfx_len;
|
|
|
|
}
|
|
|
|
name.remove_suffix(sfx_len);
|
|
|
|
arg.remove_prefix(property.size() - sfx_len);
|
|
|
|
return {name, arg};
|
2014-07-11 22:03:30 +00:00
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
} // anonymous namespace
|
2014-07-11 22:03:30 +00:00
|
|
|
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string rocksdb_prefix = "rocksdb.";
|
|
|
|
|
|
|
|
static const std::string num_files_at_level_prefix = "num-files-at-level";
|
2016-04-21 01:46:54 +00:00
|
|
|
static const std::string compression_ratio_at_level_prefix =
|
|
|
|
"compression-ratio-at-level";
|
2015-04-01 02:03:18 +00:00
|
|
|
static const std::string allstats = "stats";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string sstables = "sstables";
|
|
|
|
static const std::string cfstats = "cfstats";
|
2017-04-11 15:38:22 +00:00
|
|
|
static const std::string cfstats_no_file_histogram =
|
|
|
|
"cfstats-no-file-histogram";
|
|
|
|
static const std::string cf_file_histogram = "cf-file-histogram";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string dbstats = "dbstats";
|
|
|
|
static const std::string levelstats = "levelstats";
|
|
|
|
static const std::string num_immutable_mem_table = "num-immutable-mem-table";
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
static const std::string num_immutable_mem_table_flushed =
|
|
|
|
"num-immutable-mem-table-flushed";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string mem_table_flush_pending = "mem-table-flush-pending";
|
|
|
|
static const std::string compaction_pending = "compaction-pending";
|
|
|
|
static const std::string background_errors = "background-errors";
|
|
|
|
static const std::string cur_size_active_mem_table =
|
|
|
|
"cur-size-active-mem-table";
|
2016-01-21 19:38:15 +00:00
|
|
|
static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables";
|
|
|
|
static const std::string size_all_mem_tables = "size-all-mem-tables";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string num_entries_active_mem_table =
|
|
|
|
"num-entries-active-mem-table";
|
|
|
|
static const std::string num_entries_imm_mem_tables =
|
|
|
|
"num-entries-imm-mem-tables";
|
|
|
|
static const std::string num_deletes_active_mem_table =
|
|
|
|
"num-deletes-active-mem-table";
|
|
|
|
static const std::string num_deletes_imm_mem_tables =
|
|
|
|
"num-deletes-imm-mem-tables";
|
|
|
|
static const std::string estimate_num_keys = "estimate-num-keys";
|
|
|
|
static const std::string estimate_table_readers_mem =
|
|
|
|
"estimate-table-readers-mem";
|
|
|
|
static const std::string is_file_deletions_enabled =
|
|
|
|
"is-file-deletions-enabled";
|
|
|
|
static const std::string num_snapshots = "num-snapshots";
|
|
|
|
static const std::string oldest_snapshot_time = "oldest-snapshot-time";
|
|
|
|
static const std::string num_live_versions = "num-live-versions";
|
2016-03-03 21:18:56 +00:00
|
|
|
static const std::string current_version_number =
|
|
|
|
"current-super-version-number";
|
2015-07-22 04:33:20 +00:00
|
|
|
static const std::string estimate_live_data_size = "estimate-live-data-size";
|
2016-12-28 19:53:29 +00:00
|
|
|
static const std::string min_log_number_to_keep = "min-log-number-to-keep";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string base_level = "base-level";
|
2015-08-20 18:47:19 +00:00
|
|
|
static const std::string total_sst_files_size = "total-sst-files-size";
|
2015-08-14 04:42:20 +00:00
|
|
|
static const std::string estimate_pending_comp_bytes =
|
|
|
|
"estimate-pending-compaction-bytes";
|
2015-08-25 19:03:54 +00:00
|
|
|
static const std::string aggregated_table_properties =
|
|
|
|
"aggregated-table-properties";
|
|
|
|
static const std::string aggregated_table_properties_at_level =
|
|
|
|
aggregated_table_properties + "-at-level";
|
2015-10-17 07:16:36 +00:00
|
|
|
static const std::string num_running_compactions = "num-running-compactions";
|
|
|
|
static const std::string num_running_flushes = "num-running-flushes";
|
2017-03-29 18:42:56 +00:00
|
|
|
static const std::string actual_delayed_write_rate =
|
|
|
|
"actual-delayed-write-rate";
|
|
|
|
static const std::string is_write_stopped = "is-write-stopped";
|
2015-03-27 01:09:08 +00:00
|
|
|
|
|
|
|
const std::string DB::Properties::kNumFilesAtLevelPrefix =
|
|
|
|
rocksdb_prefix + num_files_at_level_prefix;
|
2016-04-21 01:46:54 +00:00
|
|
|
const std::string DB::Properties::kCompressionRatioAtLevelPrefix =
|
|
|
|
rocksdb_prefix + compression_ratio_at_level_prefix;
|
2015-04-01 02:03:18 +00:00
|
|
|
const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
|
|
|
|
const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
|
2017-04-11 15:38:22 +00:00
|
|
|
const std::string DB::Properties::kCFStatsNoFileHistogram =
|
|
|
|
rocksdb_prefix + cfstats_no_file_histogram;
|
|
|
|
const std::string DB::Properties::kCFFileHistogram =
|
|
|
|
rocksdb_prefix + cf_file_histogram;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
|
2016-01-21 18:59:36 +00:00
|
|
|
const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kNumImmutableMemTable =
|
|
|
|
rocksdb_prefix + num_immutable_mem_table;
|
2016-01-21 18:59:36 +00:00
|
|
|
const std::string DB::Properties::kNumImmutableMemTableFlushed =
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
rocksdb_prefix + num_immutable_mem_table_flushed;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kMemTableFlushPending =
|
|
|
|
rocksdb_prefix + mem_table_flush_pending;
|
|
|
|
const std::string DB::Properties::kCompactionPending =
|
|
|
|
rocksdb_prefix + compaction_pending;
|
2015-10-17 07:16:36 +00:00
|
|
|
const std::string DB::Properties::kNumRunningCompactions =
|
|
|
|
rocksdb_prefix + num_running_compactions;
|
|
|
|
const std::string DB::Properties::kNumRunningFlushes =
|
|
|
|
rocksdb_prefix + num_running_flushes;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kBackgroundErrors =
|
|
|
|
rocksdb_prefix + background_errors;
|
|
|
|
const std::string DB::Properties::kCurSizeActiveMemTable =
|
|
|
|
rocksdb_prefix + cur_size_active_mem_table;
|
|
|
|
const std::string DB::Properties::kCurSizeAllMemTables =
|
2015-08-19 20:32:09 +00:00
|
|
|
rocksdb_prefix + cur_size_all_mem_tables;
|
2016-01-21 19:38:15 +00:00
|
|
|
const std::string DB::Properties::kSizeAllMemTables =
|
|
|
|
rocksdb_prefix + size_all_mem_tables;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kNumEntriesActiveMemTable =
|
|
|
|
rocksdb_prefix + num_entries_active_mem_table;
|
|
|
|
const std::string DB::Properties::kNumEntriesImmMemTables =
|
|
|
|
rocksdb_prefix + num_entries_imm_mem_tables;
|
|
|
|
const std::string DB::Properties::kNumDeletesActiveMemTable =
|
|
|
|
rocksdb_prefix + num_deletes_active_mem_table;
|
|
|
|
const std::string DB::Properties::kNumDeletesImmMemTables =
|
|
|
|
rocksdb_prefix + num_deletes_imm_mem_tables;
|
|
|
|
const std::string DB::Properties::kEstimateNumKeys =
|
|
|
|
rocksdb_prefix + estimate_num_keys;
|
|
|
|
const std::string DB::Properties::kEstimateTableReadersMem =
|
|
|
|
rocksdb_prefix + estimate_table_readers_mem;
|
|
|
|
const std::string DB::Properties::kIsFileDeletionsEnabled =
|
|
|
|
rocksdb_prefix + is_file_deletions_enabled;
|
|
|
|
const std::string DB::Properties::kNumSnapshots =
|
|
|
|
rocksdb_prefix + num_snapshots;
|
|
|
|
const std::string DB::Properties::kOldestSnapshotTime =
|
|
|
|
rocksdb_prefix + oldest_snapshot_time;
|
|
|
|
const std::string DB::Properties::kNumLiveVersions =
|
|
|
|
rocksdb_prefix + num_live_versions;
|
2016-03-03 21:18:56 +00:00
|
|
|
const std::string DB::Properties::kCurrentSuperVersionNumber =
|
2016-03-01 17:34:50 +00:00
|
|
|
rocksdb_prefix + current_version_number;
|
2015-07-22 04:33:20 +00:00
|
|
|
const std::string DB::Properties::kEstimateLiveDataSize =
|
|
|
|
rocksdb_prefix + estimate_live_data_size;
|
2016-12-28 19:53:29 +00:00
|
|
|
const std::string DB::Properties::kMinLogNumberToKeep =
|
|
|
|
rocksdb_prefix + min_log_number_to_keep;
|
2015-08-20 18:47:19 +00:00
|
|
|
const std::string DB::Properties::kTotalSstFilesSize =
|
|
|
|
rocksdb_prefix + total_sst_files_size;
|
2016-01-25 22:04:55 +00:00
|
|
|
const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level;
|
2015-08-14 04:42:20 +00:00
|
|
|
const std::string DB::Properties::kEstimatePendingCompactionBytes =
|
|
|
|
rocksdb_prefix + estimate_pending_comp_bytes;
|
2015-08-25 19:03:54 +00:00
|
|
|
const std::string DB::Properties::kAggregatedTableProperties =
|
|
|
|
rocksdb_prefix + aggregated_table_properties;
|
|
|
|
const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
|
|
|
|
rocksdb_prefix + aggregated_table_properties_at_level;
|
2017-03-29 18:42:56 +00:00
|
|
|
const std::string DB::Properties::kActualDelayedWriteRate =
|
|
|
|
rocksdb_prefix + actual_delayed_write_rate;
|
|
|
|
const std::string DB::Properties::kIsWriteStopped =
|
|
|
|
rocksdb_prefix + is_write_stopped;
|
2015-03-27 01:09:08 +00:00
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
const std::unordered_map<std::string, DBPropertyInfo>
|
|
|
|
InternalStats::ppt_name_to_info = {
|
|
|
|
{DB::Properties::kNumFilesAtLevelPrefix,
|
|
|
|
{false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}},
|
|
|
|
{DB::Properties::kCompressionRatioAtLevelPrefix,
|
|
|
|
{false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kLevelStats,
|
|
|
|
{false, &InternalStats::HandleLevelStats, nullptr, nullptr}},
|
|
|
|
{DB::Properties::kStats,
|
|
|
|
{false, &InternalStats::HandleStats, nullptr, nullptr}},
|
|
|
|
{DB::Properties::kCFStats,
|
|
|
|
{false, &InternalStats::HandleCFStats, nullptr,
|
|
|
|
&InternalStats::HandleCFMapStats}},
|
2017-04-11 15:38:22 +00:00
|
|
|
{DB::Properties::kCFStatsNoFileHistogram,
|
|
|
|
{false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kCFFileHistogram,
|
|
|
|
{false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}},
|
2016-11-12 04:45:47 +00:00
|
|
|
{DB::Properties::kDBStats,
|
|
|
|
{false, &InternalStats::HandleDBStats, nullptr, nullptr}},
|
|
|
|
{DB::Properties::kSSTables,
|
|
|
|
{false, &InternalStats::HandleSsTables, nullptr, nullptr}},
|
|
|
|
{DB::Properties::kAggregatedTableProperties,
|
|
|
|
{false, &InternalStats::HandleAggregatedTableProperties, nullptr,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kAggregatedTablePropertiesAtLevel,
|
|
|
|
{false, &InternalStats::HandleAggregatedTablePropertiesAtLevel,
|
|
|
|
nullptr, nullptr}},
|
|
|
|
{DB::Properties::kNumImmutableMemTable,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}},
|
|
|
|
{DB::Properties::kNumImmutableMemTableFlushed,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kMemTableFlushPending,
|
|
|
|
{false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}},
|
|
|
|
{DB::Properties::kCompactionPending,
|
|
|
|
{false, nullptr, &InternalStats::HandleCompactionPending, nullptr}},
|
|
|
|
{DB::Properties::kBackgroundErrors,
|
|
|
|
{false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}},
|
|
|
|
{DB::Properties::kCurSizeActiveMemTable,
|
|
|
|
{false, nullptr, &InternalStats::HandleCurSizeActiveMemTable,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kCurSizeAllMemTables,
|
|
|
|
{false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}},
|
|
|
|
{DB::Properties::kSizeAllMemTables,
|
|
|
|
{false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}},
|
|
|
|
{DB::Properties::kNumEntriesActiveMemTable,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kNumEntriesImmMemTables,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumEntriesImmMemTables,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kNumDeletesActiveMemTable,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kNumDeletesImmMemTables,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumDeletesImmMemTables,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kEstimateNumKeys,
|
|
|
|
{false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}},
|
|
|
|
{DB::Properties::kEstimateTableReadersMem,
|
|
|
|
{true, nullptr, &InternalStats::HandleEstimateTableReadersMem,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kIsFileDeletionsEnabled,
|
|
|
|
{false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kNumSnapshots,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}},
|
|
|
|
{DB::Properties::kOldestSnapshotTime,
|
|
|
|
{false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}},
|
|
|
|
{DB::Properties::kNumLiveVersions,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}},
|
|
|
|
{DB::Properties::kCurrentSuperVersionNumber,
|
|
|
|
{false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kEstimateLiveDataSize,
|
|
|
|
{true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}},
|
2016-12-28 19:53:29 +00:00
|
|
|
{DB::Properties::kMinLogNumberToKeep,
|
|
|
|
{false, nullptr, &InternalStats::HandleMinLogNumberToKeep, nullptr}},
|
2016-11-12 04:45:47 +00:00
|
|
|
{DB::Properties::kBaseLevel,
|
|
|
|
{false, nullptr, &InternalStats::HandleBaseLevel, nullptr}},
|
|
|
|
{DB::Properties::kTotalSstFilesSize,
|
|
|
|
{false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}},
|
|
|
|
{DB::Properties::kEstimatePendingCompactionBytes,
|
|
|
|
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kNumRunningFlushes,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}},
|
|
|
|
{DB::Properties::kNumRunningCompactions,
|
|
|
|
{false, nullptr, &InternalStats::HandleNumRunningCompactions,
|
|
|
|
nullptr}},
|
2017-03-29 18:42:56 +00:00
|
|
|
{DB::Properties::kActualDelayedWriteRate,
|
|
|
|
{false, nullptr, &InternalStats::HandleActualDelayedWriteRate,
|
|
|
|
nullptr}},
|
|
|
|
{DB::Properties::kIsWriteStopped,
|
|
|
|
{false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr}},
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
|
|
|
|
std::string ppt_name = GetPropertyNameAndArg(property).first.ToString();
|
|
|
|
auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name);
|
|
|
|
if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) {
|
|
|
|
return nullptr;
|
2014-08-05 18:27:34 +00:00
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
return &ppt_info_iter->second;
|
|
|
|
}
|
2014-08-05 18:27:34 +00:00
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
|
|
|
|
const Slice& property,
|
|
|
|
std::string* value) {
|
|
|
|
assert(value != nullptr);
|
|
|
|
assert(property_info.handle_string != nullptr);
|
|
|
|
Slice arg = GetPropertyNameAndArg(property).second;
|
|
|
|
return (this->*(property_info.handle_string))(value, arg);
|
2014-03-18 19:37:42 +00:00
|
|
|
}
|
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
|
|
|
|
const Slice& property,
|
|
|
|
std::map<std::string, double>* value) {
|
|
|
|
assert(value != nullptr);
|
|
|
|
assert(property_info.handle_map != nullptr);
|
|
|
|
return (this->*(property_info.handle_map))(value);
|
|
|
|
}
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
|
|
|
|
uint64_t* value, DBImpl* db) {
|
2014-08-05 18:27:34 +00:00
|
|
|
assert(value != nullptr);
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
assert(property_info.handle_int != nullptr &&
|
|
|
|
!property_info.need_out_of_mutex);
|
|
|
|
db->mutex_.AssertHeld();
|
|
|
|
return (this->*(property_info.handle_int))(value, db, nullptr /* version */);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::GetIntPropertyOutOfMutex(
|
|
|
|
const DBPropertyInfo& property_info, Version* version, uint64_t* value) {
|
|
|
|
assert(value != nullptr);
|
|
|
|
assert(property_info.handle_int != nullptr &&
|
|
|
|
property_info.need_out_of_mutex);
|
|
|
|
return (this->*(property_info.handle_int))(value, nullptr /* db */, version);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) {
|
|
|
|
uint64_t level;
|
2015-07-22 04:33:20 +00:00
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
|
|
|
|
if (!ok || static_cast<int>(level) >= number_levels_) {
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "%d",
|
|
|
|
vstorage->NumLevelFiles(static_cast<int>(level)));
|
|
|
|
*value = buf;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-21 01:46:54 +00:00
|
|
|
bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value,
|
|
|
|
Slice suffix) {
|
|
|
|
uint64_t level;
|
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
|
|
|
bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
|
|
|
|
if (!ok || level >= static_cast<uint64_t>(number_levels_)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
*value = ToString(
|
|
|
|
vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level)));
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleLevelStats(std::string* value, Slice suffix) {
|
|
|
|
char buf[1000];
|
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Level Files Size(MB)\n"
|
|
|
|
"--------------------\n");
|
|
|
|
value->append(buf);
|
2015-07-22 04:33:20 +00:00
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
|
|
|
snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
|
|
|
|
vstorage->NumLevelFiles(level),
|
|
|
|
vstorage->NumLevelBytes(level) / kMB);
|
|
|
|
value->append(buf);
|
2014-08-05 18:27:34 +00:00
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
return true;
|
2014-08-05 18:27:34 +00:00
|
|
|
}
|
2014-07-28 22:28:53 +00:00
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleStats(std::string* value, Slice suffix) {
|
|
|
|
if (!HandleCFStats(value, suffix)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (!HandleDBStats(value, suffix)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
bool InternalStats::HandleCFMapStats(std::map<std::string, double>* cf_stats) {
|
|
|
|
DumpCFMapStats(cf_stats);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
|
|
|
|
DumpCFStats(value);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-04-11 15:38:22 +00:00
|
|
|
bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
|
|
|
|
Slice suffix) {
|
|
|
|
DumpCFStatsNoFileHistogram(value);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleCFFileHistogram(std::string* value, Slice suffix) {
|
|
|
|
DumpCFFileHistogram(value);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
|
|
|
|
DumpDBStats(value);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleSsTables(std::string* value, Slice suffix) {
|
2014-10-27 22:49:46 +00:00
|
|
|
auto* current = cfd_->current();
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
*value = current->DebugString();
|
|
|
|
return true;
|
|
|
|
}
|
2014-01-30 04:40:41 +00:00
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleAggregatedTableProperties(std::string* value,
|
|
|
|
Slice suffix) {
|
|
|
|
std::shared_ptr<const TableProperties> tp;
|
|
|
|
auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return false;
|
2014-07-28 22:28:53 +00:00
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
*value = tp->ToString();
|
|
|
|
return true;
|
2014-07-28 22:28:53 +00:00
|
|
|
}
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value,
|
|
|
|
Slice suffix) {
|
|
|
|
uint64_t level;
|
|
|
|
bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
|
|
|
|
if (!ok || static_cast<int>(level) >= number_levels_) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
std::shared_ptr<const TableProperties> tp;
|
|
|
|
auto s = cfd_->current()->GetAggregatedTableProperties(
|
|
|
|
&tp, static_cast<int>(level));
|
|
|
|
if (!s.ok()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
*value = tp->ToString();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = cfd_->imm()->NumNotFlushed();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value,
|
|
|
|
DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = cfd_->imm()->NumFlushed();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Return number of mem tables that are ready to flush (made immutable)
|
|
|
|
*value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = db->num_running_flushes();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// 1 if the system already determines at least one compaction is needed.
|
|
|
|
// 0 otherwise,
|
2014-10-31 15:48:19 +00:00
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
*value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
|
|
|
|
return true;
|
|
|
|
}
|
2014-07-28 22:28:53 +00:00
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = db->num_running_compactions_;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Accumulated number of errors in background flushes or compactions.
|
|
|
|
*value = GetBackgroundErrorCount();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Current size of the active memtable
|
|
|
|
*value = cfd_->mem()->ApproximateMemoryUsage();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Current size of the active memtable + immutable memtables
|
|
|
|
*value = cfd_->mem()->ApproximateMemoryUsage() +
|
|
|
|
cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = cfd_->mem()->ApproximateMemoryUsage() +
|
|
|
|
cfd_->imm()->ApproximateMemoryUsage();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Current number of entires in the active memtable
|
|
|
|
*value = cfd_->mem()->num_entries();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Current number of entries in the immutable memtables
|
|
|
|
*value = cfd_->imm()->current()->GetTotalNumEntries();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Current number of entires in the active memtable
|
|
|
|
*value = cfd_->mem()->num_deletes();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Current number of entries in the immutable memtables
|
|
|
|
*value = cfd_->imm()->current()->GetTotalNumDeletes();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
// Estimate number of entries in the column family:
|
|
|
|
// Use estimated entries in tables + total entries in memtables.
|
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
2017-05-23 17:32:02 +00:00
|
|
|
uint64_t estimate_keys = cfd_->mem()->num_entries() +
|
|
|
|
cfd_->imm()->current()->GetTotalNumEntries() +
|
|
|
|
vstorage->GetEstimatedActiveKeys();
|
|
|
|
uint64_t estimate_deletes =
|
|
|
|
cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes();
|
|
|
|
*value = estimate_keys > estimate_deletes * 2
|
|
|
|
? estimate_keys - (estimate_deletes * 2)
|
|
|
|
: 0;
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = db->snapshots().count();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = cfd_->GetNumLiveVersions();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-03-03 21:18:56 +00:00
|
|
|
bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
2016-03-01 17:34:50 +00:00
|
|
|
*value = cfd_->GetSuperVersionNumber();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = db->IsFileDeletionsEnabled();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
|
|
|
*value = vstorage->base_level();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = cfd_->GetTotalSstFilesSize();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value,
|
|
|
|
DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
|
|
|
*value = vstorage->estimated_compaction_needed_bytes();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = (version == nullptr) ? 0 : version->GetMemoryUsageByTableReaders();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
|
|
|
*value = vstorage->EstimateLiveDataSize();
|
|
|
|
return true;
|
2014-01-30 04:40:41 +00:00
|
|
|
}
|
|
|
|
|
2016-12-28 19:53:29 +00:00
|
|
|
bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = db->MinLogNumberToKeep();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-03-29 18:42:56 +00:00
|
|
|
bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
const WriteController& wc = db->write_controller();
|
|
|
|
if (!wc.NeedsDelay()) {
|
|
|
|
*value = 0;
|
|
|
|
} else {
|
|
|
|
*value = wc.delayed_write_rate();
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db,
|
|
|
|
Version* version) {
|
|
|
|
*value = db->write_controller().IsStopped() ? 1 : 0;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
void InternalStats::DumpDBStats(std::string* value) {
|
|
|
|
char buf[1000];
|
|
|
|
// DB-level stats, only available from default column family
|
2015-09-02 20:58:22 +00:00
|
|
|
double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
|
|
|
|
seconds_up, interval_seconds_up);
|
|
|
|
value->append(buf);
|
|
|
|
// Cumulative
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
uint64_t user_bytes_written = GetDBStats(InternalStats::BYTES_WRITTEN);
|
|
|
|
uint64_t num_keys_written = GetDBStats(InternalStats::NUMBER_KEYS_WRITTEN);
|
|
|
|
uint64_t write_other = GetDBStats(InternalStats::WRITE_DONE_BY_OTHER);
|
|
|
|
uint64_t write_self = GetDBStats(InternalStats::WRITE_DONE_BY_SELF);
|
|
|
|
uint64_t wal_bytes = GetDBStats(InternalStats::WAL_FILE_BYTES);
|
|
|
|
uint64_t wal_synced = GetDBStats(InternalStats::WAL_FILE_SYNCED);
|
|
|
|
uint64_t write_with_wal = GetDBStats(InternalStats::WRITE_WITH_WAL);
|
|
|
|
uint64_t write_stall_micros = GetDBStats(InternalStats::WRITE_STALL_MICROS);
|
2015-05-19 23:19:30 +00:00
|
|
|
|
Fix a bug in stall time counter. Improve its output format.
Summary: Fix a bug in stall time counter. Improve its output format.
Test Plan:
export ROCKSDB_TESTS=Timeout
./db_test
./db_bench --benchmarks=fillrandom --stats_interval=10000 --statistics=true --stats_per_interval=1 --num=1000000 --threads=4 --level0_stop_writes_trigger=3 --level0_slowdown_writes_trigger=2
sample output:
Uptime(secs): 35.8 total, 0.0 interval
Cumulative writes: 359590 writes, 359589 keys, 183047 batches, 2.0 writes per batch, 0.04 GB user ingest, stall seconds: 1786.008 ms
Cumulative WAL: 359591 writes, 183046 syncs, 1.96 writes per sync, 0.04 GB written
Interval writes: 253 writes, 253 keys, 128 batches, 2.0 writes per batch, 0.0 MB user ingest, stall time: 0 us
Interval WAL: 253 writes, 128 syncs, 1.96 writes per sync, 0.00 MB written
Reviewers: MarkCallaghan, igor, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D34275
2015-03-03 20:48:12 +00:00
|
|
|
const int kHumanMicrosLen = 32;
|
|
|
|
char human_micros[kHumanMicrosLen];
|
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// Data
|
2014-10-23 01:24:14 +00:00
|
|
|
// writes: total number of write requests.
|
|
|
|
// keys: total number of key updates issued by all the write requests
|
2016-08-15 17:29:29 +00:00
|
|
|
// commit groups: number of group commits issued to the DB. Each group can
|
|
|
|
// contain one or more writes.
|
2014-10-23 01:24:14 +00:00
|
|
|
// so writes/keys is the average number of put in multi-put or put
|
2016-08-15 17:29:29 +00:00
|
|
|
// writes/groups is the average group commit size.
|
2014-10-23 01:24:14 +00:00
|
|
|
//
|
|
|
|
// The format is the same for interval stats.
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
snprintf(buf, sizeof(buf),
|
2016-08-15 17:29:29 +00:00
|
|
|
"Cumulative writes: %s writes, %s keys, %s commit groups, "
|
|
|
|
"%.1f writes per commit group, ingest: %.2f GB, %.2f MB/s\n",
|
2015-03-30 18:28:25 +00:00
|
|
|
NumberToHumanString(write_other + write_self).c_str(),
|
|
|
|
NumberToHumanString(num_keys_written).c_str(),
|
|
|
|
NumberToHumanString(write_self).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
(write_other + write_self) / static_cast<double>(write_self + 1),
|
2015-03-30 18:28:25 +00:00
|
|
|
user_bytes_written / kGB, user_bytes_written / kMB / seconds_up);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
// WAL
|
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Cumulative WAL: %s writes, %s syncs, "
|
|
|
|
"%.2f writes per sync, written: %.2f GB, %.2f MB/s\n",
|
|
|
|
NumberToHumanString(write_with_wal).c_str(),
|
|
|
|
NumberToHumanString(wal_synced).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
write_with_wal / static_cast<double>(wal_synced + 1),
|
2015-03-30 18:28:25 +00:00
|
|
|
wal_bytes / kGB, wal_bytes / kMB / seconds_up);
|
|
|
|
value->append(buf);
|
|
|
|
// Stall
|
|
|
|
AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Cumulative stall: %s, %.1f percent\n",
|
|
|
|
human_micros,
|
|
|
|
// 10000 = divide by 1M to get secs, then multiply by 100 for pct
|
|
|
|
write_stall_micros / 10000.0 / std::max(seconds_up, 0.001));
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// Interval
|
|
|
|
uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other;
|
|
|
|
uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self;
|
2014-10-23 01:24:14 +00:00
|
|
|
uint64_t interval_num_keys_written =
|
|
|
|
num_keys_written - db_stats_snapshot_.num_keys_written;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
snprintf(buf, sizeof(buf),
|
2016-08-15 17:29:29 +00:00
|
|
|
"Interval writes: %s writes, %s keys, %s commit groups, "
|
|
|
|
"%.1f writes per commit group, ingest: %.2f MB, %.2f MB/s\n",
|
2015-03-30 18:28:25 +00:00
|
|
|
NumberToHumanString(
|
|
|
|
interval_write_other + interval_write_self).c_str(),
|
|
|
|
NumberToHumanString(interval_num_keys_written).c_str(),
|
|
|
|
NumberToHumanString(interval_write_self).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
static_cast<double>(interval_write_other + interval_write_self) /
|
|
|
|
(interval_write_self + 1),
|
DB Stats Dump to print total stall time
Summary:
Add printing of stall time in DB Stats:
Sample outputs:
** DB Stats **
Uptime(secs): 53.2 total, 1.7 interval
Cumulative writes: 625940 writes, 625939 keys, 625940 batches, 1.0 writes per batch, 0.49 GB user ingest, stall micros: 50691070
Cumulative WAL: 625940 writes, 625939 syncs, 1.00 writes per sync, 0.49 GB written
Interval writes: 10859 writes, 10859 keys, 10859 batches, 1.0 writes per batch, 8.7 MB user ingest, stall micros: 1692319
Interval WAL: 10859 writes, 10859 syncs, 1.00 writes per sync, 0.01 MB written
Test Plan:
make all check
verify printing using db_bench
Reviewers: igor, yhchiang, rven, MarkCallaghan
Reviewed By: MarkCallaghan
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D31239
2015-01-09 01:51:08 +00:00
|
|
|
(user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB,
|
2015-03-30 18:28:25 +00:00
|
|
|
(user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB /
|
|
|
|
std::max(interval_seconds_up, 0.001)),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t interval_write_with_wal =
|
|
|
|
write_with_wal - db_stats_snapshot_.write_with_wal;
|
|
|
|
uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced;
|
|
|
|
uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes;
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Interval WAL: %s writes, %s syncs, "
|
|
|
|
"%.2f writes per sync, written: %.2f MB, %.2f MB/s\n",
|
|
|
|
NumberToHumanString(interval_write_with_wal).c_str(),
|
|
|
|
NumberToHumanString(interval_wal_synced).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
interval_write_with_wal /
|
|
|
|
static_cast<double>(interval_wal_synced + 1),
|
2015-03-30 18:28:25 +00:00
|
|
|
interval_wal_bytes / kGB,
|
|
|
|
interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001));
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// Stall
|
|
|
|
AppendHumanMicros(
|
|
|
|
write_stall_micros - db_stats_snapshot_.write_stall_micros,
|
|
|
|
human_micros, kHumanMicrosLen, true);
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Interval stall: %s, %.1f percent\n",
|
|
|
|
human_micros,
|
|
|
|
// 10000 = divide by 1M to get secs, then multiply by 100 for pct
|
|
|
|
(write_stall_micros - db_stats_snapshot_.write_stall_micros) /
|
|
|
|
10000.0 / std::max(interval_seconds_up, 0.001));
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
db_stats_snapshot_.seconds_up = seconds_up;
|
|
|
|
db_stats_snapshot_.ingest_bytes = user_bytes_written;
|
|
|
|
db_stats_snapshot_.write_other = write_other;
|
|
|
|
db_stats_snapshot_.write_self = write_self;
|
2014-10-23 01:24:14 +00:00
|
|
|
db_stats_snapshot_.num_keys_written = num_keys_written;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
db_stats_snapshot_.wal_bytes = wal_bytes;
|
|
|
|
db_stats_snapshot_.wal_synced = wal_synced;
|
|
|
|
db_stats_snapshot_.write_with_wal = write_with_wal;
|
DB Stats Dump to print total stall time
Summary:
Add printing of stall time in DB Stats:
Sample outputs:
** DB Stats **
Uptime(secs): 53.2 total, 1.7 interval
Cumulative writes: 625940 writes, 625939 keys, 625940 batches, 1.0 writes per batch, 0.49 GB user ingest, stall micros: 50691070
Cumulative WAL: 625940 writes, 625939 syncs, 1.00 writes per sync, 0.49 GB written
Interval writes: 10859 writes, 10859 keys, 10859 batches, 1.0 writes per batch, 8.7 MB user ingest, stall micros: 1692319
Interval WAL: 10859 writes, 10859 syncs, 1.00 writes per sync, 0.01 MB written
Test Plan:
make all check
verify printing using db_bench
Reviewers: igor, yhchiang, rven, MarkCallaghan
Reviewed By: MarkCallaghan
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D31239
2015-01-09 01:51:08 +00:00
|
|
|
db_stats_snapshot_.write_stall_micros = write_stall_micros;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
}
|
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
/**
|
|
|
|
* Dump Compaction Level stats to a map of stat name to value in double.
|
|
|
|
* The level in stat name is represented with a prefix "Lx" where "x"
|
|
|
|
* is the level number. A special level "Sum" represents the sum of a stat
|
|
|
|
* for all levels.
|
|
|
|
*/
|
|
|
|
void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
|
|
|
|
CompactionStats compaction_stats_sum(0);
|
|
|
|
std::map<int, std::map<LevelStatType, double>> levels_stats;
|
|
|
|
DumpCFMapStats(&levels_stats, &compaction_stats_sum);
|
|
|
|
for (auto const& level_ent : levels_stats) {
|
|
|
|
auto level_str =
|
|
|
|
level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
|
|
|
|
for (auto const& stat_ent : level_ent.second) {
|
|
|
|
auto stat_type = stat_ent.first;
|
|
|
|
auto key_str =
|
|
|
|
level_str + "." +
|
|
|
|
InternalStats::compaction_level_stats.at(stat_type).property_name;
|
|
|
|
(*cf_stats)[key_str] = stat_ent.second;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-08 18:31:59 +00:00
|
|
|
void InternalStats::DumpCFMapStats(
|
2016-11-12 04:45:47 +00:00
|
|
|
std::map<int, std::map<LevelStatType, double>>* levels_stats,
|
|
|
|
CompactionStats* compaction_stats_sum) {
|
2014-10-31 15:48:19 +00:00
|
|
|
const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
|
|
|
|
int num_levels_to_check =
|
2015-05-22 19:30:43 +00:00
|
|
|
(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
|
2014-11-04 01:45:55 +00:00
|
|
|
? vstorage->num_levels() - 1
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
: 1;
|
2014-10-27 22:49:46 +00:00
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
// Compaction scores are sorted based on its value. Restore them to the
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// level order
|
|
|
|
std::vector<double> compaction_score(number_levels_, 0);
|
|
|
|
for (int i = 0; i < num_levels_to_check; ++i) {
|
2014-10-31 22:04:39 +00:00
|
|
|
compaction_score[vstorage->CompactionScoreLevel(i)] =
|
|
|
|
vstorage->CompactionScore(i);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
}
|
|
|
|
// Count # of files being compacted for each level
|
|
|
|
std::vector<int> files_being_compacted(number_levels_, 0);
|
2015-05-22 19:30:43 +00:00
|
|
|
for (int level = 0; level < number_levels_; ++level) {
|
2014-10-31 22:04:39 +00:00
|
|
|
for (auto* f : vstorage->LevelFiles(level)) {
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
if (f->being_compacted) {
|
|
|
|
++files_being_compacted[level];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int total_files = 0;
|
|
|
|
int total_files_being_compacted = 0;
|
|
|
|
double total_file_size = 0;
|
2017-04-03 18:10:34 +00:00
|
|
|
uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
|
|
|
|
uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
|
|
|
|
uint64_t curr_ingest = flush_ingest + add_file_ingest;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
2014-10-27 22:49:46 +00:00
|
|
|
int files = vstorage->NumLevelFiles(level);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
total_files += files;
|
|
|
|
total_files_being_compacted += files_being_compacted[level];
|
|
|
|
if (comp_stats_[level].micros > 0 || files > 0) {
|
2016-11-12 04:45:47 +00:00
|
|
|
compaction_stats_sum->Add(comp_stats_[level]);
|
2014-10-27 22:49:46 +00:00
|
|
|
total_file_size += vstorage->NumLevelBytes(level);
|
2017-04-03 18:10:34 +00:00
|
|
|
uint64_t input_bytes;
|
|
|
|
if (level == 0) {
|
|
|
|
input_bytes = curr_ingest;
|
|
|
|
} else {
|
|
|
|
input_bytes = comp_stats_[level].bytes_read_non_output_levels;
|
|
|
|
}
|
2015-06-18 06:40:34 +00:00
|
|
|
double w_amp =
|
2017-04-03 18:10:34 +00:00
|
|
|
(input_bytes == 0)
|
2016-11-12 04:45:47 +00:00
|
|
|
? 0.0
|
|
|
|
: static_cast<double>(comp_stats_[level].bytes_written) /
|
2017-04-03 18:10:34 +00:00
|
|
|
input_bytes;
|
2016-11-12 04:45:47 +00:00
|
|
|
std::map<LevelStatType, double> level_stats;
|
|
|
|
PrepareLevelStats(&level_stats, files, files_being_compacted[level],
|
|
|
|
static_cast<double>(vstorage->NumLevelBytes(level)),
|
|
|
|
compaction_score[level], w_amp, comp_stats_[level]);
|
|
|
|
(*levels_stats)[level] = level_stats;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
}
|
|
|
|
}
|
2016-11-12 04:45:47 +00:00
|
|
|
// Cumulative summary
|
|
|
|
double w_amp = compaction_stats_sum->bytes_written /
|
|
|
|
static_cast<double>(curr_ingest + 1);
|
|
|
|
// Stats summary across levels
|
|
|
|
std::map<LevelStatType, double> sum_stats;
|
|
|
|
PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted,
|
|
|
|
total_file_size, 0, w_amp, *compaction_stats_sum);
|
|
|
|
(*levels_stats)[-1] = sum_stats; // -1 is for the Sum level
|
|
|
|
}
|
|
|
|
|
|
|
|
void InternalStats::DumpCFStats(std::string* value) {
|
2017-04-11 15:38:22 +00:00
|
|
|
DumpCFStatsNoFileHistogram(value);
|
|
|
|
DumpCFFileHistogram(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
|
2016-12-09 19:39:16 +00:00
|
|
|
char buf[2000];
|
2016-11-12 04:45:47 +00:00
|
|
|
// Per-ColumnFamily stats
|
|
|
|
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// Print stats for each level
|
|
|
|
std::map<int, std::map<LevelStatType, double>> levels_stats;
|
|
|
|
CompactionStats compaction_stats_sum(0);
|
2017-02-08 18:31:59 +00:00
|
|
|
DumpCFMapStats(&levels_stats, &compaction_stats_sum);
|
|
|
|
for (int l = 0; l < number_levels_; ++l) {
|
2016-11-12 04:45:47 +00:00
|
|
|
if (levels_stats.find(l) != levels_stats.end()) {
|
|
|
|
PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
|
|
|
|
value->append(buf);
|
|
|
|
}
|
|
|
|
}
|
2017-04-06 00:18:21 +00:00
|
|
|
|
2016-11-12 04:45:47 +00:00
|
|
|
// Print sum of level stats
|
|
|
|
PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
|
|
|
|
uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
|
2016-09-21 21:24:08 +00:00
|
|
|
uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL];
|
|
|
|
uint64_t ingest_l0_files_addfile =
|
|
|
|
cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL];
|
|
|
|
uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// Cumulative summary
|
2015-11-19 02:10:20 +00:00
|
|
|
uint64_t total_stall_count =
|
|
|
|
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
|
|
|
|
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
|
|
|
|
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
|
|
|
|
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
|
Slowdown when writing to the last write buffer
Summary: Now if inserting to mem table is much faster than writing to files, there is no mechanism users can rely on to avoid stopping for reaching options.max_write_buffer_number. With the commit, if there are more than four maximum write buffers configured, we slow down to the rate of options.delayed_write_rate while we reach the last one.
Test Plan:
1. Add a new unit test.
2. Run db_bench with
./db_bench --benchmarks=fillrandom --num=10000000 --max_background_flushes=6 --batch_size=32 -max_write_buffer_number=4 --delayed_write_rate=500000 --statistics
based on hard drive and see stopping is avoided with the commit.
Reviewers: yhchiang, IslamAbdelRahman, anthony, rven, kradhakrishnan, igor
Reviewed By: igor
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52047
2015-12-17 01:08:09 +00:00
|
|
|
cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN];
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// Interval summary
|
2016-06-16 23:21:41 +00:00
|
|
|
uint64_t interval_flush_ingest =
|
|
|
|
flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
|
|
|
|
uint64_t interval_add_file_inget =
|
2016-09-21 21:24:08 +00:00
|
|
|
add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
uint64_t interval_ingest =
|
2016-06-16 23:21:41 +00:00
|
|
|
interval_flush_ingest + interval_add_file_inget + 1;
|
2016-11-12 04:45:47 +00:00
|
|
|
CompactionStats interval_stats(compaction_stats_sum);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
|
2016-11-12 04:45:47 +00:00
|
|
|
double w_amp =
|
|
|
|
interval_stats.bytes_written / static_cast<double>(interval_ingest);
|
2015-11-19 02:10:20 +00:00
|
|
|
PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
2016-04-21 23:58:38 +00:00
|
|
|
double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
|
|
|
|
double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up;
|
|
|
|
snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n",
|
|
|
|
seconds_up, interval_seconds_up);
|
|
|
|
value->append(buf);
|
2016-06-16 23:21:41 +00:00
|
|
|
snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n",
|
|
|
|
flush_ingest / kGB, interval_flush_ingest / kGB);
|
2016-09-21 21:24:08 +00:00
|
|
|
value->append(buf);
|
2016-06-16 23:21:41 +00:00
|
|
|
snprintf(buf, sizeof(buf), "AddFile(GB): cumulative %.3f, interval %.3f\n",
|
|
|
|
add_file_ingest / kGB, interval_add_file_inget / kGB);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
2016-09-21 21:24:08 +00:00
|
|
|
uint64_t interval_ingest_files_addfile =
|
|
|
|
ingest_files_addfile - cf_stats_snapshot_.ingest_files_addfile;
|
2016-09-22 03:22:09 +00:00
|
|
|
snprintf(buf, sizeof(buf), "AddFile(Total Files): cumulative %" PRIu64
|
|
|
|
", interval %" PRIu64 "\n",
|
2016-09-21 21:24:08 +00:00
|
|
|
ingest_files_addfile, interval_ingest_files_addfile);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t interval_ingest_l0_files_addfile =
|
|
|
|
ingest_l0_files_addfile - cf_stats_snapshot_.ingest_l0_files_addfile;
|
|
|
|
snprintf(buf, sizeof(buf),
|
2016-09-22 03:22:09 +00:00
|
|
|
"AddFile(L0 Files): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
|
2016-09-21 21:24:08 +00:00
|
|
|
ingest_l0_files_addfile, interval_ingest_l0_files_addfile);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t interval_ingest_keys_addfile =
|
|
|
|
ingest_keys_addfile - cf_stats_snapshot_.ingest_keys_addfile;
|
2016-09-22 03:22:09 +00:00
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"AddFile(Keys): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
|
2016-09-21 21:24:08 +00:00
|
|
|
ingest_keys_addfile, interval_ingest_keys_addfile);
|
|
|
|
value->append(buf);
|
|
|
|
|
2016-04-21 23:58:38 +00:00
|
|
|
// Compact
|
|
|
|
uint64_t compact_bytes_read = 0;
|
|
|
|
uint64_t compact_bytes_write = 0;
|
|
|
|
uint64_t compact_micros = 0;
|
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
|
|
|
compact_bytes_read += comp_stats_[level].bytes_read_output_level +
|
|
|
|
comp_stats_[level].bytes_read_non_output_levels;
|
|
|
|
compact_bytes_write += comp_stats_[level].bytes_written;
|
|
|
|
compact_micros += comp_stats_[level].micros;
|
|
|
|
}
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Cumulative compaction: %.2f GB write, %.2f MB/s write, "
|
|
|
|
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
|
|
|
|
compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
|
|
|
|
compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
|
|
|
|
compact_micros / kMicrosInSec);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// Compaction interval
|
|
|
|
uint64_t interval_compact_bytes_write =
|
|
|
|
compact_bytes_write - cf_stats_snapshot_.compact_bytes_write;
|
|
|
|
uint64_t interval_compact_bytes_read =
|
|
|
|
compact_bytes_read - cf_stats_snapshot_.compact_bytes_read;
|
|
|
|
uint64_t interval_compact_micros =
|
|
|
|
compact_micros - cf_stats_snapshot_.compact_micros;
|
|
|
|
|
|
|
|
snprintf(
|
|
|
|
buf, sizeof(buf),
|
|
|
|
"Interval compaction: %.2f GB write, %.2f MB/s write, "
|
|
|
|
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
|
|
|
|
interval_compact_bytes_write / kGB,
|
|
|
|
interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
|
|
|
|
interval_compact_bytes_read / kGB,
|
|
|
|
interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
|
|
|
|
interval_compact_micros / kMicrosInSec);
|
|
|
|
value->append(buf);
|
|
|
|
cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
|
|
|
|
cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
|
|
|
|
cf_stats_snapshot_.compact_micros = compact_micros;
|
|
|
|
|
2015-09-11 21:31:23 +00:00
|
|
|
snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64
|
|
|
|
" level0_slowdown, "
|
|
|
|
"%" PRIu64
|
|
|
|
" level0_slowdown_with_compaction, "
|
|
|
|
"%" PRIu64
|
|
|
|
" level0_numfiles, "
|
|
|
|
"%" PRIu64
|
|
|
|
" level0_numfiles_with_compaction, "
|
|
|
|
"%" PRIu64
|
2015-11-19 02:10:20 +00:00
|
|
|
" stop for pending_compaction_bytes, "
|
2015-09-11 21:31:23 +00:00
|
|
|
"%" PRIu64
|
2015-11-19 02:10:20 +00:00
|
|
|
" slowdown for pending_compaction_bytes, "
|
2015-09-11 21:31:23 +00:00
|
|
|
"%" PRIu64
|
2015-11-19 02:10:20 +00:00
|
|
|
" memtable_compaction, "
|
Slowdown when writing to the last write buffer
Summary: Now if inserting to mem table is much faster than writing to files, there is no mechanism users can rely on to avoid stopping for reaching options.max_write_buffer_number. With the commit, if there are more than four maximum write buffers configured, we slow down to the rate of options.delayed_write_rate while we reach the last one.
Test Plan:
1. Add a new unit test.
2. Run db_bench with
./db_bench --benchmarks=fillrandom --num=10000000 --max_background_flushes=6 --batch_size=32 -max_write_buffer_number=4 --delayed_write_rate=500000 --statistics
based on hard drive and see stopping is avoided with the commit.
Reviewers: yhchiang, IslamAbdelRahman, anthony, rven, kradhakrishnan, igor
Reviewed By: igor
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52047
2015-12-17 01:08:09 +00:00
|
|
|
"%" PRIu64
|
|
|
|
" memtable_slowdown, "
|
2015-11-19 02:10:20 +00:00
|
|
|
"interval %" PRIu64 " total count\n",
|
2015-09-14 18:03:37 +00:00
|
|
|
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL],
|
|
|
|
cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION],
|
|
|
|
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL],
|
|
|
|
cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION],
|
2015-09-11 21:31:23 +00:00
|
|
|
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT],
|
2015-11-19 02:10:20 +00:00
|
|
|
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT],
|
|
|
|
cf_stats_count_[MEMTABLE_COMPACTION],
|
Slowdown when writing to the last write buffer
Summary: Now if inserting to mem table is much faster than writing to files, there is no mechanism users can rely on to avoid stopping for reaching options.max_write_buffer_number. With the commit, if there are more than four maximum write buffers configured, we slow down to the rate of options.delayed_write_rate while we reach the last one.
Test Plan:
1. Add a new unit test.
2. Run db_bench with
./db_bench --benchmarks=fillrandom --num=10000000 --max_background_flushes=6 --batch_size=32 -max_write_buffer_number=4 --delayed_write_rate=500000 --statistics
based on hard drive and see stopping is avoided with the commit.
Reviewers: yhchiang, IslamAbdelRahman, anthony, rven, kradhakrishnan, igor
Reviewed By: igor
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52047
2015-12-17 01:08:09 +00:00
|
|
|
cf_stats_count_[MEMTABLE_SLOWDOWN],
|
2015-11-19 02:10:20 +00:00
|
|
|
total_stall_count - cf_stats_snapshot_.stall_count);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
2016-06-16 23:21:41 +00:00
|
|
|
cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
|
2016-09-21 21:24:08 +00:00
|
|
|
cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
|
|
|
|
cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
|
|
|
|
cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
|
|
|
|
cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
|
2016-11-12 04:45:47 +00:00
|
|
|
cf_stats_snapshot_.comp_stats = compaction_stats_sum;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
cf_stats_snapshot_.stall_count = total_stall_count;
|
|
|
|
}
|
|
|
|
|
2017-04-11 15:38:22 +00:00
|
|
|
void InternalStats::DumpCFFileHistogram(std::string* value) {
|
|
|
|
char buf[2000];
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"\n** File Read Latency Histogram By Level [%s] **\n",
|
|
|
|
cfd_->GetName().c_str());
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
|
|
|
if (!file_read_latency_[level].Empty()) {
|
|
|
|
char buf2[5000];
|
|
|
|
snprintf(buf2, sizeof(buf2),
|
|
|
|
"** Level %d read latency histogram (micros):\n%s\n", level,
|
|
|
|
file_read_latency_[level].ToString().c_str());
|
|
|
|
value->append(buf2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-11-25 20:01:27 +00:00
|
|
|
|
|
|
|
#else
|
|
|
|
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 03:14:56 +00:00
|
|
|
const DBPropertyInfo* GetPropertyInfo(const Slice& property) { return nullptr; }
|
2014-11-25 20:01:27 +00:00
|
|
|
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
2014-01-30 04:40:41 +00:00
|
|
|
} // namespace rocksdb
|