2014-01-30 04:40:41 +00:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#include "db/internal_stats.h"
|
2014-09-05 06:14:37 +00:00
|
|
|
|
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-07-14 13:56:52 +00:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 06:14:37 +00:00
|
|
|
#endif
|
|
|
|
|
2014-07-14 13:56:52 +00:00
|
|
|
#include <inttypes.h>
|
2015-08-25 19:03:54 +00:00
|
|
|
#include <string>
|
2015-03-30 18:28:25 +00:00
|
|
|
#include <algorithm>
|
2014-01-30 04:40:41 +00:00
|
|
|
#include <vector>
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
#include "db/column_family.h"
|
2014-11-25 20:01:27 +00:00
|
|
|
|
2014-08-26 23:26:29 +00:00
|
|
|
#include "db/db_impl.h"
|
2014-11-25 04:44:49 +00:00
|
|
|
#include "util/string_util.h"
|
2014-01-30 04:40:41 +00:00
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
2014-11-25 20:01:27 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2014-07-11 22:03:30 +00:00
|
|
|
namespace {
|
|
|
|
const double kMB = 1048576.0;
|
|
|
|
const double kGB = kMB * 1024;
|
2015-09-02 20:58:22 +00:00
|
|
|
const double kMicrosInSec = 1000000.0;
|
2014-07-11 22:03:30 +00:00
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
|
2014-07-11 22:03:30 +00:00
|
|
|
snprintf(
|
|
|
|
buf, len,
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
"\n** Compaction Stats [%s] **\n"
|
2015-03-16 18:50:23 +00:00
|
|
|
"Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) "
|
2014-12-04 02:28:39 +00:00
|
|
|
"Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
|
2015-11-19 02:10:20 +00:00
|
|
|
"Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop\n"
|
2014-07-11 22:03:30 +00:00
|
|
|
"--------------------------------------------------------------------"
|
2015-03-14 22:12:41 +00:00
|
|
|
"-----------------------------------------------------------"
|
2015-03-14 22:01:43 +00:00
|
|
|
"--------------------------------------\n",
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
cf_name.c_str());
|
2014-07-11 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void PrintLevelStats(char* buf, size_t len, const std::string& name,
|
2015-12-12 00:34:57 +00:00
|
|
|
int num_files, int being_compacted, double total_file_size,
|
|
|
|
double score, double w_amp,
|
|
|
|
const InternalStats::CompactionStats& stats) {
|
2015-06-18 06:40:34 +00:00
|
|
|
uint64_t bytes_read =
|
|
|
|
stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
|
|
|
|
int64_t bytes_new =
|
|
|
|
stats.bytes_written - stats.bytes_read_output_level;
|
2015-09-02 20:58:22 +00:00
|
|
|
double elapsed = (stats.micros + 1) / kMicrosInSec;
|
2015-03-14 22:12:41 +00:00
|
|
|
std::string num_input_records = NumberToHumanString(stats.num_input_records);
|
|
|
|
std::string num_dropped_records =
|
|
|
|
NumberToHumanString(stats.num_dropped_records);
|
2014-07-11 22:03:30 +00:00
|
|
|
|
|
|
|
snprintf(buf, len,
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 18:42:56 +00:00
|
|
|
"%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */
|
2014-11-01 01:36:07 +00:00
|
|
|
"%8.1f " /* Read(GB) */
|
|
|
|
"%7.1f " /* Rn(GB) */
|
|
|
|
"%8.1f " /* Rnp1(GB) */
|
|
|
|
"%9.1f " /* Write(GB) */
|
|
|
|
"%8.1f " /* Wnew(GB) */
|
2014-12-04 02:28:39 +00:00
|
|
|
"%9.1f " /* Moved(GB) */
|
2014-11-01 01:36:07 +00:00
|
|
|
"%5.1f " /* W-Amp */
|
|
|
|
"%8.1f " /* Rd(MB/s) */
|
|
|
|
"%8.1f " /* Wr(MB/s) */
|
2015-03-14 22:12:41 +00:00
|
|
|
"%9.0f " /* Comp(sec) */
|
2014-11-01 01:36:07 +00:00
|
|
|
"%9d " /* Comp(cnt) */
|
|
|
|
"%8.3f " /* Avg(sec) */
|
2015-12-12 00:34:57 +00:00
|
|
|
"%7s " /* KeyIn */
|
|
|
|
"%6s\n", /* KeyDrop */
|
2015-09-02 20:58:22 +00:00
|
|
|
name.c_str(),
|
|
|
|
num_files, being_compacted, total_file_size / kMB, score,
|
|
|
|
bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
|
2015-06-18 06:40:34 +00:00
|
|
|
stats.bytes_read_output_level / kGB, stats.bytes_written / kGB,
|
2015-09-02 20:58:22 +00:00
|
|
|
bytes_new / kGB, stats.bytes_moved / kGB, w_amp,
|
|
|
|
bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed,
|
|
|
|
stats.micros / kMicrosInSec, stats.count,
|
|
|
|
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count,
|
2015-11-19 02:10:20 +00:00
|
|
|
num_input_records.c_str(), num_dropped_records.c_str());
|
2014-07-11 22:03:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string rocksdb_prefix = "rocksdb.";
|
|
|
|
|
|
|
|
static const std::string num_files_at_level_prefix = "num-files-at-level";
|
2015-04-01 02:03:18 +00:00
|
|
|
static const std::string allstats = "stats";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string sstables = "sstables";
|
|
|
|
static const std::string cfstats = "cfstats";
|
|
|
|
static const std::string dbstats = "dbstats";
|
|
|
|
static const std::string levelstats = "levelstats";
|
|
|
|
static const std::string num_immutable_mem_table = "num-immutable-mem-table";
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
static const std::string num_immutable_mem_table_flushed =
|
|
|
|
"num-immutable-mem-table-flushed";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string mem_table_flush_pending = "mem-table-flush-pending";
|
|
|
|
static const std::string compaction_pending = "compaction-pending";
|
|
|
|
static const std::string background_errors = "background-errors";
|
|
|
|
static const std::string cur_size_active_mem_table =
|
|
|
|
"cur-size-active-mem-table";
|
2015-08-19 20:32:09 +00:00
|
|
|
static const std::string cur_size_unflushed_mem_tables =
|
2015-08-24 23:43:18 +00:00
|
|
|
"cur-size-all-mem-tables";
|
2015-08-19 20:32:09 +00:00
|
|
|
static const std::string cur_size_all_mem_tables = "size-all-mem-tables";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string num_entries_active_mem_table =
|
|
|
|
"num-entries-active-mem-table";
|
|
|
|
static const std::string num_entries_imm_mem_tables =
|
|
|
|
"num-entries-imm-mem-tables";
|
|
|
|
static const std::string num_deletes_active_mem_table =
|
|
|
|
"num-deletes-active-mem-table";
|
|
|
|
static const std::string num_deletes_imm_mem_tables =
|
|
|
|
"num-deletes-imm-mem-tables";
|
|
|
|
static const std::string estimate_num_keys = "estimate-num-keys";
|
|
|
|
static const std::string estimate_table_readers_mem =
|
|
|
|
"estimate-table-readers-mem";
|
|
|
|
static const std::string is_file_deletions_enabled =
|
|
|
|
"is-file-deletions-enabled";
|
|
|
|
static const std::string num_snapshots = "num-snapshots";
|
|
|
|
static const std::string oldest_snapshot_time = "oldest-snapshot-time";
|
|
|
|
static const std::string num_live_versions = "num-live-versions";
|
2015-07-22 04:33:20 +00:00
|
|
|
static const std::string estimate_live_data_size = "estimate-live-data-size";
|
2015-03-27 01:09:08 +00:00
|
|
|
static const std::string base_level = "base-level";
|
2015-08-20 18:47:19 +00:00
|
|
|
static const std::string total_sst_files_size = "total-sst-files-size";
|
2015-08-14 04:42:20 +00:00
|
|
|
static const std::string estimate_pending_comp_bytes =
|
|
|
|
"estimate-pending-compaction-bytes";
|
2015-08-25 19:03:54 +00:00
|
|
|
static const std::string aggregated_table_properties =
|
|
|
|
"aggregated-table-properties";
|
|
|
|
static const std::string aggregated_table_properties_at_level =
|
|
|
|
aggregated_table_properties + "-at-level";
|
2015-10-17 07:16:36 +00:00
|
|
|
static const std::string num_running_compactions = "num-running-compactions";
|
|
|
|
static const std::string num_running_flushes = "num-running-flushes";
|
2015-03-27 01:09:08 +00:00
|
|
|
|
|
|
|
const std::string DB::Properties::kNumFilesAtLevelPrefix =
|
|
|
|
rocksdb_prefix + num_files_at_level_prefix;
|
2015-04-01 02:03:18 +00:00
|
|
|
const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
|
|
|
|
const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
|
|
|
|
const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
|
|
|
|
const std::string DB::Properties::kNumImmutableMemTable =
|
|
|
|
rocksdb_prefix + num_immutable_mem_table;
|
|
|
|
const std::string DB::Properties::kMemTableFlushPending =
|
|
|
|
rocksdb_prefix + mem_table_flush_pending;
|
|
|
|
const std::string DB::Properties::kCompactionPending =
|
|
|
|
rocksdb_prefix + compaction_pending;
|
2015-10-17 07:16:36 +00:00
|
|
|
const std::string DB::Properties::kNumRunningCompactions =
|
|
|
|
rocksdb_prefix + num_running_compactions;
|
|
|
|
const std::string DB::Properties::kNumRunningFlushes =
|
|
|
|
rocksdb_prefix + num_running_flushes;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kBackgroundErrors =
|
|
|
|
rocksdb_prefix + background_errors;
|
|
|
|
const std::string DB::Properties::kCurSizeActiveMemTable =
|
|
|
|
rocksdb_prefix + cur_size_active_mem_table;
|
|
|
|
const std::string DB::Properties::kCurSizeAllMemTables =
|
2015-08-19 20:32:09 +00:00
|
|
|
rocksdb_prefix + cur_size_unflushed_mem_tables;
|
|
|
|
const std::string DB::Properties::kSizeAllMemTables =
|
|
|
|
rocksdb_prefix + cur_size_all_mem_tables;
|
2015-03-27 01:09:08 +00:00
|
|
|
const std::string DB::Properties::kNumEntriesActiveMemTable =
|
|
|
|
rocksdb_prefix + num_entries_active_mem_table;
|
|
|
|
const std::string DB::Properties::kNumEntriesImmMemTables =
|
|
|
|
rocksdb_prefix + num_entries_imm_mem_tables;
|
|
|
|
const std::string DB::Properties::kNumDeletesActiveMemTable =
|
|
|
|
rocksdb_prefix + num_deletes_active_mem_table;
|
|
|
|
const std::string DB::Properties::kNumDeletesImmMemTables =
|
|
|
|
rocksdb_prefix + num_deletes_imm_mem_tables;
|
|
|
|
const std::string DB::Properties::kEstimateNumKeys =
|
|
|
|
rocksdb_prefix + estimate_num_keys;
|
|
|
|
const std::string DB::Properties::kEstimateTableReadersMem =
|
|
|
|
rocksdb_prefix + estimate_table_readers_mem;
|
|
|
|
const std::string DB::Properties::kIsFileDeletionsEnabled =
|
|
|
|
rocksdb_prefix + is_file_deletions_enabled;
|
|
|
|
const std::string DB::Properties::kNumSnapshots =
|
|
|
|
rocksdb_prefix + num_snapshots;
|
|
|
|
const std::string DB::Properties::kOldestSnapshotTime =
|
|
|
|
rocksdb_prefix + oldest_snapshot_time;
|
|
|
|
const std::string DB::Properties::kNumLiveVersions =
|
|
|
|
rocksdb_prefix + num_live_versions;
|
2015-07-22 04:33:20 +00:00
|
|
|
const std::string DB::Properties::kEstimateLiveDataSize =
|
|
|
|
rocksdb_prefix + estimate_live_data_size;
|
2015-08-20 18:47:19 +00:00
|
|
|
const std::string DB::Properties::kTotalSstFilesSize =
|
|
|
|
rocksdb_prefix + total_sst_files_size;
|
2015-08-14 04:42:20 +00:00
|
|
|
const std::string DB::Properties::kEstimatePendingCompactionBytes =
|
|
|
|
rocksdb_prefix + estimate_pending_comp_bytes;
|
2015-08-25 19:03:54 +00:00
|
|
|
const std::string DB::Properties::kAggregatedTableProperties =
|
|
|
|
rocksdb_prefix + aggregated_table_properties;
|
|
|
|
const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
|
|
|
|
rocksdb_prefix + aggregated_table_properties_at_level;
|
2015-03-27 01:09:08 +00:00
|
|
|
|
2014-08-05 18:27:34 +00:00
|
|
|
DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
|
|
|
|
bool* need_out_of_mutex) {
|
|
|
|
assert(is_int_property != nullptr);
|
|
|
|
assert(need_out_of_mutex != nullptr);
|
2014-01-30 04:40:41 +00:00
|
|
|
Slice in = property;
|
2015-03-27 01:09:08 +00:00
|
|
|
Slice prefix(rocksdb_prefix);
|
2014-08-05 18:27:34 +00:00
|
|
|
*need_out_of_mutex = false;
|
2014-08-07 00:37:35 +00:00
|
|
|
*is_int_property = false;
|
2014-08-07 00:46:26 +00:00
|
|
|
if (!in.starts_with(prefix)) {
|
|
|
|
return kUnknown;
|
|
|
|
}
|
2014-01-30 04:40:41 +00:00
|
|
|
in.remove_prefix(prefix.size());
|
|
|
|
|
2015-03-27 01:09:08 +00:00
|
|
|
if (in.starts_with(num_files_at_level_prefix)) {
|
2014-03-18 19:37:42 +00:00
|
|
|
return kNumFilesAtLevel;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == levelstats) {
|
2014-03-18 19:37:42 +00:00
|
|
|
return kLevelStats;
|
2015-04-01 02:03:18 +00:00
|
|
|
} else if (in == allstats) {
|
2014-03-18 19:37:42 +00:00
|
|
|
return kStats;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == cfstats) {
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
return kCFStats;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == dbstats) {
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
return kDBStats;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == sstables) {
|
2014-03-18 19:37:42 +00:00
|
|
|
return kSsTables;
|
2015-08-25 19:03:54 +00:00
|
|
|
} else if (in == aggregated_table_properties) {
|
|
|
|
return kAggregatedTableProperties;
|
|
|
|
} else if (in.starts_with(aggregated_table_properties_at_level)) {
|
|
|
|
return kAggregatedTablePropertiesAtLevel;
|
2014-08-05 18:27:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
*is_int_property = true;
|
2015-03-27 01:09:08 +00:00
|
|
|
if (in == num_immutable_mem_table) {
|
2014-03-18 19:37:42 +00:00
|
|
|
return kNumImmutableMemTable;
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
} else if (in == num_immutable_mem_table_flushed) {
|
|
|
|
return kNumImmutableMemTableFlushed;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == mem_table_flush_pending) {
|
2014-03-18 19:25:08 +00:00
|
|
|
return kMemtableFlushPending;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == compaction_pending) {
|
2014-03-18 19:25:08 +00:00
|
|
|
return kCompactionPending;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == background_errors) {
|
2014-03-18 19:25:08 +00:00
|
|
|
return kBackgroundErrors;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == cur_size_active_mem_table) {
|
2014-03-27 18:59:37 +00:00
|
|
|
return kCurSizeActiveMemTable;
|
2015-08-19 20:32:09 +00:00
|
|
|
} else if (in == cur_size_unflushed_mem_tables) {
|
2014-10-17 16:26:27 +00:00
|
|
|
return kCurSizeAllMemTables;
|
2015-08-19 20:32:09 +00:00
|
|
|
} else if (in == cur_size_all_mem_tables) {
|
|
|
|
return kSizeAllMemTables;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == num_entries_active_mem_table) {
|
2014-04-23 00:17:33 +00:00
|
|
|
return kNumEntriesInMutableMemtable;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == num_entries_imm_mem_tables) {
|
2014-04-23 00:17:33 +00:00
|
|
|
return kNumEntriesInImmutableMemtable;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == num_deletes_active_mem_table) {
|
2015-03-18 23:11:02 +00:00
|
|
|
return kNumDeletesInMutableMemtable;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == num_deletes_imm_mem_tables) {
|
2015-03-18 23:11:02 +00:00
|
|
|
return kNumDeletesInImmutableMemtable;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == estimate_num_keys) {
|
2014-07-28 21:50:16 +00:00
|
|
|
return kEstimatedNumKeys;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == estimate_table_readers_mem) {
|
2014-08-05 18:27:34 +00:00
|
|
|
*need_out_of_mutex = true;
|
|
|
|
return kEstimatedUsageByTableReaders;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == is_file_deletions_enabled) {
|
2014-08-26 23:26:29 +00:00
|
|
|
return kIsFileDeletionEnabled;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == num_snapshots) {
|
2014-12-06 00:12:10 +00:00
|
|
|
return kNumSnapshots;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == oldest_snapshot_time) {
|
2014-12-06 00:12:10 +00:00
|
|
|
return kOldestSnapshotTime;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == num_live_versions) {
|
2015-02-12 01:10:43 +00:00
|
|
|
return kNumLiveVersions;
|
2015-07-22 04:33:20 +00:00
|
|
|
} else if (in == estimate_live_data_size) {
|
|
|
|
*need_out_of_mutex = true;
|
|
|
|
return kEstimateLiveDataSize;
|
2015-03-27 01:09:08 +00:00
|
|
|
} else if (in == base_level) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
return kBaseLevel;
|
2015-08-20 18:47:19 +00:00
|
|
|
} else if (in == total_sst_files_size) {
|
|
|
|
return kTotalSstFilesSize;
|
2015-08-14 04:42:20 +00:00
|
|
|
} else if (in == estimate_pending_comp_bytes) {
|
|
|
|
return kEstimatePendingCompactionBytes;
|
2015-10-17 07:16:36 +00:00
|
|
|
} else if (in == num_running_flushes) {
|
|
|
|
return kNumRunningFlushes;
|
|
|
|
} else if (in == num_running_compactions) {
|
|
|
|
return kNumRunningCompactions;
|
2014-03-18 19:37:42 +00:00
|
|
|
}
|
|
|
|
return kUnknown;
|
|
|
|
}
|
|
|
|
|
2014-08-05 18:27:34 +00:00
|
|
|
bool InternalStats::GetIntPropertyOutOfMutex(DBPropertyType property_type,
|
|
|
|
Version* version,
|
|
|
|
uint64_t* value) const {
|
|
|
|
assert(value != nullptr);
|
2015-07-22 04:33:20 +00:00
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
|
|
|
|
|
|
|
switch (property_type) {
|
|
|
|
case kEstimatedUsageByTableReaders:
|
|
|
|
*value = (version == nullptr) ?
|
|
|
|
0 : version->GetMemoryUsageByTableReaders();
|
|
|
|
return true;
|
|
|
|
case kEstimateLiveDataSize:
|
|
|
|
*value = vstorage->EstimateLiveDataSize();
|
|
|
|
return true;
|
|
|
|
default:
|
|
|
|
return false;
|
2014-08-05 18:27:34 +00:00
|
|
|
}
|
|
|
|
}
|
2014-07-28 22:28:53 +00:00
|
|
|
|
2014-08-05 18:27:34 +00:00
|
|
|
bool InternalStats::GetStringProperty(DBPropertyType property_type,
|
|
|
|
const Slice& property,
|
|
|
|
std::string* value) {
|
|
|
|
assert(value != nullptr);
|
2014-10-27 22:49:46 +00:00
|
|
|
auto* current = cfd_->current();
|
2014-10-31 15:48:19 +00:00
|
|
|
const auto* vstorage = current->storage_info();
|
2014-03-18 19:37:42 +00:00
|
|
|
Slice in = property;
|
|
|
|
|
|
|
|
switch (property_type) {
|
|
|
|
case kNumFilesAtLevel: {
|
|
|
|
in.remove_prefix(strlen("rocksdb.num-files-at-level"));
|
|
|
|
uint64_t level;
|
|
|
|
bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
|
|
|
|
if (!ok || (int)level >= number_levels_) {
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "%d",
|
2014-10-27 22:49:46 +00:00
|
|
|
vstorage->NumLevelFiles(static_cast<int>(level)));
|
2014-03-18 19:37:42 +00:00
|
|
|
*value = buf;
|
|
|
|
return true;
|
|
|
|
}
|
2014-01-30 04:40:41 +00:00
|
|
|
}
|
2014-03-18 19:37:42 +00:00
|
|
|
case kLevelStats: {
|
|
|
|
char buf[1000];
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Level Files Size(MB)\n"
|
|
|
|
"--------------------\n");
|
|
|
|
value->append(buf);
|
2014-01-30 04:40:41 +00:00
|
|
|
|
2014-03-18 19:37:42 +00:00
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
|
|
|
snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
|
2014-10-27 22:49:46 +00:00
|
|
|
vstorage->NumLevelFiles(level),
|
|
|
|
vstorage->NumLevelBytes(level) / kMB);
|
2014-01-30 04:40:41 +00:00
|
|
|
value->append(buf);
|
|
|
|
}
|
2014-03-18 19:37:42 +00:00
|
|
|
return true;
|
2014-01-30 04:40:41 +00:00
|
|
|
}
|
2014-03-18 19:37:42 +00:00
|
|
|
case kStats: {
|
2015-03-27 01:09:08 +00:00
|
|
|
if (!GetStringProperty(kCFStats, DB::Properties::kCFStats, value)) {
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
return false;
|
2014-03-18 19:37:42 +00:00
|
|
|
}
|
2015-03-27 01:09:08 +00:00
|
|
|
if (!GetStringProperty(kDBStats, DB::Properties::kDBStats, value)) {
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
return false;
|
2014-07-11 22:03:30 +00:00
|
|
|
}
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case kCFStats: {
|
2014-07-28 19:27:08 +00:00
|
|
|
DumpCFStats(value);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case kDBStats: {
|
|
|
|
DumpDBStats(value);
|
2014-03-18 19:37:42 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case kSsTables:
|
|
|
|
*value = current->DebugString();
|
|
|
|
return true;
|
2015-08-25 19:03:54 +00:00
|
|
|
case kAggregatedTableProperties: {
|
|
|
|
std::shared_ptr<const TableProperties> tp;
|
|
|
|
auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
*value = tp->ToString();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case kAggregatedTablePropertiesAtLevel: {
|
|
|
|
in.remove_prefix(
|
|
|
|
DB::Properties::kAggregatedTablePropertiesAtLevel.length());
|
|
|
|
uint64_t level;
|
|
|
|
bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
|
|
|
|
if (!ok || static_cast<int>(level) >= number_levels_) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
std::shared_ptr<const TableProperties> tp;
|
|
|
|
auto s = cfd_->current()->GetAggregatedTableProperties(
|
|
|
|
&tp, static_cast<int>(level));
|
|
|
|
if (!s.ok()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
*value = tp->ToString();
|
|
|
|
return true;
|
|
|
|
}
|
2014-07-28 22:28:53 +00:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool InternalStats::GetIntProperty(DBPropertyType property_type,
|
2014-08-26 23:26:29 +00:00
|
|
|
uint64_t* value, DBImpl* db) const {
|
2015-02-12 01:10:43 +00:00
|
|
|
db->mutex_.AssertHeld();
|
2014-10-31 15:48:19 +00:00
|
|
|
const auto* vstorage = cfd_->current()->storage_info();
|
2014-07-28 22:28:53 +00:00
|
|
|
|
|
|
|
switch (property_type) {
|
2014-03-18 19:37:42 +00:00
|
|
|
case kNumImmutableMemTable:
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
*value = cfd_->imm()->NumNotFlushed();
|
|
|
|
return true;
|
|
|
|
case kNumImmutableMemTableFlushed:
|
|
|
|
*value = cfd_->imm()->NumFlushed();
|
2014-03-18 19:37:42 +00:00
|
|
|
return true;
|
2014-03-18 19:25:08 +00:00
|
|
|
case kMemtableFlushPending:
|
2014-03-18 19:37:42 +00:00
|
|
|
// Return number of mem tables that are ready to flush (made immutable)
|
2014-07-28 22:28:53 +00:00
|
|
|
*value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
|
2014-03-18 19:37:42 +00:00
|
|
|
return true;
|
2015-10-17 07:16:36 +00:00
|
|
|
case kNumRunningFlushes:
|
|
|
|
*value = db->num_running_flushes();
|
|
|
|
return true;
|
2014-03-18 19:25:08 +00:00
|
|
|
case kCompactionPending:
|
2015-08-11 18:46:15 +00:00
|
|
|
// 1 if the system already determines at least one compaction is needed.
|
2014-03-18 19:37:42 +00:00
|
|
|
// 0 otherwise,
|
2014-11-13 23:21:04 +00:00
|
|
|
*value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
|
2014-03-18 19:37:42 +00:00
|
|
|
return true;
|
2015-10-17 07:16:36 +00:00
|
|
|
case kNumRunningCompactions:
|
|
|
|
*value = db->num_running_compactions_;
|
|
|
|
return true;
|
2014-03-18 19:25:08 +00:00
|
|
|
case kBackgroundErrors:
|
|
|
|
// Accumulated number of errors in background flushes or compactions.
|
2014-07-28 22:28:53 +00:00
|
|
|
*value = GetBackgroundErrorCount();
|
2014-03-18 19:25:08 +00:00
|
|
|
return true;
|
2014-03-27 18:59:37 +00:00
|
|
|
case kCurSizeActiveMemTable:
|
|
|
|
// Current size of the active memtable
|
2014-07-28 22:28:53 +00:00
|
|
|
*value = cfd_->mem()->ApproximateMemoryUsage();
|
2014-03-27 18:59:37 +00:00
|
|
|
return true;
|
2014-10-17 16:26:27 +00:00
|
|
|
case kCurSizeAllMemTables:
|
|
|
|
// Current size of the active memtable + immutable memtables
|
2015-08-19 20:32:09 +00:00
|
|
|
*value = cfd_->mem()->ApproximateMemoryUsage() +
|
|
|
|
cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
|
|
|
|
return true;
|
|
|
|
case kSizeAllMemTables:
|
2014-10-17 16:26:27 +00:00
|
|
|
*value = cfd_->mem()->ApproximateMemoryUsage() +
|
|
|
|
cfd_->imm()->ApproximateMemoryUsage();
|
|
|
|
return true;
|
2014-04-23 00:17:33 +00:00
|
|
|
case kNumEntriesInMutableMemtable:
|
2014-10-17 16:26:27 +00:00
|
|
|
// Current number of entires in the active memtable
|
2015-03-18 23:11:02 +00:00
|
|
|
*value = cfd_->mem()->num_entries();
|
2014-04-23 00:17:33 +00:00
|
|
|
return true;
|
|
|
|
case kNumEntriesInImmutableMemtable:
|
2014-10-17 16:26:27 +00:00
|
|
|
// Current number of entries in the immutable memtables
|
2014-07-28 22:28:53 +00:00
|
|
|
*value = cfd_->imm()->current()->GetTotalNumEntries();
|
2014-04-23 00:17:33 +00:00
|
|
|
return true;
|
2015-03-18 23:11:02 +00:00
|
|
|
case kNumDeletesInMutableMemtable:
|
|
|
|
// Current number of entires in the active memtable
|
|
|
|
*value = cfd_->mem()->num_deletes();
|
|
|
|
return true;
|
|
|
|
case kNumDeletesInImmutableMemtable:
|
|
|
|
// Current number of entries in the immutable memtables
|
|
|
|
*value = cfd_->imm()->current()->GetTotalNumDeletes();
|
|
|
|
return true;
|
2014-07-28 21:50:16 +00:00
|
|
|
case kEstimatedNumKeys:
|
|
|
|
// Estimate number of entries in the column family:
|
|
|
|
// Use estimated entries in tables + total entries in memtables.
|
2015-03-18 23:11:02 +00:00
|
|
|
*value = cfd_->mem()->num_entries() +
|
|
|
|
cfd_->imm()->current()->GetTotalNumEntries() -
|
|
|
|
(cfd_->mem()->num_deletes() +
|
|
|
|
cfd_->imm()->current()->GetTotalNumDeletes()) *
|
|
|
|
2 +
|
2014-10-27 22:49:46 +00:00
|
|
|
vstorage->GetEstimatedActiveKeys();
|
2014-07-28 21:50:16 +00:00
|
|
|
return true;
|
2014-12-06 00:12:10 +00:00
|
|
|
case kNumSnapshots:
|
|
|
|
*value = db->snapshots().count();
|
|
|
|
return true;
|
|
|
|
case kOldestSnapshotTime:
|
|
|
|
*value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
|
|
|
|
return true;
|
2015-02-12 01:10:43 +00:00
|
|
|
case kNumLiveVersions:
|
|
|
|
*value = cfd_->GetNumLiveVersions();
|
|
|
|
return true;
|
2014-08-26 23:26:29 +00:00
|
|
|
case kIsFileDeletionEnabled:
|
|
|
|
*value = db->IsFileDeletionsEnabled();
|
|
|
|
return true;
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
case kBaseLevel:
|
|
|
|
*value = vstorage->base_level();
|
|
|
|
return true;
|
2015-08-20 18:47:19 +00:00
|
|
|
case kTotalSstFilesSize:
|
|
|
|
*value = cfd_->GetTotalSstFilesSize();
|
|
|
|
return true;
|
2015-08-14 04:42:20 +00:00
|
|
|
case kEstimatePendingCompactionBytes:
|
|
|
|
*value = vstorage->estimated_compaction_needed_bytes();
|
|
|
|
return true;
|
2014-03-18 19:37:42 +00:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
2014-01-30 04:40:41 +00:00
|
|
|
}
|
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
void InternalStats::DumpDBStats(std::string* value) {
|
|
|
|
char buf[1000];
|
|
|
|
// DB-level stats, only available from default column family
|
2015-09-02 20:58:22 +00:00
|
|
|
double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
|
|
|
|
seconds_up, interval_seconds_up);
|
|
|
|
value->append(buf);
|
|
|
|
// Cumulative
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
uint64_t user_bytes_written = GetDBStats(InternalStats::BYTES_WRITTEN);
|
|
|
|
uint64_t num_keys_written = GetDBStats(InternalStats::NUMBER_KEYS_WRITTEN);
|
|
|
|
uint64_t write_other = GetDBStats(InternalStats::WRITE_DONE_BY_OTHER);
|
|
|
|
uint64_t write_self = GetDBStats(InternalStats::WRITE_DONE_BY_SELF);
|
|
|
|
uint64_t wal_bytes = GetDBStats(InternalStats::WAL_FILE_BYTES);
|
|
|
|
uint64_t wal_synced = GetDBStats(InternalStats::WAL_FILE_SYNCED);
|
|
|
|
uint64_t write_with_wal = GetDBStats(InternalStats::WRITE_WITH_WAL);
|
|
|
|
uint64_t write_stall_micros = GetDBStats(InternalStats::WRITE_STALL_MICROS);
|
2015-05-19 23:19:30 +00:00
|
|
|
uint64_t compact_bytes_read = 0;
|
|
|
|
uint64_t compact_bytes_write = 0;
|
|
|
|
uint64_t compact_micros = 0;
|
|
|
|
|
Fix a bug in stall time counter. Improve its output format.
Summary: Fix a bug in stall time counter. Improve its output format.
Test Plan:
export ROCKSDB_TESTS=Timeout
./db_test
./db_bench --benchmarks=fillrandom --stats_interval=10000 --statistics=true --stats_per_interval=1 --num=1000000 --threads=4 --level0_stop_writes_trigger=3 --level0_slowdown_writes_trigger=2
sample output:
Uptime(secs): 35.8 total, 0.0 interval
Cumulative writes: 359590 writes, 359589 keys, 183047 batches, 2.0 writes per batch, 0.04 GB user ingest, stall seconds: 1786.008 ms
Cumulative WAL: 359591 writes, 183046 syncs, 1.96 writes per sync, 0.04 GB written
Interval writes: 253 writes, 253 keys, 128 batches, 2.0 writes per batch, 0.0 MB user ingest, stall time: 0 us
Interval WAL: 253 writes, 128 syncs, 1.96 writes per sync, 0.00 MB written
Reviewers: MarkCallaghan, igor, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D34275
2015-03-03 20:48:12 +00:00
|
|
|
const int kHumanMicrosLen = 32;
|
|
|
|
char human_micros[kHumanMicrosLen];
|
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// Data
|
2014-10-23 01:24:14 +00:00
|
|
|
// writes: total number of write requests.
|
|
|
|
// keys: total number of key updates issued by all the write requests
|
|
|
|
// batches: number of group commits issued to the DB. Each group can contain
|
|
|
|
// one or more writes.
|
|
|
|
// so writes/keys is the average number of put in multi-put or put
|
|
|
|
// writes/batches is the average group commit size.
|
|
|
|
//
|
|
|
|
// The format is the same for interval stats.
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Cumulative writes: %s writes, %s keys, %s batches, "
|
|
|
|
"%.1f writes per batch, ingest: %.2f GB, %.2f MB/s\n",
|
|
|
|
NumberToHumanString(write_other + write_self).c_str(),
|
|
|
|
NumberToHumanString(num_keys_written).c_str(),
|
|
|
|
NumberToHumanString(write_self).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
(write_other + write_self) / static_cast<double>(write_self + 1),
|
2015-03-30 18:28:25 +00:00
|
|
|
user_bytes_written / kGB, user_bytes_written / kMB / seconds_up);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
// WAL
|
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Cumulative WAL: %s writes, %s syncs, "
|
|
|
|
"%.2f writes per sync, written: %.2f GB, %.2f MB/s\n",
|
|
|
|
NumberToHumanString(write_with_wal).c_str(),
|
|
|
|
NumberToHumanString(wal_synced).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
write_with_wal / static_cast<double>(wal_synced + 1),
|
2015-03-30 18:28:25 +00:00
|
|
|
wal_bytes / kGB, wal_bytes / kMB / seconds_up);
|
|
|
|
value->append(buf);
|
2015-05-19 23:19:30 +00:00
|
|
|
// Compact
|
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
2015-06-18 06:40:34 +00:00
|
|
|
compact_bytes_read += comp_stats_[level].bytes_read_output_level +
|
|
|
|
comp_stats_[level].bytes_read_non_output_levels;
|
2015-05-19 23:19:30 +00:00
|
|
|
compact_bytes_write += comp_stats_[level].bytes_written;
|
|
|
|
compact_micros += comp_stats_[level].micros;
|
|
|
|
}
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Cumulative compaction: %.2f GB write, %.2f MB/s write, "
|
|
|
|
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
|
2015-09-02 20:58:22 +00:00
|
|
|
compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
|
|
|
|
compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
|
|
|
|
compact_micros / kMicrosInSec);
|
2015-05-19 23:19:30 +00:00
|
|
|
value->append(buf);
|
2015-03-30 18:28:25 +00:00
|
|
|
// Stall
|
|
|
|
AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Cumulative stall: %s, %.1f percent\n",
|
|
|
|
human_micros,
|
|
|
|
// 10000 = divide by 1M to get secs, then multiply by 100 for pct
|
|
|
|
write_stall_micros / 10000.0 / std::max(seconds_up, 0.001));
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// Interval
|
|
|
|
uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other;
|
|
|
|
uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self;
|
2014-10-23 01:24:14 +00:00
|
|
|
uint64_t interval_num_keys_written =
|
|
|
|
num_keys_written - db_stats_snapshot_.num_keys_written;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Interval writes: %s writes, %s keys, %s batches, "
|
|
|
|
"%.1f writes per batch, ingest: %.2f MB, %.2f MB/s\n",
|
|
|
|
NumberToHumanString(
|
|
|
|
interval_write_other + interval_write_self).c_str(),
|
|
|
|
NumberToHumanString(interval_num_keys_written).c_str(),
|
|
|
|
NumberToHumanString(interval_write_self).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
static_cast<double>(interval_write_other + interval_write_self) /
|
|
|
|
(interval_write_self + 1),
|
DB Stats Dump to print total stall time
Summary:
Add printing of stall time in DB Stats:
Sample outputs:
** DB Stats **
Uptime(secs): 53.2 total, 1.7 interval
Cumulative writes: 625940 writes, 625939 keys, 625940 batches, 1.0 writes per batch, 0.49 GB user ingest, stall micros: 50691070
Cumulative WAL: 625940 writes, 625939 syncs, 1.00 writes per sync, 0.49 GB written
Interval writes: 10859 writes, 10859 keys, 10859 batches, 1.0 writes per batch, 8.7 MB user ingest, stall micros: 1692319
Interval WAL: 10859 writes, 10859 syncs, 1.00 writes per sync, 0.01 MB written
Test Plan:
make all check
verify printing using db_bench
Reviewers: igor, yhchiang, rven, MarkCallaghan
Reviewed By: MarkCallaghan
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D31239
2015-01-09 01:51:08 +00:00
|
|
|
(user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB,
|
2015-03-30 18:28:25 +00:00
|
|
|
(user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB /
|
|
|
|
std::max(interval_seconds_up, 0.001)),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t interval_write_with_wal =
|
|
|
|
write_with_wal - db_stats_snapshot_.write_with_wal;
|
|
|
|
uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced;
|
|
|
|
uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes;
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Interval WAL: %s writes, %s syncs, "
|
|
|
|
"%.2f writes per sync, written: %.2f MB, %.2f MB/s\n",
|
|
|
|
NumberToHumanString(interval_write_with_wal).c_str(),
|
|
|
|
NumberToHumanString(interval_wal_synced).c_str(),
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
interval_write_with_wal /
|
|
|
|
static_cast<double>(interval_wal_synced + 1),
|
2015-03-30 18:28:25 +00:00
|
|
|
interval_wal_bytes / kGB,
|
|
|
|
interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001));
|
|
|
|
value->append(buf);
|
|
|
|
|
2015-05-19 23:19:30 +00:00
|
|
|
// Compaction
|
|
|
|
uint64_t interval_compact_bytes_write =
|
|
|
|
compact_bytes_write - db_stats_snapshot_.compact_bytes_write;
|
|
|
|
uint64_t interval_compact_bytes_read =
|
|
|
|
compact_bytes_read - db_stats_snapshot_.compact_bytes_read;
|
|
|
|
uint64_t interval_compact_micros =
|
|
|
|
compact_micros - db_stats_snapshot_.compact_micros;
|
|
|
|
|
2015-09-02 20:58:22 +00:00
|
|
|
snprintf(
|
|
|
|
buf, sizeof(buf),
|
|
|
|
"Interval compaction: %.2f GB write, %.2f MB/s write, "
|
|
|
|
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
|
|
|
|
interval_compact_bytes_write / kGB,
|
|
|
|
interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
|
|
|
|
interval_compact_bytes_read / kGB,
|
|
|
|
interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
|
|
|
|
interval_compact_micros / kMicrosInSec);
|
2015-05-19 23:19:30 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
2015-03-30 18:28:25 +00:00
|
|
|
// Stall
|
|
|
|
AppendHumanMicros(
|
|
|
|
write_stall_micros - db_stats_snapshot_.write_stall_micros,
|
|
|
|
human_micros, kHumanMicrosLen, true);
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Interval stall: %s, %.1f percent\n",
|
|
|
|
human_micros,
|
|
|
|
// 10000 = divide by 1M to get secs, then multiply by 100 for pct
|
|
|
|
(write_stall_micros - db_stats_snapshot_.write_stall_micros) /
|
|
|
|
10000.0 / std::max(interval_seconds_up, 0.001));
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
Measure file read latency histogram per level
Summary: In internal stats, remember read latency histogram, if statistics is enabled. It can be retrieved from DB::GetProperty() with "rocksdb.dbstats" property, if it is enabled.
Test Plan: Manually run db_bench and prints out "rocksdb.dbstats" by hand and make sure it prints out as expected
Reviewers: igor, IslamAbdelRahman, rven, kradhakrishnan, anthony, yhchiang
Reviewed By: yhchiang
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44193
2015-08-13 21:35:54 +00:00
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
|
|
|
if (!file_read_latency_[level].Empty()) {
|
|
|
|
char buf2[5000];
|
|
|
|
snprintf(buf2, sizeof(buf2),
|
|
|
|
"** Level %d read latency histogram (micros):\n%s\n", level,
|
|
|
|
file_read_latency_[level].ToString().c_str());
|
|
|
|
value->append(buf2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
db_stats_snapshot_.seconds_up = seconds_up;
|
|
|
|
db_stats_snapshot_.ingest_bytes = user_bytes_written;
|
|
|
|
db_stats_snapshot_.write_other = write_other;
|
|
|
|
db_stats_snapshot_.write_self = write_self;
|
2014-10-23 01:24:14 +00:00
|
|
|
db_stats_snapshot_.num_keys_written = num_keys_written;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
db_stats_snapshot_.wal_bytes = wal_bytes;
|
|
|
|
db_stats_snapshot_.wal_synced = wal_synced;
|
|
|
|
db_stats_snapshot_.write_with_wal = write_with_wal;
|
DB Stats Dump to print total stall time
Summary:
Add printing of stall time in DB Stats:
Sample outputs:
** DB Stats **
Uptime(secs): 53.2 total, 1.7 interval
Cumulative writes: 625940 writes, 625939 keys, 625940 batches, 1.0 writes per batch, 0.49 GB user ingest, stall micros: 50691070
Cumulative WAL: 625940 writes, 625939 syncs, 1.00 writes per sync, 0.49 GB written
Interval writes: 10859 writes, 10859 keys, 10859 batches, 1.0 writes per batch, 8.7 MB user ingest, stall micros: 1692319
Interval WAL: 10859 writes, 10859 syncs, 1.00 writes per sync, 0.01 MB written
Test Plan:
make all check
verify printing using db_bench
Reviewers: igor, yhchiang, rven, MarkCallaghan
Reviewed By: MarkCallaghan
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D31239
2015-01-09 01:51:08 +00:00
|
|
|
db_stats_snapshot_.write_stall_micros = write_stall_micros;
|
2015-05-19 23:19:30 +00:00
|
|
|
db_stats_snapshot_.compact_bytes_write = compact_bytes_write;
|
|
|
|
db_stats_snapshot_.compact_bytes_read = compact_bytes_read;
|
|
|
|
db_stats_snapshot_.compact_micros = compact_micros;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
}
|
|
|
|
|
2014-07-28 19:27:08 +00:00
|
|
|
void InternalStats::DumpCFStats(std::string* value) {
|
2014-10-31 15:48:19 +00:00
|
|
|
const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
|
|
|
|
int num_levels_to_check =
|
2015-05-22 19:30:43 +00:00
|
|
|
(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
|
2014-11-04 01:45:55 +00:00
|
|
|
? vstorage->num_levels() - 1
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
: 1;
|
2014-10-27 22:49:46 +00:00
|
|
|
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// Compaction scores are sorted base on its value. Restore them to the
|
|
|
|
// level order
|
|
|
|
std::vector<double> compaction_score(number_levels_, 0);
|
|
|
|
for (int i = 0; i < num_levels_to_check; ++i) {
|
2014-10-31 22:04:39 +00:00
|
|
|
compaction_score[vstorage->CompactionScoreLevel(i)] =
|
|
|
|
vstorage->CompactionScore(i);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
}
|
|
|
|
// Count # of files being compacted for each level
|
|
|
|
std::vector<int> files_being_compacted(number_levels_, 0);
|
2015-05-22 19:30:43 +00:00
|
|
|
for (int level = 0; level < number_levels_; ++level) {
|
2014-10-31 22:04:39 +00:00
|
|
|
for (auto* f : vstorage->LevelFiles(level)) {
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
if (f->being_compacted) {
|
|
|
|
++files_being_compacted[level];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
char buf[1000];
|
|
|
|
// Per-ColumnFamily stats
|
2014-07-28 19:27:08 +00:00
|
|
|
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
CompactionStats stats_sum(0);
|
|
|
|
int total_files = 0;
|
|
|
|
int total_files_being_compacted = 0;
|
|
|
|
double total_file_size = 0;
|
|
|
|
for (int level = 0; level < number_levels_; level++) {
|
2014-10-27 22:49:46 +00:00
|
|
|
int files = vstorage->NumLevelFiles(level);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
total_files += files;
|
|
|
|
total_files_being_compacted += files_being_compacted[level];
|
|
|
|
if (comp_stats_[level].micros > 0 || files > 0) {
|
|
|
|
stats_sum.Add(comp_stats_[level]);
|
2014-10-27 22:49:46 +00:00
|
|
|
total_file_size += vstorage->NumLevelBytes(level);
|
2015-06-18 06:40:34 +00:00
|
|
|
double w_amp =
|
|
|
|
(comp_stats_[level].bytes_read_non_output_levels == 0) ? 0.0
|
|
|
|
: static_cast<double>(comp_stats_[level].bytes_written) /
|
|
|
|
comp_stats_[level].bytes_read_non_output_levels;
|
2014-11-25 04:44:49 +00:00
|
|
|
PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
|
2014-10-27 22:49:46 +00:00
|
|
|
files_being_compacted[level],
|
2015-12-12 00:34:57 +00:00
|
|
|
static_cast<double>(vstorage->NumLevelBytes(level)),
|
|
|
|
compaction_score[level],
|
2015-11-19 02:10:20 +00:00
|
|
|
w_amp, comp_stats_[level]);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
uint64_t curr_ingest = cf_stats_value_[BYTES_FLUSHED];
|
|
|
|
// Cumulative summary
|
|
|
|
double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
|
2015-11-19 02:10:20 +00:00
|
|
|
uint64_t total_stall_count =
|
|
|
|
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
|
|
|
|
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
|
|
|
|
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
|
|
|
|
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
|
Slowdown when writing to the last write buffer
Summary: Now if inserting to mem table is much faster than writing to files, there is no mechanism users can rely on to avoid stopping for reaching options.max_write_buffer_number. With the commit, if there are more than four maximum write buffers configured, we slow down to the rate of options.delayed_write_rate while we reach the last one.
Test Plan:
1. Add a new unit test.
2. Run db_bench with
./db_bench --benchmarks=fillrandom --num=10000000 --max_background_flushes=6 --batch_size=32 -max_write_buffer_number=4 --delayed_write_rate=500000 --statistics
based on hard drive and see stopping is avoided with the commit.
Reviewers: yhchiang, IslamAbdelRahman, anthony, rven, kradhakrishnan, igor
Reviewed By: igor
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52047
2015-12-17 01:08:09 +00:00
|
|
|
cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN];
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
// Stats summary across levels
|
|
|
|
PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
|
2015-11-19 02:10:20 +00:00
|
|
|
total_files_being_compacted, total_file_size, 0, w_amp,
|
|
|
|
stats_sum);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
// Interval summary
|
|
|
|
uint64_t interval_ingest =
|
|
|
|
curr_ingest - cf_stats_snapshot_.ingest_bytes + 1;
|
|
|
|
CompactionStats interval_stats(stats_sum);
|
|
|
|
interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
|
|
|
|
w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
|
2015-11-19 02:10:20 +00:00
|
|
|
PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
2015-03-30 18:28:25 +00:00
|
|
|
"Flush(GB): cumulative %.3f, interval %.3f\n",
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
curr_ingest / kGB, interval_ingest / kGB);
|
|
|
|
value->append(buf);
|
|
|
|
|
2015-09-11 21:31:23 +00:00
|
|
|
snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64
|
|
|
|
" level0_slowdown, "
|
|
|
|
"%" PRIu64
|
|
|
|
" level0_slowdown_with_compaction, "
|
|
|
|
"%" PRIu64
|
|
|
|
" level0_numfiles, "
|
|
|
|
"%" PRIu64
|
|
|
|
" level0_numfiles_with_compaction, "
|
|
|
|
"%" PRIu64
|
2015-11-19 02:10:20 +00:00
|
|
|
" stop for pending_compaction_bytes, "
|
2015-09-11 21:31:23 +00:00
|
|
|
"%" PRIu64
|
2015-11-19 02:10:20 +00:00
|
|
|
" slowdown for pending_compaction_bytes, "
|
2015-09-11 21:31:23 +00:00
|
|
|
"%" PRIu64
|
2015-11-19 02:10:20 +00:00
|
|
|
" memtable_compaction, "
|
Slowdown when writing to the last write buffer
Summary: Now if inserting to mem table is much faster than writing to files, there is no mechanism users can rely on to avoid stopping for reaching options.max_write_buffer_number. With the commit, if there are more than four maximum write buffers configured, we slow down to the rate of options.delayed_write_rate while we reach the last one.
Test Plan:
1. Add a new unit test.
2. Run db_bench with
./db_bench --benchmarks=fillrandom --num=10000000 --max_background_flushes=6 --batch_size=32 -max_write_buffer_number=4 --delayed_write_rate=500000 --statistics
based on hard drive and see stopping is avoided with the commit.
Reviewers: yhchiang, IslamAbdelRahman, anthony, rven, kradhakrishnan, igor
Reviewed By: igor
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52047
2015-12-17 01:08:09 +00:00
|
|
|
"%" PRIu64
|
|
|
|
" memtable_slowdown, "
|
2015-11-19 02:10:20 +00:00
|
|
|
"interval %" PRIu64 " total count\n",
|
2015-09-14 18:03:37 +00:00
|
|
|
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL],
|
|
|
|
cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION],
|
|
|
|
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL],
|
|
|
|
cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION],
|
2015-09-11 21:31:23 +00:00
|
|
|
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT],
|
2015-11-19 02:10:20 +00:00
|
|
|
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT],
|
|
|
|
cf_stats_count_[MEMTABLE_COMPACTION],
|
Slowdown when writing to the last write buffer
Summary: Now if inserting to mem table is much faster than writing to files, there is no mechanism users can rely on to avoid stopping for reaching options.max_write_buffer_number. With the commit, if there are more than four maximum write buffers configured, we slow down to the rate of options.delayed_write_rate while we reach the last one.
Test Plan:
1. Add a new unit test.
2. Run db_bench with
./db_bench --benchmarks=fillrandom --num=10000000 --max_background_flushes=6 --batch_size=32 -max_write_buffer_number=4 --delayed_write_rate=500000 --statistics
based on hard drive and see stopping is avoided with the commit.
Reviewers: yhchiang, IslamAbdelRahman, anthony, rven, kradhakrishnan, igor
Reviewed By: igor
Subscribers: MarkCallaghan, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52047
2015-12-17 01:08:09 +00:00
|
|
|
cf_stats_count_[MEMTABLE_SLOWDOWN],
|
2015-11-19 02:10:20 +00:00
|
|
|
total_stall_count - cf_stats_snapshot_.stall_count);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 19:57:29 +00:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
cf_stats_snapshot_.ingest_bytes = curr_ingest;
|
|
|
|
cf_stats_snapshot_.comp_stats = stats_sum;
|
|
|
|
cf_stats_snapshot_.stall_count = total_stall_count;
|
|
|
|
}
|
|
|
|
|
2014-11-25 20:01:27 +00:00
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
|
|
|
|
bool* need_out_of_mutex) {
|
|
|
|
return kUnknown;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
2014-01-30 04:40:41 +00:00
|
|
|
} // namespace rocksdb
|