From 1415f8820df63a59d760659b763a8642b8d9781d Mon Sep 17 00:00:00 2001 From: kailiu Date: Tue, 19 Nov 2013 16:29:42 -0800 Subject: [PATCH] Improve the "table stats" Summary: The primary motivation of the changes is to make it easier to figure out the inside of the tables. * rename "table stats" to "table properties" since now we have more than "integers" to store in the property block. * Add filter block size to the basic table properties. * Whenever a table is built, we'll log the table properties (the sample output is in Test Plan). * Make an api to expose deleted keys. Test Plan: Passed all existing test. and the sample output of table stats: ================================================================== Basic Properties ------------------------------------------------------------------ # data blocks: 1 # entries: 1 raw key size: 9 raw average key size: 9 raw value size: 9 raw average value size: 0 data block size: 25 index block size: 27 filter block size: 18 (estimated) table size: 70 filter policy: rocksdb.BuiltinBloomFilter ================================================================== User collected properties: InternalKeyPropertiesCollector ------------------------------------------------------------------ kDeletedKeys: 1 ================================================================== Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D14187 --- Makefile | 12 +- db/db_impl.cc | 18 +- db/simple_table_db_test.cc | 8 +- db/table_properties_collector.cc | 164 ++++++++++++++++++ db/table_properties_collector.h | 76 ++++++++ ....cc => table_properties_collector_test.cc} | 58 ++++--- db/table_stats_collector.cc | 55 ------ db/table_stats_collector.h | 58 ------- include/rocksdb/options.h | 5 +- include/rocksdb/table.h | 4 +- include/rocksdb/table_properties.h | 90 ++++++++++ include/rocksdb/table_stats.h | 67 ------- table/block_based_table_builder.cc | 159 +++++++++++------ table/block_based_table_reader.cc | 100 ++++++----- table/block_based_table_reader.h | 17 +- table/table_test.cc | 66 +++---- util/options.cc | 4 +- 17 files changed, 589 insertions(+), 372 deletions(-) create mode 100644 db/table_properties_collector.cc create mode 100644 db/table_properties_collector.h rename db/{table_stats_collector_test.cc => table_properties_collector_test.cc} (81%) delete mode 100644 db/table_stats_collector.cc delete mode 100644 db/table_stats_collector.h create mode 100644 include/rocksdb/table_properties.h delete mode 100644 include/rocksdb/table_stats.h diff --git a/Makefile b/Makefile index 0027870cd6..06856eeb19 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind) VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full TESTS = \ - table_stats_collector_test \ + table_properties_collector_test \ arena_test \ auto_roll_logger_test \ block_test \ @@ -46,7 +46,6 @@ TESTS = \ coding_test \ corruption_test \ crc32c_test \ - db_test \ dbformat_test \ env_test \ blob_store_test \ @@ -63,12 +62,13 @@ TESTS = \ simple_table_db_test \ skiplist_test \ stringappend_test \ - table_test \ ttl_test \ version_edit_test \ version_set_test \ write_batch_test\ - deletefile_test + deletefile_test \ + table_test \ + db_test TOOLS = \ sst_dump \ @@ -201,8 +201,8 @@ signal_test: util/signal_test.o $(LIBOBJECTS) arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -table_stats_collector_test: db/table_stats_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/table_stats_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) +table_properties_collector_test: db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(CXX) db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) diff --git a/db/db_impl.cc b/db/db_impl.cc index 0037b91a0e..fa35581ae3 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -30,7 +30,7 @@ #include "db/merge_helper.h" #include "db/prefix_filter_iterator.h" #include "db/table_cache.h" -#include "db/table_stats_collector.h" +#include "db/table_properties_collector.h" #include "db/transaction_log_impl.h" #include "db/version_set.h" #include "db/write_batch_internal.h" @@ -182,20 +182,20 @@ Options SanitizeOptions(const std::string& dbname, result.wal_dir = dbname; } - // -- Sanitize the table stats collector - // All user defined stats collectors will be wrapped by - // UserKeyTableStatsCollector since for them they only have the knowledge of - // the user keys; internal keys are invisible to them. - auto& collectors = result.table_stats_collectors; - for (size_t i = 0; i < result.table_stats_collectors.size(); ++i) { + // -- Sanitize the table properties collector + // All user defined properties collectors will be wrapped by + // UserKeyTablePropertiesCollector since for them they only have the + // knowledge of the user keys; internal keys are invisible to them. + auto& collectors = result.table_properties_collectors; + for (size_t i = 0; i < result.table_properties_collectors.size(); ++i) { assert(collectors[i]); collectors[i] = - std::make_shared(collectors[i]); + std::make_shared(collectors[i]); } // Add collector to collect internal key statistics collectors.push_back( - std::make_shared() + std::make_shared() ); if (!result.flush_block_policy_factory) { diff --git a/db/simple_table_db_test.cc b/db/simple_table_db_test.cc index f7056464c9..555d318932 100644 --- a/db/simple_table_db_test.cc +++ b/db/simple_table_db_test.cc @@ -95,7 +95,7 @@ public: void SetupForCompaction() override; - TableStats& GetTableStats() override; + TableProperties& GetTableProperties() override; ~SimpleTableReader(); @@ -171,7 +171,7 @@ struct SimpleTableReader::Rep { unique_ptr file; uint64_t index_start_offset; int num_entries; - TableStats table_stats; + TableProperties table_properties; const static int user_key_size = 16; const static int offset_length = 8; @@ -214,8 +214,8 @@ Status SimpleTableReader::Open(const Options& options, void SimpleTableReader::SetupForCompaction() { } -TableStats& SimpleTableReader::GetTableStats() { - return rep_->table_stats; +TableProperties& SimpleTableReader::GetTableProperties() { + return rep_->table_properties; } bool SimpleTableReader::PrefixMayMatch(const Slice& internal_prefix) { diff --git a/db/table_properties_collector.cc b/db/table_properties_collector.cc new file mode 100644 index 0000000000..d0cd1520d9 --- /dev/null +++ b/db/table_properties_collector.cc @@ -0,0 +1,164 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "db/table_properties_collector.h" + +#include "db/dbformat.h" +#include "util/coding.h" + +namespace rocksdb { + +namespace { + void AppendProperty( + std::string& props, + const std::string& key, + const std::string& value, + const std::string& prop_delim, + const std::string& kv_delim) { + props.append(key); + props.append(kv_delim); + props.append(value); + props.append(prop_delim); + } + + template + void AppendProperty( + std::string& props, + const std::string& key, + const TValue& value, + const std::string& prop_delim, + const std::string& kv_delim) { + AppendProperty( + props, key, std::to_string(value), prop_delim, kv_delim + ); + } +} + +std::string TableProperties::ToString( + const std::string& prop_delim, + const std::string& kv_delim) const { + std::string result; + result.reserve(1024); + + // Basic Info + AppendProperty( + result, "# data blocks", num_data_blocks, prop_delim, kv_delim + ); + AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); + + AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); + AppendProperty( + result, + "raw average key size", + num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, + prop_delim, + kv_delim + ); + AppendProperty( + result, "raw value size", raw_value_size, prop_delim, kv_delim + ); + AppendProperty( + result, + "raw average value size", + num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, + prop_delim, + kv_delim + ); + + AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); + AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); + AppendProperty( + result, "filter block size", filter_size, prop_delim, kv_delim + ); + AppendProperty( + result, + "(estimated) table size=", + data_size + index_size + filter_size, + prop_delim, + kv_delim + ); + + AppendProperty( + result, + "filter policy name", + filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, + prop_delim, + kv_delim + ); + + return result; +} + +Status InternalKeyPropertiesCollector::Add( + const Slice& key, const Slice& value) { + ParsedInternalKey ikey; + if (!ParseInternalKey(key, &ikey)) { + return Status::InvalidArgument("Invalid internal key"); + } + + if (ikey.type == ValueType::kTypeDeletion) { + ++deleted_keys_; + } + + return Status::OK(); +} + +Status InternalKeyPropertiesCollector::Finish( + TableProperties::UserCollectedProperties* properties) { + assert(properties); + assert(properties->find( + InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end()); + std::string val; + + PutVarint64(&val, deleted_keys_); + properties->insert({ InternalKeyTablePropertiesNames::kDeletedKeys, val }); + + return Status::OK(); +} + +TableProperties::UserCollectedProperties +InternalKeyPropertiesCollector::GetReadableProperties() const { + return { + { "kDeletedKeys", std::to_string(deleted_keys_) } + }; +} + + +Status UserKeyTablePropertiesCollector::Add( + const Slice& key, const Slice& value) { + ParsedInternalKey ikey; + if (!ParseInternalKey(key, &ikey)) { + return Status::InvalidArgument("Invalid internal key"); + } + + return collector_->Add(ikey.user_key, value); +} + +Status UserKeyTablePropertiesCollector::Finish( + TableProperties::UserCollectedProperties* properties) { + return collector_->Finish(properties); +} + +TableProperties::UserCollectedProperties +UserKeyTablePropertiesCollector::GetReadableProperties() const { + return collector_->GetReadableProperties(); +} + + +const std::string InternalKeyTablePropertiesNames::kDeletedKeys + = "rocksdb.deleted.keys"; + +uint64_t GetDeletedKeys( + const TableProperties::UserCollectedProperties& props) { + auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys); + if (pos == props.end()) { + return 0; + } + Slice raw = pos->second; + uint64_t val = 0; + return GetVarint64(&raw, &val) ? val : 0; +} + +} // namespace rocksdb diff --git a/db/table_properties_collector.h b/db/table_properties_collector.h new file mode 100644 index 0000000000..533130db74 --- /dev/null +++ b/db/table_properties_collector.h @@ -0,0 +1,76 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file defines a collection of statistics collectors. +#pragma once + +#include "rocksdb/table_properties.h" + +#include +#include +#include + +namespace rocksdb { + +struct InternalKeyTablePropertiesNames { + static const std::string kDeletedKeys; +}; + +// Collecting the statistics for internal keys. Visible only by internal +// rocksdb modules. +class InternalKeyPropertiesCollector : public TablePropertiesCollector { + public: + virtual Status Add(const Slice& key, const Slice& value) override; + + virtual Status Finish( + TableProperties::UserCollectedProperties* properties) override; + + virtual const char* Name() const override { + return "InternalKeyPropertiesCollector"; + } + + TableProperties::UserCollectedProperties + GetReadableProperties() const override; + + private: + uint64_t deleted_keys_ = 0; +}; + +// When rocksdb creates a new table, it will encode all "user keys" into +// "internal keys", which contains meta information of a given entry. +// +// This class extracts user key from the encoded internal key when Add() is +// invoked. +class UserKeyTablePropertiesCollector : public TablePropertiesCollector { + public: + explicit UserKeyTablePropertiesCollector( + TablePropertiesCollector* collector) : + UserKeyTablePropertiesCollector( + std::shared_ptr(collector) + ) { + } + + explicit UserKeyTablePropertiesCollector( + std::shared_ptr collector) : + collector_(collector) { + } + + virtual ~UserKeyTablePropertiesCollector() { } + + virtual Status Add(const Slice& key, const Slice& value) override; + + virtual Status Finish( + TableProperties::UserCollectedProperties* properties) override; + + virtual const char* Name() const override { return collector_->Name(); } + + TableProperties::UserCollectedProperties + GetReadableProperties() const override; + + protected: + std::shared_ptr collector_; +}; + +} // namespace rocksdb diff --git a/db/table_stats_collector_test.cc b/db/table_properties_collector_test.cc similarity index 81% rename from db/table_stats_collector_test.cc rename to db/table_properties_collector_test.cc index 0f92de3ef7..145165da19 100644 --- a/db/table_stats_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -9,8 +9,8 @@ #include "db/dbformat.h" #include "db/db_impl.h" -#include "db/table_stats_collector.h" -#include "rocksdb/table_stats.h" +#include "db/table_properties_collector.h" +#include "rocksdb/table_properties.h" #include "rocksdb/table.h" #include "util/coding.h" #include "util/testharness.h" @@ -18,7 +18,7 @@ namespace rocksdb { -class TableStatsTest { +class TablePropertiesTest { private: unique_ptr table_reader_; }; @@ -111,15 +111,15 @@ void OpenTable( } // Collects keys that starts with "A" in a table. -class RegularKeysStartWithA: public TableStatsCollector { +class RegularKeysStartWithA: public TablePropertiesCollector { public: const char* Name() const { return "RegularKeysStartWithA"; } - Status Finish(TableStats::UserCollectedStats* stats) { + Status Finish(TableProperties::UserCollectedProperties* properties) { std::string encoded; PutVarint32(&encoded, count_); - *stats = TableStats::UserCollectedStats { - { "TableStatsTest", "Rocksdb" }, + *properties = TableProperties::UserCollectedProperties { + { "TablePropertiesTest", "Rocksdb" }, { "Count", encoded } }; return Status::OK(); @@ -133,11 +133,17 @@ class RegularKeysStartWithA: public TableStatsCollector { return Status::OK(); } + virtual TableProperties::UserCollectedProperties + GetReadableProperties() const { + return {}; + } + + private: uint32_t count_ = 0; }; -TEST(TableStatsTest, CustomizedTableStatsCollector) { +TEST(TablePropertiesTest, CustomizedTablePropertiesCollector) { Options options; // make sure the entries will be inserted with order. @@ -151,17 +157,17 @@ TEST(TableStatsTest, CustomizedTableStatsCollector) { {"Find", "val6"}, }; - // Test stats collectors with internal keys or regular keys + // Test properties collectors with internal keys or regular keys for (bool encode_as_internal : { true, false }) { // -- Step 1: build table auto collector = new RegularKeysStartWithA(); if (encode_as_internal) { - options.table_stats_collectors = { - std::make_shared(collector) + options.table_properties_collectors = { + std::make_shared(collector) }; } else { - options.table_stats_collectors.resize(1); - options.table_stats_collectors[0].reset(collector); + options.table_properties_collectors.resize(1); + options.table_properties_collectors[0].reset(collector); } std::unique_ptr builder; std::unique_ptr writable; @@ -180,18 +186,19 @@ TEST(TableStatsTest, CustomizedTableStatsCollector) { // -- Step 2: Open table std::unique_ptr table_reader; OpenTable(options, writable->contents(), &table_reader); - const auto& stats = table_reader->GetTableStats().user_collected_stats; + const auto& properties = + table_reader->GetTableProperties().user_collected_properties; - ASSERT_EQ("Rocksdb", stats.at("TableStatsTest")); + ASSERT_EQ("Rocksdb", properties.at("TablePropertiesTest")); uint32_t starts_with_A = 0; - Slice key(stats.at("Count")); + Slice key(properties.at("Count")); ASSERT_TRUE(GetVarint32(&key, &starts_with_A)); ASSERT_EQ(3u, starts_with_A); } } -TEST(TableStatsTest, InternalKeyStatsCollector) { +TEST(TablePropertiesTest, InternalKeyPropertiesCollector) { InternalKey keys[] = { InternalKey("A", 0, ValueType::kTypeValue), InternalKey("B", 0, ValueType::kTypeValue), @@ -207,10 +214,10 @@ TEST(TableStatsTest, InternalKeyStatsCollector) { std::unique_ptr writable; Options options; if (sanitized) { - options.table_stats_collectors = { + options.table_properties_collectors = { std::make_shared() }; - // with sanitization, even regular stats collector will be able to + // with sanitization, even regular properties collector will be able to // handle internal keys. auto comparator = options.comparator; // HACK: Set options.info_log to avoid writing log in @@ -224,8 +231,8 @@ TEST(TableStatsTest, InternalKeyStatsCollector) { ); options.comparator = comparator; } else { - options.table_stats_collectors = { - std::make_shared() + options.table_properties_collectors = { + std::make_shared() }; } @@ -238,16 +245,15 @@ TEST(TableStatsTest, InternalKeyStatsCollector) { std::unique_ptr table_reader; OpenTable(options, writable->contents(), &table_reader); - const auto& stats = table_reader->GetTableStats().user_collected_stats; + const auto& properties = + table_reader->GetTableProperties().user_collected_properties; - uint64_t deleted = 0; - Slice key(stats.at(InternalKeyTableStatsNames::kDeletedKeys)); - ASSERT_TRUE(GetVarint64(&key, &deleted)); + uint64_t deleted = GetDeletedKeys(properties); ASSERT_EQ(4u, deleted); if (sanitized) { uint32_t starts_with_A = 0; - Slice key(stats.at("Count")); + Slice key(properties.at("Count")); ASSERT_TRUE(GetVarint32(&key, &starts_with_A)); ASSERT_EQ(1u, starts_with_A); } diff --git a/db/table_stats_collector.cc b/db/table_stats_collector.cc deleted file mode 100644 index 91e89ce4c0..0000000000 --- a/db/table_stats_collector.cc +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2013, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "db/table_stats_collector.h" - -#include "db/dbformat.h" -#include "util/coding.h" - -namespace rocksdb { - -Status InternalKeyStatsCollector::Add(const Slice& key, const Slice& value) { - ParsedInternalKey ikey; - if (!ParseInternalKey(key, &ikey)) { - return Status::InvalidArgument("Invalid internal key"); - } - - if (ikey.type == ValueType::kTypeDeletion) { - ++deleted_keys_; - } - - return Status::OK(); -} - -Status InternalKeyStatsCollector::Finish( - TableStats::UserCollectedStats* stats) { - assert(stats); - assert(stats->find(InternalKeyTableStatsNames::kDeletedKeys) == stats->end()); - std::string val; - - PutVarint64(&val, deleted_keys_); - stats->insert(std::make_pair(InternalKeyTableStatsNames::kDeletedKeys, val)); - - return Status::OK(); -} - -Status UserKeyTableStatsCollector::Add(const Slice& key, const Slice& value) { - ParsedInternalKey ikey; - if (!ParseInternalKey(key, &ikey)) { - return Status::InvalidArgument("Invalid internal key"); - } - - return collector_->Add(ikey.user_key, value); -} - -Status UserKeyTableStatsCollector::Finish( - TableStats::UserCollectedStats* stats) { - return collector_->Finish(stats); -} - -const std::string InternalKeyTableStatsNames::kDeletedKeys - = "rocksdb.deleted.keys"; - -} // namespace rocksdb diff --git a/db/table_stats_collector.h b/db/table_stats_collector.h deleted file mode 100644 index e4e3685f74..0000000000 --- a/db/table_stats_collector.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2013, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. -// -// This file defines a collection of statistics collectors. -#pragma once - -#include "rocksdb/table_stats.h" - -#include -#include -#include - -namespace rocksdb { - -struct InternalKeyTableStatsNames { - static const std::string kDeletedKeys; -}; - -// Collecting the statistics for internal keys. Visible only by internal -// rocksdb modules. -class InternalKeyStatsCollector : public TableStatsCollector { - public: - virtual Status Add(const Slice& key, const Slice& value); - virtual Status Finish(TableStats::UserCollectedStats* stats); - virtual const char* Name() const { return "InternalKeyStatsCollector"; } - - private: - uint64_t deleted_keys_ = 0; -}; - -// When rocksdb creates a new table, it will encode all "user keys" into -// "internal keys", which contains meta information of a given entry. -// -// This class extracts user key from the encoded internal key when Add() is -// invoked. -class UserKeyTableStatsCollector : public TableStatsCollector { - public: - explicit UserKeyTableStatsCollector(TableStatsCollector* collector): - UserKeyTableStatsCollector( - std::shared_ptr(collector) - ) { - } - - explicit UserKeyTableStatsCollector( - std::shared_ptr collector) : collector_(collector) { - } - virtual ~UserKeyTableStatsCollector() { } - virtual Status Add(const Slice& key, const Slice& value); - virtual Status Finish(TableStats::UserCollectedStats* stats); - virtual const char* Name() const { return collector_->Name(); } - - protected: - std::shared_ptr collector_; -}; - -} // namespace rocksdb diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index c36d1c6131..5554e716f1 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -21,7 +21,7 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/slice_transform.h" #include "rocksdb/statistics.h" -#include "rocksdb/table_stats.h" +#include "rocksdb/table_properties.h" #include "rocksdb/universal_compaction.h" namespace rocksdb { @@ -619,7 +619,8 @@ struct Options { // the tables. // Default: emtpy vector -- no user-defined statistics collection will be // performed. - std::vector> table_stats_collectors; + std::vector> + table_properties_collectors; // Allows thread-safe inplace updates. Requires Updates iff // * key exists in current memtable diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index f1b3632e84..2d2bfacc4e 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -12,7 +12,7 @@ #include #include "rocksdb/env.h" #include "rocksdb/iterator.h" -#include "rocksdb/table_stats.h" +#include "rocksdb/table_properties.h" #include "rocksdb/options.h" namespace rocksdb { @@ -98,7 +98,7 @@ class TableReader { // posix_fadvise virtual void SetupForCompaction() = 0; - virtual TableStats& GetTableStats() = 0; + virtual TableProperties& GetTableProperties() = 0; // Calls (*result_handler)(handle_context, ...) repeatedly, starting with // the entry found after a call to Seek(key), until result_handler returns diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h new file mode 100644 index 0000000000..8824ca13c8 --- /dev/null +++ b/include/rocksdb/table_properties.h @@ -0,0 +1,90 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +#pragma once + +#include +#include + +#include "rocksdb/status.h" + +namespace rocksdb { + +// TableProperties contains a bunch of read-only properties of its associated +// table. +struct TableProperties { + public: + // Other than basic table properties, each table may also have the user + // collected properties. + // The value of the user-collected properties are encoded as raw bytes -- + // users have to interprete these values by themselves. + typedef + std::unordered_map + UserCollectedProperties; + + // the total size of all data blocks. + uint64_t data_size = 0; + // the size of index block. + uint64_t index_size = 0; + // the size of filter block. + uint64_t filter_size = 0; + // total raw key size + uint64_t raw_key_size = 0; + // total raw value size + uint64_t raw_value_size = 0; + // the number of blocks in this table + uint64_t num_data_blocks = 0; + // the number of entries in this table + uint64_t num_entries = 0; + + // The name of the filter policy used in this table. + // If no filter policy is used, `filter_policy_name` will be an empty string. + std::string filter_policy_name; + + // user collected properties + UserCollectedProperties user_collected_properties; + + // convert this object to a human readable form + // @prop_delim: delimiter for each property. + std::string ToString( + const std::string& prop_delim = "; ", + const std::string& kv_delim = "=") const; +}; + +// `TablePropertiesCollector` provides the mechanism for users to collect +// their own interested properties. This class is essentially a collection +// of callback functions that will be invoked during table building. +class TablePropertiesCollector { + public: + virtual ~TablePropertiesCollector() { } + + // Add() will be called when a new key/value pair is inserted into the table. + // @params key the original key that is inserted into the table. + // @params value the original value that is inserted into the table. + virtual Status Add(const Slice& key, const Slice& value) = 0; + + // Finish() will be called when a table has already been built and is ready + // for writing the properties block. + // @params properties User will add their collected statistics to + // `properties`. + virtual Status Finish( + TableProperties::UserCollectedProperties* properties) = 0; + + // The name of the properties collector can be used for debugging purpose. + virtual const char* Name() const = 0; + + // Return the human-readable properties, where the key is property name and + // the value is the human-readable form of value. + virtual TableProperties::UserCollectedProperties + GetReadableProperties() const = 0; +}; + +// Extra properties +// Below is a list of non-basic properties that are collected by database +// itself. Especially some properties regarding to the internal keys (which +// is unknown to `table`). +extern uint64_t GetDeletedKeys( + const TableProperties::UserCollectedProperties& props); + +} // namespace rocksdb diff --git a/include/rocksdb/table_stats.h b/include/rocksdb/table_stats.h deleted file mode 100644 index 09acf89bd9..0000000000 --- a/include/rocksdb/table_stats.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2013, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. -#pragma once - -#include -#include - -#include "rocksdb/status.h" - -namespace rocksdb { - -// TableStats contains a bunch of read-only stats of its associated -// table. -struct TableStats { - public: - // Other than basic table stats, each table may also have the user - // collected stats. - // The value of the user-collected stats are encoded as raw bytes -- - // users have to interprete these values by themselves. - typedef - std::unordered_map - UserCollectedStats; - - // the total size of all data blocks. - uint64_t data_size = 0; - // the total size of all index blocks. - uint64_t index_size = 0; - // total raw key size - uint64_t raw_key_size = 0; - // total raw value size - uint64_t raw_value_size = 0; - // the number of blocks in this table - uint64_t num_data_blocks = 0; - // the number of entries in this table - uint64_t num_entries = 0; - - // The name of the filter policy used in this table. - // If no filter policy is used, `filter_policy_name` will be an empty string. - std::string filter_policy_name; - - // user collected stats - UserCollectedStats user_collected_stats; -}; - -// `TableStatsCollector` provides the mechanism for users to collect their own -// interested stats. This class is essentially a collection of callback -// functions that will be invoked during table building. -class TableStatsCollector { - public: - virtual ~TableStatsCollector() { } - // Add() will be called when a new key/value pair is inserted into the table. - // @params key the original key that is inserted into the table. - // @params value the original value that is inserted into the table. - virtual Status Add(const Slice& key, const Slice& value) = 0; - - // Finish() will be called when a table has already been built and is ready - // for writing the stats block. - // @params stats User will add their collected statistics to `stats`. - virtual Status Finish(TableStats::UserCollectedStats* stats) = 0; - - // The name of the stats collector can be used for debugging purpose. - virtual const char* Name() const = 0; -}; - -} // namespace rocksdb diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc index 237d889356..de0098a24c 100644 --- a/table/block_based_table_builder.cc +++ b/table/block_based_table_builder.cc @@ -10,7 +10,9 @@ #include "table/block_based_table_builder.h" #include +#include #include +#include #include "rocksdb/flush_block_policy.h" #include "rocksdb/cache.h" @@ -45,13 +47,13 @@ struct BytewiseLessThan { // before writng to store. typedef std::map BytewiseSortedMap; -void AddStats(BytewiseSortedMap& stats, std::string name, uint64_t val) { - assert(stats.find(name) == stats.end()); +void AddProperties(BytewiseSortedMap& props, std::string name, uint64_t val) { + assert(props.find(name) == props.end()); std::string dst; PutVarint64(&dst, val); - stats.insert( + props.insert( std::make_pair(name, dst) ); } @@ -63,12 +65,12 @@ static bool GoodCompressionRatio(size_t compressed_size, size_t raw_size) { // Were we encounter any error occurs during user-defined statistics collection, // we'll write the warning message to info log. -void LogStatsCollectionError( +void LogPropertiesCollectionError( Logger* info_log, const std::string& method, const std::string& name) { assert(method == "Add" || method == "Finish"); std::string msg = - "[Warning] encountered error when calling TableStatsCollector::" + + "[Warning] encountered error when calling TablePropertiesCollector::" + method + "() with collector name: " + name; Log(info_log, "%s", msg.c_str()); } @@ -84,12 +86,7 @@ struct BlockBasedTableBuilder::Rep { BlockBuilder index_block; std::string last_key; CompressionType compression_type; - - uint64_t num_entries = 0; - uint64_t num_data_blocks = 0; - uint64_t raw_key_size = 0; - uint64_t raw_value_size = 0; - uint64_t data_size = 0; + TableProperties props; bool closed = false; // Either Finish() or Abandon() has been called. FilterBlockBuilder* filter_block; @@ -142,7 +139,7 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) { Rep* r = rep_; assert(!r->closed); if (!ok()) return; - if (r->num_entries > 0) { + if (r->props.num_entries > 0) { assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0); } @@ -173,14 +170,14 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) { r->last_key.assign(key.data(), key.size()); r->data_block.Add(key, value); - r->num_entries++; - r->raw_key_size += key.size(); - r->raw_value_size += value.size(); + r->props.num_entries++; + r->props.raw_key_size += key.size(); + r->props.raw_value_size += value.size(); - for (auto collector : r->options.table_stats_collectors) { + for (auto collector : r->options.table_properties_collectors) { Status s = collector->Add(key, value); if (!s.ok()) { - LogStatsCollectionError( + LogPropertiesCollectionError( r->options.info_log.get(), "Add", /* method */ collector->Name() @@ -201,8 +198,8 @@ void BlockBasedTableBuilder::Flush() { if (r->filter_block != nullptr) { r->filter_block->StartBlock(r->offset); } - r->data_size = r->offset; - ++r->num_data_blocks; + r->props.data_size = r->offset; + ++r->props.num_data_blocks; } void BlockBasedTableBuilder::WriteBlock(BlockBuilder* block, @@ -358,11 +355,12 @@ Status BlockBasedTableBuilder::Finish() { // Write filter block if (ok() && r->filter_block != nullptr) { - WriteRawBlock(r->filter_block->Finish(), kNoCompression, - &filter_block_handle); + auto filter_contents = r->filter_block->Finish(); + r->props.filter_size = filter_contents.size(); + WriteRawBlock(filter_contents, kNoCompression, &filter_block_handle); } - // To make sure stats block is able to keep the accurate size of index + // To make sure properties block is able to keep the accurate size of index // block, we will finish writing all index entries here and flush them // to storage after metaindex block is written. if (ok() && !empty_data_block) { @@ -375,7 +373,7 @@ Status BlockBasedTableBuilder::Finish() { // Write meta blocks and metaindex block with the following order. // 1. [meta block: filter] - // 2. [meta block: stats] + // 2. [meta block: properties] // 3. [metaindex block] if (ok()) { // We use `BytewiseComparator` as the comparator for meta block. @@ -400,67 +398,91 @@ Status BlockBasedTableBuilder::Finish() { ); } - // Write stats block. + // Write properties block. { - BlockBuilder stats_block( + BlockBuilder properties_block( r->options.block_restart_interval, BytewiseComparator() ); - BytewiseSortedMap stats; + BytewiseSortedMap properties; - // Add basic stats - AddStats(stats, BlockBasedTableStatsNames::kRawKeySize, r->raw_key_size); - AddStats(stats, BlockBasedTableStatsNames::kRawValueSize, - r->raw_value_size); - AddStats(stats, BlockBasedTableStatsNames::kDataSize, r->data_size); - AddStats( - stats, - BlockBasedTableStatsNames::kIndexSize, - r->index_block.CurrentSizeEstimate() + kBlockTrailerSize + // Add basic properties + AddProperties( + properties, + BlockBasedTablePropertiesNames::kRawKeySize, + r->props.raw_key_size ); - AddStats(stats, BlockBasedTableStatsNames::kNumEntries, r->num_entries); - AddStats(stats, BlockBasedTableStatsNames::kNumDataBlocks, - r->num_data_blocks); + AddProperties( + properties, + BlockBasedTablePropertiesNames::kRawValueSize, + r->props.raw_value_size + ); + AddProperties( + properties, + BlockBasedTablePropertiesNames::kDataSize, + r->props.data_size + ); + r->props.index_size = + r->index_block.CurrentSizeEstimate() + kBlockTrailerSize; + AddProperties( + properties, + BlockBasedTablePropertiesNames::kIndexSize, + r->props.index_size + ); + AddProperties( + properties, + BlockBasedTablePropertiesNames::kNumEntries, + r->props.num_entries + ); + AddProperties( + properties, + BlockBasedTablePropertiesNames::kNumDataBlocks, + r->props.num_data_blocks); if (r->filter_block != nullptr) { - stats.insert(std::make_pair( - BlockBasedTableStatsNames::kFilterPolicy, + properties.insert({ + BlockBasedTablePropertiesNames::kFilterPolicy, r->options.filter_policy->Name() - )); + }); } + AddProperties( + properties, + BlockBasedTablePropertiesNames::kFilterSize, + r->props.filter_size + ); - for (auto collector : r->options.table_stats_collectors) { - TableStats::UserCollectedStats user_collected_stats; + for (auto collector : r->options.table_properties_collectors) { + TableProperties::UserCollectedProperties user_collected_properties; Status s = - collector->Finish(&user_collected_stats); + collector->Finish(&user_collected_properties); if (!s.ok()) { - LogStatsCollectionError( + LogPropertiesCollectionError( r->options.info_log.get(), "Finish", /* method */ collector->Name() ); } else { - stats.insert( - user_collected_stats.begin(), - user_collected_stats.end() + properties.insert( + user_collected_properties.begin(), + user_collected_properties.end() ); } } - for (const auto& stat : stats) { - stats_block.Add(stat.first, stat.second); + for (const auto& stat : properties) { + properties_block.Add(stat.first, stat.second); } - BlockHandle stats_block_handle; - WriteBlock(&stats_block, &stats_block_handle); + BlockHandle properties_block_handle; + WriteBlock(&properties_block, &properties_block_handle); std::string handle_encoding; - stats_block_handle.EncodeTo(&handle_encoding); + properties_block_handle.EncodeTo(&handle_encoding); meta_block_handles.insert( - std::make_pair(BlockBasedTable::kStatsBlock, handle_encoding) + { BlockBasedTable::kPropertiesBlock, handle_encoding } ); - } // end of stats block writing + } // end of properties block writing for (const auto& metablock : meta_block_handles) { meta_index_block.Add(metablock.first, metablock.second); @@ -486,6 +508,31 @@ Status BlockBasedTableBuilder::Finish() { r->offset += footer_encoding.size(); } } + + // Print out the table stats + if (ok()) { + // user collected properties + std::string user_collected; + user_collected.reserve(1024); + for (auto collector : r->options.table_properties_collectors) { + for (const auto& prop : collector->GetReadableProperties()) { + user_collected.append(prop.first); + user_collected.append("="); + user_collected.append(prop.second); + user_collected.append("; "); + } + } + + Log( + r->options.info_log, + "Table was constructed:\n" + " basic properties: %s\n" + " user collected properties: %s", + r->props.ToString().c_str(), + user_collected.c_str() + ); + } + return r->status; } @@ -496,7 +543,7 @@ void BlockBasedTableBuilder::Abandon() { } uint64_t BlockBasedTableBuilder::NumEntries() const { - return rep_->num_entries; + return rep_->props.num_entries; } uint64_t BlockBasedTableBuilder::FileSize() const { diff --git a/table/block_based_table_reader.cc b/table/block_based_table_reader.cc index c9a48f4f47..5a26901030 100644 --- a/table/block_based_table_reader.cc +++ b/table/block_based_table_reader.cc @@ -58,7 +58,7 @@ struct BlockBasedTable::Rep { unique_ptr index_block; unique_ptr filter; - TableStats table_stats; + TableProperties table_properties; }; BlockBasedTable::~BlockBasedTable() { @@ -261,18 +261,18 @@ Status BlockBasedTable::Open(const Options& options, std::unique_ptr meta_iter; s = ReadMetaBlock(rep, &meta, &meta_iter); - // Read the stats - meta_iter->Seek(kStatsBlock); - if (meta_iter->Valid() && meta_iter->key() == Slice(kStatsBlock)) { + // Read the properties + meta_iter->Seek(kPropertiesBlock); + if (meta_iter->Valid() && meta_iter->key() == Slice(kPropertiesBlock)) { s = meta_iter->status(); if (s.ok()) { - s = ReadStats(meta_iter->value(), rep, &rep->table_stats); + s = ReadProperties(meta_iter->value(), rep, &rep->table_properties); } if (!s.ok()) { auto err_msg = - "[Warning] Encountered error while reading data from stats block " + - s.ToString(); + "[Warning] Encountered error while reading data from properties " + "block " + s.ToString(); Log(rep->options.info_log, "%s", err_msg.c_str()); } } @@ -349,8 +349,8 @@ void BlockBasedTable::SetupForCompaction() { compaction_optimized_ = true; } -TableStats& BlockBasedTable::GetTableStats() { - return rep_->table_stats; +TableProperties& BlockBasedTable::GetTableProperties() { + return rep_->table_properties; } // Load the meta-block from the file. On success, return the loaded meta block @@ -372,8 +372,8 @@ Status BlockBasedTable::ReadMetaBlock( if (!s.ok()) { auto err_msg = - "[Warning] Encountered error while reading data from stats block " + - s.ToString(); + "[Warning] Encountered error while reading data from properties" + "block " + s.ToString(); Log(rep->options.info_log, "%s", err_msg.c_str()); } if (!s.ok()) { @@ -414,14 +414,14 @@ FilterBlockReader* BlockBasedTable::ReadFilter ( rep->options, block.data, block.heap_allocated); } -Status BlockBasedTable::ReadStats( - const Slice& handle_value, Rep* rep, TableStats* table_stats) { - assert(table_stats); +Status BlockBasedTable::ReadProperties( + const Slice& handle_value, Rep* rep, TableProperties* table_properties) { + assert(table_properties); Slice v = handle_value; BlockHandle handle; if (!handle.DecodeFrom(&v).ok()) { - return Status::InvalidArgument("Failed to decode stats block handle"); + return Status::InvalidArgument("Failed to decode properties block handle"); } BlockContents block_contents; @@ -438,20 +438,27 @@ Status BlockBasedTable::ReadStats( return s; } - Block stats_block(block_contents); + Block properties_block(block_contents); std::unique_ptr iter( - stats_block.NewIterator(BytewiseComparator()) + properties_block.NewIterator(BytewiseComparator()) ); - // All pre-defined stats of type uint64_t - std::unordered_map predefined_uint64_stats = { - { BlockBasedTableStatsNames::kDataSize, &table_stats->data_size }, - { BlockBasedTableStatsNames::kIndexSize, &table_stats->index_size }, - { BlockBasedTableStatsNames::kRawKeySize, &table_stats->raw_key_size }, - { BlockBasedTableStatsNames::kRawValueSize, &table_stats->raw_value_size }, - { BlockBasedTableStatsNames::kNumDataBlocks, - &table_stats->num_data_blocks }, - { BlockBasedTableStatsNames::kNumEntries, &table_stats->num_entries }, + // All pre-defined properties of type uint64_t + std::unordered_map predefined_uint64_properties = { + { BlockBasedTablePropertiesNames::kDataSize, + &table_properties->data_size }, + { BlockBasedTablePropertiesNames::kIndexSize, + &table_properties->index_size }, + { BlockBasedTablePropertiesNames::kFilterSize, + &table_properties->filter_size }, + { BlockBasedTablePropertiesNames::kRawKeySize, + &table_properties->raw_key_size }, + { BlockBasedTablePropertiesNames::kRawValueSize, + &table_properties->raw_value_size }, + { BlockBasedTablePropertiesNames::kNumDataBlocks, + &table_properties->num_data_blocks }, + { BlockBasedTablePropertiesNames::kNumEntries, + &table_properties->num_entries }, }; std::string last_key; @@ -462,7 +469,7 @@ Status BlockBasedTable::ReadStats( } auto key = iter->key().ToString(); - // stats block is strictly sorted with no duplicate key. + // properties block is strictly sorted with no duplicate key. assert( last_key.empty() || BytewiseComparator()->Compare(key, last_key) > 0 @@ -470,25 +477,25 @@ Status BlockBasedTable::ReadStats( last_key = key; auto raw_val = iter->value(); - auto pos = predefined_uint64_stats.find(key); + auto pos = predefined_uint64_properties.find(key); - if (pos != predefined_uint64_stats.end()) { - // handle predefined rocksdb stats + if (pos != predefined_uint64_properties.end()) { + // handle predefined rocksdb properties uint64_t val; if (!GetVarint64(&raw_val, &val)) { // skip malformed value auto error_msg = - "[Warning] detect malformed value in stats meta-block:" + "[Warning] detect malformed value in properties meta-block:" "\tkey: " + key + "\tval: " + raw_val.ToString(); Log(rep->options.info_log, "%s", error_msg.c_str()); continue; } *(pos->second) = val; - } else if (key == BlockBasedTableStatsNames::kFilterPolicy) { - table_stats->filter_policy_name = raw_val.ToString(); + } else if (key == BlockBasedTablePropertiesNames::kFilterPolicy) { + table_properties->filter_policy_name = raw_val.ToString(); } else { // handle user-collected - table_stats->user_collected_stats.insert( + table_properties->user_collected_properties.insert( std::make_pair(key, raw_val.ToString()) ); } @@ -1066,20 +1073,25 @@ uint64_t BlockBasedTable::ApproximateOffsetOf(const Slice& key) { return result; } -const std::string BlockBasedTable::kFilterBlockPrefix = "filter."; -const std::string BlockBasedTable::kStatsBlock = "rocksdb.stats"; - -const std::string BlockBasedTableStatsNames::kDataSize = "rocksdb.data.size"; -const std::string BlockBasedTableStatsNames::kIndexSize = "rocksdb.index.size"; -const std::string BlockBasedTableStatsNames::kRawKeySize = +const std::string BlockBasedTable::kFilterBlockPrefix = + "filter."; +const std::string BlockBasedTable::kPropertiesBlock = + "rocksdb.properties"; +const std::string BlockBasedTablePropertiesNames::kDataSize = + "rocksdb.data.size"; +const std::string BlockBasedTablePropertiesNames::kIndexSize = + "rocksdb.index.size"; +const std::string BlockBasedTablePropertiesNames::kFilterSize = + "rocksdb.filter.size"; +const std::string BlockBasedTablePropertiesNames::kRawKeySize = "rocksdb.raw.key.size"; -const std::string BlockBasedTableStatsNames::kRawValueSize = +const std::string BlockBasedTablePropertiesNames::kRawValueSize = "rocksdb.raw.value.size"; -const std::string BlockBasedTableStatsNames::kNumDataBlocks = +const std::string BlockBasedTablePropertiesNames::kNumDataBlocks = "rocksdb.num.data.blocks"; -const std::string BlockBasedTableStatsNames::kNumEntries = +const std::string BlockBasedTablePropertiesNames::kNumEntries = "rocksdb.num.entries"; -const std::string BlockBasedTableStatsNames::kFilterPolicy = +const std::string BlockBasedTablePropertiesNames::kFilterPolicy = "rocksdb.filter.policy"; } // namespace rocksdb diff --git a/table/block_based_table_reader.h b/table/block_based_table_reader.h index aba6085a59..02bbfd74cf 100644 --- a/table/block_based_table_reader.h +++ b/table/block_based_table_reader.h @@ -14,7 +14,7 @@ #include "rocksdb/env.h" #include "rocksdb/iterator.h" #include "rocksdb/statistics.h" -#include "rocksdb/table_stats.h" +#include "rocksdb/table_properties.h" #include "rocksdb/table.h" #include "util/coding.h" @@ -38,7 +38,7 @@ using std::unique_ptr; class BlockBasedTable : public TableReader { public: static const std::string kFilterBlockPrefix; - static const std::string kStatsBlock; + static const std::string kPropertiesBlock; // Attempt to open the table that is stored in bytes [0..file_size) // of "file", and read the metadata entries necessary to allow @@ -88,7 +88,7 @@ class BlockBasedTable : public TableReader { // posix_fadvise void SetupForCompaction() override; - TableStats& GetTableStats() override; + TableProperties& GetTableProperties() override; ~BlockBasedTable(); @@ -142,7 +142,7 @@ class BlockBasedTable : public TableReader { void ReadMeta(const Footer& footer); void ReadFilter(const Slice& filter_handle_value); - static Status ReadStats(const Slice& handle_value, Rep* rep); + static Status ReadProperties(const Slice& handle_value, Rep* rep); // Read the meta block from sst. static Status ReadMetaBlock( @@ -156,9 +156,9 @@ class BlockBasedTable : public TableReader { Rep* rep, size_t* filter_size = nullptr); - // Read the table stats from stats block. - static Status ReadStats( - const Slice& handle_value, Rep* rep, TableStats* stats); + // Read the table properties from properties block. + static Status ReadProperties( + const Slice& handle_value, Rep* rep, TableProperties* properties); static void SetupCacheKeyPrefix(Rep* rep); @@ -181,9 +181,10 @@ class BlockBasedTable : public TableReader { void operator=(const TableReader&) = delete; }; -struct BlockBasedTableStatsNames { +struct BlockBasedTablePropertiesNames { static const std::string kDataSize; static const std::string kIndexSize; + static const std::string kFilterSize; static const std::string kRawKeySize; static const std::string kRawValueSize; static const std::string kNumDataBlocks; diff --git a/table/table_test.cc b/table/table_test.cc index 064948692d..c1f11cd181 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -799,7 +799,7 @@ class TableTest { }; // This test include all the basic checks except those for index size and block // size, which will be conducted in separated unit tests. -TEST(TableTest, BasicTableStats) { +TEST(TableTest, BasicTableProperties) { BlockBasedTableConstructor c(BytewiseComparator()); c.Add("a1", "val1"); @@ -820,16 +820,16 @@ TEST(TableTest, BasicTableStats) { c.Finish(options, &keys, &kvmap); - auto& stats = c.table_reader()->GetTableStats(); - ASSERT_EQ(kvmap.size(), stats.num_entries); + auto& props = c.table_reader()->GetTableProperties(); + ASSERT_EQ(kvmap.size(), props.num_entries); auto raw_key_size = kvmap.size() * 2ul; auto raw_value_size = kvmap.size() * 4ul; - ASSERT_EQ(raw_key_size, stats.raw_key_size); - ASSERT_EQ(raw_value_size, stats.raw_value_size); - ASSERT_EQ(1ul, stats.num_data_blocks); - ASSERT_EQ("", stats.filter_policy_name); // no filter policy is used + ASSERT_EQ(raw_key_size, props.raw_key_size); + ASSERT_EQ(raw_value_size, props.raw_value_size); + ASSERT_EQ(1ul, props.num_data_blocks); + ASSERT_EQ("", props.filter_policy_name); // no filter policy is used // Verify data size. BlockBuilder block_builder(options); @@ -839,11 +839,11 @@ TEST(TableTest, BasicTableStats) { Slice content = block_builder.Finish(); ASSERT_EQ( content.size() + kBlockTrailerSize, - stats.data_size + props.data_size ); } -TEST(TableTest, FilterPolicyNameStats) { +TEST(TableTest, FilterPolicyNameProperties) { BlockBasedTableConstructor c(BytewiseComparator()); c.Add("a1", "val1"); std::vector keys; @@ -855,8 +855,8 @@ TEST(TableTest, FilterPolicyNameStats) { options.filter_policy = filter_policy.get(); c.Finish(options, &keys, &kvmap); - auto& stats = c.table_reader()->GetTableStats(); - ASSERT_EQ("rocksdb.BuiltinBloomFilter", stats.filter_policy_name); + auto& props = c.table_reader()->GetTableProperties(); + ASSERT_EQ("rocksdb.BuiltinBloomFilter", props.filter_policy_name); } static std::string RandomString(Random* rnd, int len) { @@ -897,7 +897,7 @@ TEST(TableTest, IndexSizeStat) { c.Finish(options, &ks, &kvmap); auto index_size = - c.table_reader()->GetTableStats().index_size; + c.table_reader()->GetTableProperties().index_size; ASSERT_GT(index_size, last_index_size); last_index_size = index_size; } @@ -927,13 +927,13 @@ TEST(TableTest, NumBlockStat) { c.Finish(options, &ks, &kvmap); ASSERT_EQ( kvmap.size(), - c.table_reader()->GetTableStats().num_data_blocks + c.table_reader()->GetTableProperties().num_data_blocks ); } -class BlockCacheStats { +class BlockCacheProperties { public: - explicit BlockCacheStats(std::shared_ptr statistics) { + explicit BlockCacheProperties(std::shared_ptr statistics) { block_cache_miss = statistics.get()->getTickerCount(BLOCK_CACHE_MISS); block_cache_hit = @@ -948,7 +948,7 @@ class BlockCacheStats { statistics.get()->getTickerCount(BLOCK_CACHE_DATA_HIT); } - // Check if the fetched stats matches the expected ones. + // Check if the fetched props matches the expected ones. void AssertEqual( long index_block_cache_miss, long index_block_cache_hit, @@ -996,9 +996,9 @@ TEST(TableTest, BlockCacheTest) { // At first, no block will be accessed. { - BlockCacheStats stats(options.statistics); + BlockCacheProperties props(options.statistics); // index will be added to block cache. - stats.AssertEqual( + props.AssertEqual( 1, // index block miss 0, 0, @@ -1009,11 +1009,11 @@ TEST(TableTest, BlockCacheTest) { // Only index block will be accessed { iter.reset(c.NewIterator()); - BlockCacheStats stats(options.statistics); + BlockCacheProperties props(options.statistics); // NOTE: to help better highlight the "detla" of each ticker, I use // + to indicate the increment of changed // value; other numbers remain the same. - stats.AssertEqual( + props.AssertEqual( 1, 0 + 1, // index block hit 0, @@ -1024,8 +1024,8 @@ TEST(TableTest, BlockCacheTest) { // Only data block will be accessed { iter->SeekToFirst(); - BlockCacheStats stats(options.statistics); - stats.AssertEqual( + BlockCacheProperties props(options.statistics); + props.AssertEqual( 1, 1, 0 + 1, // data block miss @@ -1037,8 +1037,8 @@ TEST(TableTest, BlockCacheTest) { { iter.reset(c.NewIterator()); iter->SeekToFirst(); - BlockCacheStats stats(options.statistics); - stats.AssertEqual( + BlockCacheProperties props(options.statistics); + props.AssertEqual( 1, 1 + 1, // index block hit 1, @@ -1050,16 +1050,16 @@ TEST(TableTest, BlockCacheTest) { // -- PART 2: Open without block cache options.block_cache.reset(); - options.statistics = CreateDBStatistics(); // reset the stats + options.statistics = CreateDBStatistics(); // reset the props c.Reopen(options); { iter.reset(c.NewIterator()); iter->SeekToFirst(); ASSERT_EQ("key", iter->key().ToString()); - BlockCacheStats stats(options.statistics); + BlockCacheProperties props(options.statistics); // Nothing is affected at all - stats.AssertEqual(0, 0, 0, 0); + props.AssertEqual(0, 0, 0, 0); } // -- PART 3: Open with very small block cache @@ -1068,8 +1068,8 @@ TEST(TableTest, BlockCacheTest) { options.block_cache = NewLRUCache(1); c.Reopen(options); { - BlockCacheStats stats(options.statistics); - stats.AssertEqual( + BlockCacheProperties props(options.statistics); + props.AssertEqual( 1, // index block miss 0, 0, @@ -1083,8 +1083,8 @@ TEST(TableTest, BlockCacheTest) { // It first cache index block then data block. But since the cache size // is only 1, index block will be purged after data block is inserted. iter.reset(c.NewIterator()); - BlockCacheStats stats(options.statistics); - stats.AssertEqual( + BlockCacheProperties props(options.statistics); + props.AssertEqual( 1 + 1, // index block miss 0, 0, // data block miss @@ -1096,8 +1096,8 @@ TEST(TableTest, BlockCacheTest) { // SeekToFirst() accesses data block. With similar reason, we expect data // block's cache miss. iter->SeekToFirst(); - BlockCacheStats stats(options.statistics); - stats.AssertEqual( + BlockCacheProperties props(options.statistics); + props.AssertEqual( 2, 0, 0 + 1, // data block miss diff --git a/util/options.cc b/util/options.cc index 6e5971107a..4fbba08a78 100644 --- a/util/options.cc +++ b/util/options.cc @@ -280,11 +280,11 @@ Options::Dump(Logger* log) const "max_size_amplification_percent: %u", compaction_options_universal.max_size_amplification_percent); std::string collector_names; - for (auto collector : table_stats_collectors) { + for (auto collector : table_properties_collectors) { collector_names.append(collector->Name()); collector_names.append("; "); } - Log(log, " Options.table_stats_collectors: %s", + Log(log, " Options.table_properties_collectors: %s", collector_names.c_str()); Log(log, " Options.inplace_update_support: %d", inplace_update_support);