mirror of
https://github.com/facebook/rocksdb.git
synced 2024-12-02 10:15:54 +00:00
d665afdbf3
Summary: **Context/Summary:** As revealed by heap profiling, allocation of `FileMetaData` for [newly created file added to a Version](https://github.com/facebook/rocksdb/pull/9924/files#diff-a6aa385940793f95a2c5b39cc670bd440c4547fa54fd44622f756382d5e47e43R774) can consume significant heap memory. This PR is to account that toward our global memory limit based on block cache capacity. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9924 Test Plan: - Previous `make check` verified there are only 2 places where the memory of the allocated `FileMetaData` can be released - New unit test `TEST_P(ChargeFileMetadataTestWithParam, Basic)` - db bench (CPU cost of `charge_file_metadata` in write and compact) - **write micros/op: -0.24%** : `TEST_TMPDIR=/dev/shm/testdb ./db_bench -benchmarks=fillseq -db=$TEST_TMPDIR -charge_file_metadata=1 (remove this option for pre-PR) -disable_auto_compactions=1 -write_buffer_size=100000 -num=4000000 | egrep 'fillseq'` - **compact micros/op -0.87%** : `TEST_TMPDIR=/dev/shm/testdb ./db_bench -benchmarks=fillseq -db=$TEST_TMPDIR -charge_file_metadata=1 -disable_auto_compactions=1 -write_buffer_size=100000 -num=4000000 -numdistinct=1000 && ./db_bench -benchmarks=compact -db=$TEST_TMPDIR -use_existing_db=1 -charge_file_metadata=1 -disable_auto_compactions=1 | egrep 'compact'` table 1 - write #-run | (pre-PR) avg micros/op | std micros/op | (post-PR) micros/op | std micros/op | change (%) -- | -- | -- | -- | -- | -- 10 | 3.9711 | 0.264408 | 3.9914 | 0.254563 | 0.5111933721 20 | 3.83905 | 0.0664488 | 3.8251 | 0.0695456 | -0.3633711465 40 | 3.86625 | 0.136669 | 3.8867 | 0.143765 | 0.5289363078 80 | 3.87828 | 0.119007 | 3.86791 | 0.115674 | **-0.2673865734** 160 | 3.87677 | 0.162231 | 3.86739 | 0.16663 | **-0.2419539978** table 2 - compact #-run | (pre-PR) avg micros/op | std micros/op | (post-PR) micros/op | std micros/op | change (%) -- | -- | -- | -- | -- | -- 10 | 2,399,650.00 | 96,375.80 | 2,359,537.00 | 53,243.60 | -1.67 20 | 2,410,480.00 | 89,988.00 | 2,433,580.00 | 91,121.20 | 0.96 40 | 2.41E+06 | 121811 | 2.39E+06 | 131525 | **-0.96** 80 | 2.40E+06 | 134503 | 2.39E+06 | 108799 | **-0.78** - stress test: `python3 tools/db_crashtest.py blackbox --charge_file_metadata=1 --cache_size=1` killed as normal Reviewed By: ajkr Differential Revision: D36055583 Pulled By: hx235 fbshipit-source-id: b60eab94707103cb1322cf815f05810ef0232625
131 lines
3.6 KiB
C++
131 lines
3.6 KiB
C++
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#include "cache/cache_entry_roles.h"
|
|
|
|
#include <mutex>
|
|
|
|
#include "port/lang.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
|
|
"DataBlock",
|
|
"FilterBlock",
|
|
"FilterMetaBlock",
|
|
"DeprecatedFilterBlock",
|
|
"IndexBlock",
|
|
"OtherBlock",
|
|
"WriteBuffer",
|
|
"CompressionDictionaryBuildingBuffer",
|
|
"FilterConstruction",
|
|
"BlockBasedTableReader",
|
|
"FileMetadata",
|
|
"Misc",
|
|
}};
|
|
|
|
std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
|
|
"data-block",
|
|
"filter-block",
|
|
"filter-meta-block",
|
|
"deprecated-filter-block",
|
|
"index-block",
|
|
"other-block",
|
|
"write-buffer",
|
|
"compression-dictionary-building-buffer",
|
|
"filter-construction",
|
|
"block-based-table-reader",
|
|
"file-metadata",
|
|
"misc",
|
|
}};
|
|
|
|
const std::string& GetCacheEntryRoleName(CacheEntryRole role) {
|
|
return kCacheEntryRoleToHyphenString[static_cast<size_t>(role)];
|
|
}
|
|
|
|
const std::string& BlockCacheEntryStatsMapKeys::CacheId() {
|
|
static const std::string kCacheId = "id";
|
|
return kCacheId;
|
|
}
|
|
|
|
const std::string& BlockCacheEntryStatsMapKeys::CacheCapacityBytes() {
|
|
static const std::string kCacheCapacityBytes = "capacity";
|
|
return kCacheCapacityBytes;
|
|
}
|
|
|
|
const std::string&
|
|
BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds() {
|
|
static const std::string kLastCollectionDurationSeconds =
|
|
"secs_for_last_collection";
|
|
return kLastCollectionDurationSeconds;
|
|
}
|
|
|
|
const std::string& BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds() {
|
|
static const std::string kLastCollectionAgeSeconds =
|
|
"secs_since_last_collection";
|
|
return kLastCollectionAgeSeconds;
|
|
}
|
|
|
|
namespace {
|
|
|
|
std::string GetPrefixedCacheEntryRoleName(const std::string& prefix,
|
|
CacheEntryRole role) {
|
|
const std::string& role_name = GetCacheEntryRoleName(role);
|
|
std::string prefixed_role_name;
|
|
prefixed_role_name.reserve(prefix.size() + role_name.size());
|
|
prefixed_role_name.append(prefix);
|
|
prefixed_role_name.append(role_name);
|
|
return prefixed_role_name;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
std::string BlockCacheEntryStatsMapKeys::EntryCount(CacheEntryRole role) {
|
|
const static std::string kPrefix = "count.";
|
|
return GetPrefixedCacheEntryRoleName(kPrefix, role);
|
|
}
|
|
|
|
std::string BlockCacheEntryStatsMapKeys::UsedBytes(CacheEntryRole role) {
|
|
const static std::string kPrefix = "bytes.";
|
|
return GetPrefixedCacheEntryRoleName(kPrefix, role);
|
|
}
|
|
|
|
std::string BlockCacheEntryStatsMapKeys::UsedPercent(CacheEntryRole role) {
|
|
const static std::string kPrefix = "percent.";
|
|
return GetPrefixedCacheEntryRoleName(kPrefix, role);
|
|
}
|
|
|
|
namespace {
|
|
|
|
struct Registry {
|
|
std::mutex mutex;
|
|
UnorderedMap<Cache::DeleterFn, CacheEntryRole> role_map;
|
|
void Register(Cache::DeleterFn fn, CacheEntryRole role) {
|
|
std::lock_guard<std::mutex> lock(mutex);
|
|
role_map[fn] = role;
|
|
}
|
|
UnorderedMap<Cache::DeleterFn, CacheEntryRole> Copy() {
|
|
std::lock_guard<std::mutex> lock(mutex);
|
|
return role_map;
|
|
}
|
|
};
|
|
|
|
Registry& GetRegistry() {
|
|
STATIC_AVOID_DESTRUCTION(Registry, registry);
|
|
return registry;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role) {
|
|
GetRegistry().Register(fn, role);
|
|
}
|
|
|
|
UnorderedMap<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap() {
|
|
return GetRegistry().Copy();
|
|
}
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|