dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-26 07:30:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

The sharding of the block cache is limited to 2*20 pieces.

Browse source 
Summary:
The numbers of shards that the block cache is divided into is
configurable. However, if the user specifies that he/she wants
the block cache to be divided into more than 2**20 pieces, then
the system will rey to allocate a huge array of that size) that
could fail.

It is better to limit the sharding of the block cache to an
upper bound. The default sharding is 16 shards (i.e. 2**4)
and the maximum is now 2 million shards (i.e. 2**20).

Also, fixed a bug with the LRUCache where the numShardBits
should be a private member of the LRUCache object rather than
a static variable.

Test Plan:
run db_bench with --cache_numshardbits=64.

Task ID: #

Blame Rev:

Reviewers: heyongqiang

Reviewed By: heyongqiang

Differential Revision: https://reviews.facebook.net/D5013
This commit is contained in:
Dhruba Borthakur 2012-08-29 09:47:53 -07:00
parent a4f9b8b49e
commit e5fe80e4e3
2 changed files with 18 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
db/db_bench.cc
View file

@ -1048,7 +1048,12 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--cache_size=%ld%c", &n, &junk) == 1) {
FLAGS_cache_size = n;
} else if (sscanf(argv[i], "--cache_numshardbits=%d%c", &n, &junk) == 1) {
FLAGS_cache_numshardbits = n;
if (n < 20) {
FLAGS_cache_numshardbits = n;
} else {
fprintf(stderr, "The cache cannot be sharded into 2**%d pieces\n", n);
exit(1);
}
} else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
FLAGS_bloom_bits = n;
} else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {

20
util/cache.cc
View file

@ -275,21 +275,22 @@ class ShardedLRUCache : public Cache {
LRUCache* shard_;
port::Mutex id_mutex_;
uint64_t last_id_;
int numShardBits;
static inline uint32_t HashSlice(const Slice& s) {
return Hash(s.data(), s.size(), 0);
}
static uint32_t Shard(uint32_t hash) {
return hash >> (32 - kNumShardBits);
uint32_t Shard(uint32_t hash) {
return hash >> (32 - numShardBits);
}
void init(size_t capacity, int numShardBits) {
kNumShardBits = numShardBits;
kNumShards = 1 << kNumShardBits;
shard_ = new LRUCache[kNumShards];
const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards;
for (int s = 0; s < kNumShards; s++) {
void init(size_t capacity, int numbits) {
numShardBits = numbits;
int numShards = 1 << numShardBits;
shard_ = new LRUCache[numShards];
const size_t per_shard = (capacity + (numShards - 1)) / numShards;
for (int s = 0; s < numShards; s++) {
shard_[s].SetCapacity(per_shard);
}
}
@ -337,6 +338,9 @@ Cache* NewLRUCache(size_t capacity) {
}
Cache* NewLRUCache(size_t capacity, int numShardBits) {
if (numShardBits >= 20) {
return NULL; // the cache cannot be sharded into too many fine pieces
}
return new ShardedLRUCache(capacity, numShardBits);
}