dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-26 16:30:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Some updates for SpatialDB

Browse source 
Summary:
1. add db statistics
2. write out batch every millionth write

Test Plan: unit tests

Reviewers: ljin, sdong, yinwang

Reviewed By: yinwang

Differential Revision: https://reviews.facebook.net/D22755
This commit is contained in:
Igor Canadi 2014-09-08 17:00:43 -07:00
parent 171d4ff4a2
commit 55114e7f40
1 changed files with 17 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
utilities/spatialdb/spatial_db.cc
View file

@ -22,6 +22,7 @@
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/memtablerep.h" #include "rocksdb/memtablerep.h"
#include "rocksdb/slice_transform.h" #include "rocksdb/slice_transform.h"
#include "rocksdb/statistics.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/utilities/stackable_db.h" #include "rocksdb/utilities/stackable_db.h"
@ -516,6 +517,7 @@ class SpatialDBImpl : public SpatialDB {
return Status::InvalidArgument("Spatial indexes can't be empty"); return Status::InvalidArgument("Spatial indexes can't be empty");
} }
const int kWriteOutEveryBytes = 1024 * 1024; // 1MB
uint64_t id = next_id_.fetch_add(1); uint64_t id = next_id_.fetch_add(1);
for (const auto& si : spatial_indexes) { for (const auto& si : spatial_indexes) {
@ -537,6 +539,13 @@ class SpatialDBImpl : public SpatialDB {
&key, GetQuadKeyFromTile(x, y, spatial_index.tile_bits)); &key, GetQuadKeyFromTile(x, y, spatial_index.tile_bits));
PutFixed64BigEndian(&key, id); PutFixed64BigEndian(&key, id);
batch.Put(itr->second.column_family, key, Slice()); batch.Put(itr->second.column_family, key, Slice());
if (batch.GetDataSize() >= kWriteOutEveryBytes) {
Status s = Write(write_options, &batch);
batch.Clear();
if (!s.ok()) {
return s;
}
}
} }
} }
} }
@ -553,6 +562,7 @@ class SpatialDBImpl : public SpatialDB {
} }
virtual Status Compact() override { virtual Status Compact() override {
// TODO(icanadi) maybe do this in parallel?
Status s, t; Status s, t;
for (auto& iter : name_to_index_) { for (auto& iter : name_to_index_) {
t = Flush(FlushOptions(), iter.second.column_family); t = Flush(FlushOptions(), iter.second.column_family);
@ -625,6 +635,7 @@ class SpatialDBImpl : public SpatialDB {
namespace { namespace {
DBOptions GetDBOptions(const SpatialDBOptions& options) { DBOptions GetDBOptions(const SpatialDBOptions& options) {
DBOptions db_options; DBOptions db_options;
db_options.max_open_files = 50000;
db_options.max_background_compactions = 3 * options.num_threads / 4; db_options.max_background_compactions = 3 * options.num_threads / 4;
db_options.max_background_flushes = db_options.max_background_flushes =
options.num_threads - db_options.max_background_compactions; options.num_threads - db_options.max_background_compactions;
@ -632,8 +643,12 @@ DBOptions GetDBOptions(const SpatialDBOptions& options) {
Env::LOW); Env::LOW);
db_options.env->SetBackgroundThreads(db_options.max_background_flushes, db_options.env->SetBackgroundThreads(db_options.max_background_flushes,
Env::HIGH); Env::HIGH);
db_options.statistics = CreateDBStatistics();
if (options.bulk_load) { if (options.bulk_load) {
db_options.stats_dump_period_sec = 600;
db_options.disableDataSync = true; db_options.disableDataSync = true;
} else {
db_options.stats_dump_period_sec = 1800; // 30min
} }
return db_options; return db_options;
} }
@ -643,6 +658,8 @@ ColumnFamilyOptions GetColumnFamilyOptions(const SpatialDBOptions& options,
ColumnFamilyOptions column_family_options; ColumnFamilyOptions column_family_options;
column_family_options.write_buffer_size = 128 * 1024 * 1024; // 128MB column_family_options.write_buffer_size = 128 * 1024 * 1024; // 128MB
column_family_options.max_write_buffer_number = 4; column_family_options.max_write_buffer_number = 4;
column_family_options.max_bytes_for_level_base = 256 * 1024 * 1024; // 256MB
column_family_options.target_file_size_base = 64 * 1024 * 1024; // 64MB
column_family_options.level0_file_num_compaction_trigger = 2; column_family_options.level0_file_num_compaction_trigger = 2;
column_family_options.level0_slowdown_writes_trigger = 16; column_family_options.level0_slowdown_writes_trigger = 16;
column_family_options.level0_slowdown_writes_trigger = 32; column_family_options.level0_slowdown_writes_trigger = 32;