dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove some components (#4101) 

Summary:
Remove some components that we never heard people using them.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4101

Differential Revision: D8825431

Pulled By: siying

fbshipit-source-id: 97a12ad3cad4ab12c82741a5ba49669aaa854180
This commit is contained in:
Siying Dong 2019-01-10 13:23:35 -08:00 committed by Facebook Github Bot
parent d56ac22b44
commit 1fb2e274c5
38 changed files with 4 additions and 10393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
CMakeLists.txt
View File

@ -643,18 +643,10 @@ set(SOURCES
utilities/cassandra/format.cc
utilities/cassandra/merge_operator.cc
utilities/checkpoint/checkpoint_impl.cc
utilities/col_buf_decoder.cc
utilities/col_buf_encoder.cc
utilities/column_aware_encoding_util.cc
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
utilities/date_tiered/date_tiered_db_impl.cc
utilities/debug.cc
utilities/document/document_db.cc
utilities/document/json_document.cc
utilities/document/json_document_builder.cc
utilities/env_mirror.cc
utilities/env_timed.cc
utilities/geodb/geodb_impl.cc
utilities/leveldb_options/leveldb_options.cc
utilities/lua/rocks_lua_compaction_filter.cc
utilities/memory/memory_util.cc
@ -671,9 +663,7 @@ set(SOURCES
utilities/persistent_cache/block_cache_tier_metadata.cc
utilities/persistent_cache/persistent_cache_tier.cc
utilities/persistent_cache/volatile_tier_impl.cc
utilities/redis/redis_lists.cc
utilities/simulator_cache/sim_cache.cc
utilities/spatialdb/spatial_db.cc
utilities/table_properties_collectors/compact_on_deletion_collector.cc
utilities/trace/file_trace_reader_writer.cc
utilities/transactions/optimistic_transaction_db_impl.cc
@ -975,11 +965,6 @@ if(WITH_TESTS)
utilities/cassandra/cassandra_row_merge_test.cc
utilities/cassandra/cassandra_serialize_test.cc
utilities/checkpoint/checkpoint_test.cc
utilities/column_aware_encoding_test.cc
utilities/date_tiered/date_tiered_test.cc
utilities/document/document_db_test.cc
utilities/document/json_document_test.cc
utilities/geodb/geodb_test.cc
utilities/lua/rocks_lua_test.cc
utilities/memory/memory_test.cc
utilities/merge_operators/string_append/stringappend_test.cc
@ -988,8 +973,6 @@ if(WITH_TESTS)
utilities/options/options_util_test.cc
utilities/persistent_cache/hash_table_test.cc
utilities/persistent_cache/persistent_cache_test.cc
utilities/redis/redis_lists_test.cc
utilities/spatialdb/spatial_db_test.cc
utilities/simulator_cache/sim_cache_test.cc
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
utilities/transactions/optimistic_transaction_test.cc
@ -1009,7 +992,6 @@ if(WITH_TESTS)
db/range_del_aggregator_bench.cc
tools/db_bench.cc
table/table_reader_bench.cc
utilities/column_aware_encoding_exp.cc
utilities/persistent_cache/hash_table_bench.cc)
add_library(testharness OBJECT util/testharness.cc)
foreach(sourcefile ${BENCHMARKS})

1
HISTORY.md
View File

@ -8,6 +8,7 @@
### Public API Change
* Transaction::GetForUpdate is extended with a do_validate parameter with default value of true. If false it skips validating the snapshot before doing the read. Similarly ::Merge, ::Put, ::Delete, and ::SingleDelete are extended with assume_tracked with default value of false. If true it indicates that call is assumed to be after a ::GetForUpdate.
* `TableProperties::num_entries` and `TableProperties::num_deletions` now also account for number of range tombstones.
* Remove geodb, spatial_db, document_db, json_document, date_tiered_db, and redis_lists.
### Bug Fixes
* Fix a deadlock caused by compaction and file ingestion waiting for each other in the event of write stalls.

37
Makefile
View File

@ -403,7 +403,7 @@ BENCHTOOLOBJECTS = $(BENCH_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
ANALYZETOOLOBJECTS = $(ANALYZER_LIB_SOURCES:.cc=.o)
EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
EXPOBJECTS = $(LIBOBJECTS) $(TESTUTIL)
TESTS = \
db_basic_test \
@ -482,7 +482,6 @@ TESTS = \
merger_test \
util_merge_operators_test \
options_file_test \
redis_test \
reduce_levels_test \
plain_table_db_test \
comparator_db_test \
@ -496,12 +495,8 @@ TESTS = \
cassandra_row_merge_test \
cassandra_serialize_test \
ttl_test \
date_tiered_test \
backupable_db_test \
document_db_test \
json_document_test \
sim_cache_test \
spatial_db_test \
version_edit_test \
version_set_test \
compaction_picker_test \
@ -513,7 +508,6 @@ TESTS = \
deletefile_test \
obsolete_files_test \
table_test \
geodb_test \
delete_scheduler_test \
options_test \
options_settable_test \
@ -530,7 +524,6 @@ TESTS = \
compaction_job_test \
thread_list_test \
sst_dump_test \
column_aware_encoding_test \
compact_files_test \
optimistic_transaction_test \
write_callback_test \
@ -604,7 +597,7 @@ TEST_LIBS = \
librocksdb_env_basic_test.a
# TODO: add back forward_iterator_bench, after making it build in all environemnts.
BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench column_aware_encoding_exp persistent_cache_bench range_del_aggregator_bench
BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench persistent_cache_bench range_del_aggregator_bench
# if user didn't config LIBNAME, set the default
ifeq ($(LIBNAME),)
@ -1153,9 +1146,6 @@ cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utiliti
cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
hash_table_test: utilities/persistent_cache/hash_table_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
@ -1294,18 +1284,9 @@ backupable_db_test: utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TE
checkpoint_test: utilities/checkpoint/checkpoint_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
document_db_test: utilities/document/document_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
json_document_test: utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
sim_cache_test: utilities/simulator_cache/sim_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
@ -1323,9 +1304,6 @@ object_registry_test: utilities/object_registry_test.o $(LIBOBJECTS) $(TESTHARNE
ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
date_tiered_test: utilities/date_tiered/date_tiered_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
@ -1452,9 +1430,6 @@ deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
obsolete_files_test: db/obsolete_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
geodb_test: utilities/geodb/geodb_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
rocksdb_dump: tools/dump/rocksdb_dump.o $(LIBOBJECTS)
$(AM_LINK)
@ -1503,9 +1478,6 @@ timer_queue_test: util/timer_queue_test.o $(LIBOBJECTS) $(TESTHARNESS)
sst_dump_test: tools/sst_dump_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
column_aware_encoding_test: utilities/column_aware_encoding_test.o $(TESTHARNESS) $(EXPOBJECTS)
$(AM_LINK)
optimistic_transaction_test: utilities/transactions/optimistic_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
@ -1545,9 +1517,6 @@ sst_dump: tools/sst_dump.o $(LIBOBJECTS)
blob_dump: tools/blob_dump.o $(LIBOBJECTS)
$(AM_LINK)
column_aware_encoding_exp: utilities/column_aware_encoding_exp.o $(EXPOBJECTS)
$(AM_LINK)
repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
@ -1973,7 +1942,7 @@ endif
# Source files dependencies detection
# ---------------------------------------------------------------------------
all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(EXP_LIB_SOURCES) $(ANALYZER_LIB_SOURCES)
all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES)
DEPFILES = $(all_sources:.cc=.cc.d)
# Add proper dependency support so changing a .h file forces a .cc file to

40
TARGETS
View File

@ -255,14 +255,9 @@ cpp_library(
"utilities/checkpoint/checkpoint_impl.cc",
"utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
"utilities/convenience/info_log_finder.cc",
"utilities/date_tiered/date_tiered_db_impl.cc",
"utilities/debug.cc",
"utilities/document/document_db.cc",
"utilities/document/json_document.cc",
"utilities/document/json_document_builder.cc",
"utilities/env_mirror.cc",
"utilities/env_timed.cc",
"utilities/geodb/geodb_impl.cc",
"utilities/leveldb_options/leveldb_options.cc",
"utilities/lua/rocks_lua_compaction_filter.cc",
"utilities/memory/memory_util.cc",
@ -279,9 +274,7 @@ cpp_library(
"utilities/persistent_cache/block_cache_tier_metadata.cc",
"utilities/persistent_cache/persistent_cache_tier.cc",
"utilities/persistent_cache/volatile_tier_impl.cc",
"utilities/redis/redis_lists.cc",
"utilities/simulator_cache/sim_cache.cc",
"utilities/spatialdb/spatial_db.cc",
"utilities/table_properties_collectors/compact_on_deletion_collector.cc",
"utilities/trace/file_trace_reader_writer.cc",
"utilities/transactions/optimistic_transaction.cc",
@ -319,9 +312,6 @@ cpp_library(
"util/testharness.cc",
"util/testutil.cc",
"utilities/cassandra/test_utils.cc",
"utilities/col_buf_decoder.cc",
"utilities/col_buf_encoder.cc",
"utilities/column_aware_encoding_util.cc",
],
auto_headers = AutoHeaders.RECURSIVE_GLOB,
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
@ -444,11 +434,6 @@ ROCKS_TESTS = [
"util/coding_test.cc",
"serial",
],
[
"column_aware_encoding_test",
"utilities/column_aware_encoding_test.cc",
"serial",
],
[
"column_family_test",
"db/column_family_test.cc",
@ -519,11 +504,6 @@ ROCKS_TESTS = [
"table/data_block_hash_index_test.cc",
"serial",
],
[
"date_tiered_test",
"utilities/date_tiered/date_tiered_test.cc",
"serial",
],
[
"db_basic_test",
"db/db_basic_test.cc",
@ -684,11 +664,6 @@ ROCKS_TESTS = [
"db/deletefile_test.cc",
"serial",
],
[
"document_db_test",
"utilities/document/document_db_test.cc",
"serial",
],
[
"dynamic_bloom_test",
"util/dynamic_bloom_test.cc",
@ -764,11 +739,6 @@ ROCKS_TESTS = [
"table/full_filter_block_test.cc",
"serial",
],
[
"geodb_test",
"utilities/geodb/geodb_test.cc",
"serial",
],
[
"hash_table_test",
"utilities/persistent_cache/hash_table_test.cc",
@ -799,11 +769,6 @@ ROCKS_TESTS = [
"monitoring/iostats_context_test.cc",
"serial",
],
[
"json_document_test",
"utilities/document/json_document_test.cc",
"serial",
],
[
"ldb_cmd_test",
"tools/ldb_cmd_test.cc",
@ -969,11 +934,6 @@ ROCKS_TESTS = [
"util/slice_transform_test.cc",
"serial",
],
[
"spatial_db_test",
"utilities/spatialdb/spatial_db_test.cc",
"serial",
],
[
"sst_dump_test",
"tools/sst_dump_test.cc",

108
include/rocksdb/utilities/date_tiered_db.h
View File

@ -1,108 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include <map>
#include <string>
#include <vector>
#include "rocksdb/db.h"
namespace rocksdb {
// Date tiered database is a wrapper of DB that implements
// a simplified DateTieredCompactionStrategy by using multiple column famillies
// as time windows.
//
// DateTieredDB provides an interface similar to DB, but it assumes that user
// provides keys with last 8 bytes encoded as timestamp in seconds. DateTieredDB
// is assigned with a TTL to declare when data should be deleted.
//
// DateTieredDB hides column families layer from standard RocksDB instance. It
// uses multiple column families to manage time series data, each containing a
// specific range of time. Column families are named by its maximum possible
// timestamp. A column family is created automatically when data newer than
// latest timestamp of all existing column families. The time range of a column
// family is configurable by `column_family_interval`. By doing this, we
// guarantee that compaction will only happen in a column family.
//
// DateTieredDB is assigned with a TTL. When all data in a column family are
// expired (CF_Timestamp <= CUR_Timestamp - TTL), we directly drop the whole
// column family.
//
// TODO(jhli): This is only a simplified version of DTCS. In a complete DTCS,
// time windows can be merged over time, so that older time windows will have
// larger time range. Also, compaction are executed only for adjacent SST files
// to guarantee there is no time overlap between SST files.
class DateTieredDB {
public:
// Open a DateTieredDB whose name is `dbname`.
// Similar to DB::Open(), created database object is stored in dbptr.
//
// Two parameters can be configured: `ttl` to specify the length of time that
// keys should exist in the database, and `column_family_interval` to specify
// the time range of a column family interval.
//
// Open a read only database if read only is set as true.
// TODO(jhli): Should use an option object that includes ttl and
// column_family_interval.
static Status Open(const Options& options, const std::string& dbname,
DateTieredDB** dbptr, int64_t ttl,
int64_t column_family_interval, bool read_only = false);
explicit DateTieredDB() {}
virtual ~DateTieredDB() {}
// Wrapper for Put method. Similar to DB::Put(), but column family to be
// inserted is decided by the timestamp in keys, i.e. the last 8 bytes of user
// key. If key is already obsolete, it will not be inserted.
//
// When client put a key value pair in DateTieredDB, it assumes last 8 bytes
// of keys are encoded as timestamp. Timestamp is a 64-bit signed integer
// encoded as the number of seconds since 1970-01-01 00:00:00 (UTC) (Same as
// Env::GetCurrentTime()). Timestamp should be encoded in big endian.
virtual Status Put(const WriteOptions& options, const Slice& key,
const Slice& val) = 0;
// Wrapper for Get method. Similar to DB::Get() but column family is decided
// by timestamp in keys. If key is already obsolete, it will not be found.
virtual Status Get(const ReadOptions& options, const Slice& key,
std::string* value) = 0;
// Wrapper for Delete method. Similar to DB::Delete() but column family is
// decided by timestamp in keys. If key is already obsolete, return NotFound
// status.
virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
// Wrapper for KeyMayExist method. Similar to DB::KeyMayExist() but column
// family is decided by timestamp in keys. Return false when key is already
// obsolete.
virtual bool KeyMayExist(const ReadOptions& options, const Slice& key,
std::string* value, bool* value_found = nullptr) = 0;
// Wrapper for Merge method. Similar to DB::Merge() but column family is
// decided by timestamp in keys.
virtual Status Merge(const WriteOptions& options, const Slice& key,
const Slice& value) = 0;
// Create an iterator that hides low level details. This iterator internally
// merge results from all active time series column families. Note that
// column families are not deleted until all data are obsolete, so this
// iterator can possibly access obsolete key value pairs.
virtual Iterator* NewIterator(const ReadOptions& opts) = 0;
// Explicitly drop column families in which all keys are obsolete. This
// process is also inplicitly done in Put() operation.
virtual Status DropObsoleteColumnFamilies() = 0;
static const uint64_t kTSLength = sizeof(int64_t); // size of timestamp
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

149
include/rocksdb/utilities/document_db.h
View File

@ -1,149 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include <string>
#include <vector>
#include "rocksdb/utilities/stackable_db.h"
#include "rocksdb/utilities/json_document.h"
#include "rocksdb/db.h"
namespace rocksdb {
// IMPORTANT: DocumentDB is a work in progress. It is unstable and we might
// change the API without warning. Talk to RocksDB team before using this in
// production ;)
// DocumentDB is a layer on top of RocksDB that provides a very simple JSON API.
// When creating a DB, you specify a list of indexes you want to keep on your
// data. You can insert a JSON document to the DB, which is automatically
// indexed. Every document added to the DB needs to have "_id" field which is
// automatically indexed and is an unique primary key. All other indexes are
// non-unique.
// NOTE: field names in the JSON are NOT allowed to start with '$' or
// contain '.'. We don't currently enforce that rule, but will start behaving
// badly.
// Cursor is what you get as a result of executing query. To get all
// results from a query, call Next() on a Cursor while Valid() returns true
class Cursor {
public:
Cursor() = default;
virtual ~Cursor() {}
virtual bool Valid() const = 0;
virtual void Next() = 0;
// Lifecycle of the returned JSONDocument is until the next Next() call
virtual const JSONDocument& document() const = 0;
virtual Status status() const = 0;
private:
// No copying allowed
Cursor(const Cursor&);
void operator=(const Cursor&);
};
struct DocumentDBOptions {
int background_threads = 4;
uint64_t memtable_size = 128 * 1024 * 1024; // 128 MB
uint64_t cache_size = 1 * 1024 * 1024 * 1024; // 1 GB
};
// TODO(icanadi) Add `JSONDocument* info` parameter to all calls that can be
// used by the caller to get more information about the call execution (number
// of dropped records, number of updated records, etc.)
class DocumentDB : public StackableDB {
public:
struct IndexDescriptor {
// Currently, you can only define an index on a single field. To specify an
// index on a field X, set index description to JSON "{X: 1}"
// Currently the value needs to be 1, which means ascending.
// In the future, we plan to also support indexes on multiple keys, where
// you could mix ascending sorting (1) with descending sorting indexes (-1)
JSONDocument* description;
std::string name;
};
// Open DocumentDB with specified indexes. The list of indexes has to be
// complete, i.e. include all indexes present in the DB, except the primary
// key index.
// Otherwise, Open() will return an error
static Status Open(const DocumentDBOptions& options, const std::string& name,
const std::vector<IndexDescriptor>& indexes,
DocumentDB** db, bool read_only = false);
explicit DocumentDB(DB* db) : StackableDB(db) {}
// Create a new index. It will stop all writes for the duration of the call.
// All current documents in the DB are scanned and corresponding index entries
// are created
virtual Status CreateIndex(const WriteOptions& write_options,
const IndexDescriptor& index) = 0;
// Drop an index. Client is responsible to make sure that index is not being
// used by currently executing queries
virtual Status DropIndex(const std::string& name) = 0;
// Insert a document to the DB. The document needs to have a primary key "_id"
// which can either be a string or an integer. Otherwise the write will fail
// with InvalidArgument.
virtual Status Insert(const WriteOptions& options,
const JSONDocument& document) = 0;
// Deletes all documents matching a filter atomically
virtual Status Remove(const ReadOptions& read_options,
const WriteOptions& write_options,
const JSONDocument& query) = 0;
// Does this sequence of operations:
// 1. Find all documents matching a filter
// 2. For all documents, atomically:
// 2.1. apply the update operators
// 2.2. update the secondary indexes
//
// Currently only $set update operator is supported.
// Syntax is: {$set: {key1: value1, key2: value2, etc...}}
// This operator will change a document's key1 field to value1, key2 to
// value2, etc. New values will be set even if a document didn't have an entry
// for the specified key.
//
// You can not change a primary key of a document.
//
// Update example: Update({id: {$gt: 5}, $index: id}, {$set: {enabled: true}})
virtual Status Update(const ReadOptions& read_options,
const WriteOptions& write_options,
const JSONDocument& filter,
const JSONDocument& updates) = 0;
// query has to be an array in which every element is an operator. Currently
// only $filter operator is supported. Syntax of $filter operator is:
// {$filter: {key1: condition1, key2: condition2, etc.}} where conditions can
// be either:
// 1) a single value in which case the condition is equality condition, or
// 2) a defined operators, like {$gt: 4}, which will match all documents that
// have key greater than 4.
//
// Supported operators are:
// 1) $gt -- greater than
// 2) $gte -- greater than or equal
// 3) $lt -- less than
// 4) $lte -- less than or equal
// If you want the filter to use an index, you need to specify it like this:
// {$filter: {...(conditions)..., $index: index_name}}
//
// Example query:
// * [{$filter: {name: John, age: {$gte: 18}, $index: age}}]
// will return all Johns whose age is greater or equal to 18 and it will use
// index "age" to satisfy the query.
virtual Cursor* Query(const ReadOptions& read_options,
const JSONDocument& query) = 0;
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

114
include/rocksdb/utilities/geo_db.h
View File

@ -1,114 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
#include <vector>
#include "rocksdb/utilities/stackable_db.h"
#include "rocksdb/status.h"
namespace rocksdb {
//
// Configurable options needed for setting up a Geo database
//
struct GeoDBOptions {
// Backup info and error messages will be written to info_log
// if non-nullptr.
// Default: nullptr
Logger* info_log;
explicit GeoDBOptions(Logger* _info_log = nullptr):info_log(_info_log) { }
};
//
// A position in the earth's geoid
//
class GeoPosition {
public:
double latitude;
double longitude;
explicit GeoPosition(double la = 0, double lo = 0) :
latitude(la), longitude(lo) {
}
};
//
// Description of an object on the Geoid. It is located by a GPS location,
// and is identified by the id. The value associated with this object is
// an opaque string 'value'. Different objects identified by unique id's
// can have the same gps-location associated with them.
//
class GeoObject {
public:
GeoPosition position;
std::string id;
std::string value;
GeoObject() {}
GeoObject(const GeoPosition& pos, const std::string& i,
const std::string& val) :
position(pos), id(i), value(val) {
}
};
class GeoIterator {
public:
GeoIterator() = default;
virtual ~GeoIterator() {}
virtual void Next() = 0;
virtual bool Valid() const = 0;
virtual const GeoObject& geo_object() = 0;
virtual Status status() const = 0;
};
//
// Stack your DB with GeoDB to be able to get geo-spatial support
//
class GeoDB : public StackableDB {
public:
// GeoDBOptions have to be the same as the ones used in a previous
// incarnation of the DB
//
// GeoDB owns the pointer `DB* db` now. You should not delete it or
// use it after the invocation of GeoDB
// GeoDB(DB* db, const GeoDBOptions& options) : StackableDB(db) {}
GeoDB(DB* db, const GeoDBOptions& /*options*/) : StackableDB(db) {}
virtual ~GeoDB() {}
// Insert a new object into the location database. The object is
// uniquely identified by the id. If an object with the same id already
// exists in the db, then the old one is overwritten by the new
// object being inserted here.
virtual Status Insert(const GeoObject& object) = 0;
// Retrieve the value of the object located at the specified GPS
// location and is identified by the 'id'.
virtual Status GetByPosition(const GeoPosition& pos,
const Slice& id, std::string* value) = 0;
// Retrieve the value of the object identified by the 'id'. This method
// could be potentially slower than GetByPosition
virtual Status GetById(const Slice& id, GeoObject* object) = 0;
// Delete the specified object
virtual Status Remove(const Slice& id) = 0;
// Returns an iterator for the items within a circular radius from the
// specified gps location. If 'number_of_values' is specified,
// then the iterator is capped to that number of objects.
// The radius is specified in 'meters'.
virtual GeoIterator* SearchRadial(const GeoPosition& pos,
double radius,
int number_of_values = INT_MAX) = 0;
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

195
include/rocksdb/utilities/json_document.h
View File

@ -1,195 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include <deque>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "rocksdb/slice.h"
// We use JSONDocument for DocumentDB API
// Implementation inspired by folly::dynamic, rapidjson and fbson
namespace fbson {
class FbsonValue;
class ObjectVal;
template <typename T>
class FbsonWriterT;
class FbsonOutStream;
typedef FbsonWriterT<FbsonOutStream> FbsonWriter;
} // namespace fbson
namespace rocksdb {
// NOTE: none of this is thread-safe
class JSONDocument {
public:
// return nullptr on parse failure
static JSONDocument* ParseJSON(const char* json);
enum Type {
kNull,
kArray,
kBool,
kDouble,
kInt64,
kObject,
kString,
};
/* implicit */ JSONDocument(); // null
/* implicit */ JSONDocument(bool b);
/* implicit */ JSONDocument(double d);
/* implicit */ JSONDocument(int8_t i);
/* implicit */ JSONDocument(int16_t i);
/* implicit */ JSONDocument(int32_t i);
/* implicit */ JSONDocument(int64_t i);
/* implicit */ JSONDocument(const std::string& s);
/* implicit */ JSONDocument(const char* s);
// constructs JSONDocument of specific type with default value
explicit JSONDocument(Type _type);
JSONDocument(const JSONDocument& json_document);
JSONDocument(JSONDocument&& json_document);
Type type() const;
// REQUIRES: IsObject()
bool Contains(const std::string& key) const;
// REQUIRES: IsObject()
// Returns non-owner object
JSONDocument operator[](const std::string& key) const;
// REQUIRES: IsArray() == true || IsObject() == true
size_t Count() const;
// REQUIRES: IsArray()
// Returns non-owner object
JSONDocument operator[](size_t i) const;
JSONDocument& operator=(JSONDocument jsonDocument);
bool IsNull() const;
bool IsArray() const;
bool IsBool() const;
bool IsDouble() const;
bool IsInt64() const;
bool IsObject() const;
bool IsString() const;
// REQUIRES: IsBool() == true
bool GetBool() const;
// REQUIRES: IsDouble() == true
double GetDouble() const;
// REQUIRES: IsInt64() == true
int64_t GetInt64() const;
// REQUIRES: IsString() == true
std::string GetString() const;
bool operator==(const JSONDocument& rhs) const;
bool operator!=(const JSONDocument& rhs) const;
JSONDocument Copy() const;
bool IsOwner() const;
std::string DebugString() const;
private:
class ItemsIteratorGenerator;
public:
// REQUIRES: IsObject()
ItemsIteratorGenerator Items() const;
// appends serialized object to dst
void Serialize(std::string* dst) const;
// returns nullptr if Slice doesn't represent valid serialized JSONDocument
static JSONDocument* Deserialize(const Slice& src);
private:
friend class JSONDocumentBuilder;
JSONDocument(fbson::FbsonValue* val, bool makeCopy);
void InitFromValue(const fbson::FbsonValue* val);
// iteration on objects
class const_item_iterator {
private:
class Impl;
public:
typedef std::pair<std::string, JSONDocument> value_type;
explicit const_item_iterator(Impl* impl);
const_item_iterator(const_item_iterator&&);
const_item_iterator& operator++();
bool operator!=(const const_item_iterator& other);
value_type operator*();
~const_item_iterator();
private:
friend class ItemsIteratorGenerator;
std::unique_ptr<Impl> it_;
};
class ItemsIteratorGenerator {
public:
explicit ItemsIteratorGenerator(const fbson::ObjectVal& object);
const_item_iterator begin() const;
const_item_iterator end() const;
private:
const fbson::ObjectVal& object_;
};
std::unique_ptr<char[]> data_;
mutable fbson::FbsonValue* value_;
// Our serialization format's first byte specifies the encoding version. That
// way, we can easily change our format while providing backwards
// compatibility. This constant specifies the current version of the
// serialization format
static const char kSerializationFormatVersion;
};
class JSONDocumentBuilder {
public:
JSONDocumentBuilder();
explicit JSONDocumentBuilder(fbson::FbsonOutStream* out);
void Reset();
bool WriteStartArray();
bool WriteEndArray();
bool WriteStartObject();
bool WriteEndObject();
bool WriteKeyValue(const std::string& key, const JSONDocument& value);
bool WriteJSONDocument(const JSONDocument& value);
JSONDocument GetJSONDocument();
~JSONDocumentBuilder();
private:
std::unique_ptr<fbson::FbsonWriter> writer_;
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

261
include/rocksdb/utilities/spatial_db.h
View File

@ -1,261 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include <string>
#include <vector>
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/utilities/stackable_db.h"
namespace rocksdb {
namespace spatial {
// NOTE: SpatialDB is experimental and we might change its API without warning.
// Please talk to us before developing against SpatialDB API.
//
// SpatialDB is a support for spatial indexes built on top of RocksDB.
// When creating a new SpatialDB, clients specifies a list of spatial indexes to
// build on their data. Each spatial index is defined by the area and
// granularity. If you're storing map data, different spatial index
// granularities can be used for different zoom levels.
//
// Each element inserted into SpatialDB has:
// * a bounding box, which determines how will the element be indexed
// * string blob, which will usually be WKB representation of the polygon
// (http://en.wikipedia.org/wiki/Well-known_text)
// * feature set, which is a map of key-value pairs, where value can be null,
// int, double, bool, string
// * a list of indexes to insert the element in
//
// Each query is executed on a single spatial index. Query guarantees that it
// will return all elements intersecting the specified bounding box, but it
// might also return some extra non-intersecting elements.
// Variant is a class that can be many things: null, bool, int, double or string
// It is used to store different value types in FeatureSet (see below)
struct Variant {
// Don't change the values here, they are persisted on disk
enum Type {
kNull = 0x0,
kBool = 0x1,
kInt = 0x2,
kDouble = 0x3,
kString = 0x4,
};
Variant() : type_(kNull) {}
/* implicit */ Variant(bool b) : type_(kBool) { data_.b = b; }
/* implicit */ Variant(uint64_t i) : type_(kInt) { data_.i = i; }
/* implicit */ Variant(double d) : type_(kDouble) { data_.d = d; }
/* implicit */ Variant(const std::string& s) : type_(kString) {
new (&data_.s) std::string(s);
}
Variant(const Variant& v) : type_(v.type_) { Init(v, data_); }
Variant& operator=(const Variant& v);
Variant(Variant&& rhs) : type_(kNull) { *this = std::move(rhs); }
Variant& operator=(Variant&& v);
~Variant() { Destroy(type_, data_); }
Type type() const { return type_; }
bool get_bool() const { return data_.b; }
uint64_t get_int() const { return data_.i; }
double get_double() const { return data_.d; }
const std::string& get_string() const { return *GetStringPtr(data_); }
bool operator==(const Variant& other) const;
bool operator!=(const Variant& other) const { return !(*this == other); }
private:
Type type_;
union Data {
bool b;
uint64_t i;
double d;
// Current version of MS compiler not C++11 compliant so can not put
// std::string
// however, even then we still need the rest of the maintenance.
char s[sizeof(std::string)];
} data_;
// Avoid type_punned aliasing problem
static std::string* GetStringPtr(Data& d) {
void* p = d.s;
return reinterpret_cast<std::string*>(p);
}
static const std::string* GetStringPtr(const Data& d) {
const void* p = d.s;
return reinterpret_cast<const std::string*>(p);
}
static void Init(const Variant&, Data&);
static void Destroy(Type t, Data& d) {
if (t == kString) {
using std::string;
GetStringPtr(d)->~string();
}
}
};
// FeatureSet is a map of key-value pairs. One feature set is associated with
// each element in SpatialDB. It can be used to add rich data about the element.
class FeatureSet {
private:
typedef std::unordered_map<std::string, Variant> map;
public:
class iterator {
public:
/* implicit */ iterator(const map::const_iterator itr) : itr_(itr) {}
iterator& operator++() {
++itr_;
return *this;
}
bool operator!=(const iterator& other) { return itr_ != other.itr_; }
bool operator==(const iterator& other) { return itr_ == other.itr_; }
map::value_type operator*() { return *itr_; }
private:
map::const_iterator itr_;
};
FeatureSet() = default;
FeatureSet* Set(const std::string& key, const Variant& value);
bool Contains(const std::string& key) const;
// REQUIRES: Contains(key)
const Variant& Get(const std::string& key) const;
iterator Find(const std::string& key) const;
iterator begin() const { return map_.begin(); }
iterator end() const { return map_.end(); }
void Clear();
size_t Size() const { return map_.size(); }
void Serialize(std::string* output) const;
// REQUIRED: empty FeatureSet
bool Deserialize(const Slice& input);
std::string DebugString() const;
private:
map map_;
};
// BoundingBox is a helper structure for defining rectangles representing
// bounding boxes of spatial elements.
template <typename T>
struct BoundingBox {
T min_x, min_y, max_x, max_y;
BoundingBox() = default;
BoundingBox(T _min_x, T _min_y, T _max_x, T _max_y)
: min_x(_min_x), min_y(_min_y), max_x(_max_x), max_y(_max_y) {}
bool Intersects(const BoundingBox<T>& a) const {
return !(min_x > a.max_x || min_y > a.max_y || a.min_x > max_x ||
a.min_y > max_y);
}
};
struct SpatialDBOptions {
uint64_t cache_size = 1 * 1024 * 1024 * 1024LL; // 1GB
int num_threads = 16;
bool bulk_load = true;
};
// Cursor is used to return data from the query to the client. To get all the
// data from the query, just call Next() while Valid() is true
class Cursor {
public:
Cursor() = default;
virtual ~Cursor() {}
virtual bool Valid() const = 0;
// REQUIRES: Valid()
virtual void Next() = 0;
// Lifetime of the underlying storage until the next call to Next()
// REQUIRES: Valid()
virtual const Slice blob() = 0;
// Lifetime of the underlying storage until the next call to Next()
// REQUIRES: Valid()
virtual const FeatureSet& feature_set() = 0;
virtual Status status() const = 0;
private:
// No copying allowed
Cursor(const Cursor&);
void operator=(const Cursor&);
};
// SpatialIndexOptions defines a spatial index that will be built on the data
struct SpatialIndexOptions {
// Spatial indexes are referenced by names
std::string name;
// An area that is indexed. If the element is not intersecting with spatial
// index's bbox, it will not be inserted into the index
BoundingBox<double> bbox;
// tile_bits control the granularity of the spatial index. Each dimension of
// the bbox will be split into (1 << tile_bits) tiles, so there will be a
// total of (1 << tile_bits)^2 tiles. It is recommended to configure a size of
// each tile to be approximately the size of the query on that spatial index
uint32_t tile_bits;
SpatialIndexOptions() {}
SpatialIndexOptions(const std::string& _name,
const BoundingBox<double>& _bbox, uint32_t _tile_bits)
: name(_name), bbox(_bbox), tile_bits(_tile_bits) {}
};
class SpatialDB : public StackableDB {
public:
// Creates the SpatialDB with specified list of indexes.
// REQUIRED: db doesn't exist
static Status Create(const SpatialDBOptions& options, const std::string& name,
const std::vector<SpatialIndexOptions>& spatial_indexes);
// Open the existing SpatialDB. The resulting db object will be returned
// through db parameter.
// REQUIRED: db was created using SpatialDB::Create
static Status Open(const SpatialDBOptions& options, const std::string& name,
SpatialDB** db, bool read_only = false);
explicit SpatialDB(DB* db) : StackableDB(db) {}
// Insert the element into the DB. Element will be inserted into specified
// spatial_indexes, based on specified bbox.
// REQUIRES: spatial_indexes.size() > 0
virtual Status Insert(const WriteOptions& write_options,
const BoundingBox<double>& bbox, const Slice& blob,
const FeatureSet& feature_set,
const std::vector<std::string>& spatial_indexes) = 0;
// Calling Compact() after inserting a bunch of elements should speed up
// reading. This is especially useful if you use SpatialDBOptions::bulk_load
// Num threads determines how many threads we'll use for compactions. Setting
// this to bigger number will use more IO and CPU, but finish faster
virtual Status Compact(int num_threads = 1) = 0;
// Query the specified spatial_index. Query will return all elements that
// intersect bbox, but it may also return some extra elements.
virtual Cursor* Query(const ReadOptions& read_options,
const BoundingBox<double>& bbox,
const std::string& spatial_index) = 0;
};
} // namespace spatial
} // namespace rocksdb
#endif // ROCKSDB_LITE

21
src.mk
View File

@ -177,14 +177,9 @@ LIB_SOURCES = \
utilities/checkpoint/checkpoint_impl.cc \
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc \
utilities/convenience/info_log_finder.cc \
utilities/date_tiered/date_tiered_db_impl.cc \
utilities/debug.cc \
utilities/document/document_db.cc \
utilities/document/json_document.cc \
utilities/document/json_document_builder.cc \
utilities/env_mirror.cc \
utilities/env_timed.cc \
utilities/geodb/geodb_impl.cc \
utilities/leveldb_options/leveldb_options.cc \
utilities/lua/rocks_lua_compaction_filter.cc \
utilities/memory/memory_util.cc \
@ -201,9 +196,7 @@ LIB_SOURCES = \
utilities/persistent_cache/block_cache_tier_metadata.cc \
utilities/persistent_cache/persistent_cache_tier.cc \
utilities/persistent_cache/volatile_tier_impl.cc \
utilities/redis/redis_lists.cc \
utilities/simulator_cache/sim_cache.cc \
utilities/spatialdb/spatial_db.cc \
utilities/table_properties_collectors/compact_on_deletion_collector.cc \
utilities/trace/file_trace_reader_writer.cc \
utilities/transactions/optimistic_transaction.cc \
@ -249,11 +242,6 @@ MOCK_LIB_SOURCES = \
BENCH_LIB_SOURCES = \
tools/db_bench_tool.cc \
EXP_LIB_SOURCES = \
utilities/col_buf_decoder.cc \
utilities/col_buf_encoder.cc \
utilities/column_aware_encoding_util.cc
TEST_LIB_SOURCES = \
db/db_test_util.cc \
util/testharness.cc \
@ -330,7 +318,6 @@ MAIN_SOURCES = \
db/persistent_cache_test.cc \
db/plain_table_db_test.cc \
db/prefix_test.cc \
db/redis_test.cc \
db/repair_test.cc \
db/range_del_aggregator_test.cc \
db/range_del_aggregator_bench.cc \
@ -397,21 +384,13 @@ MAIN_SOURCES = \
utilities/cassandra/cassandra_row_merge_test.cc \
utilities/cassandra/cassandra_serialize_test.cc \
utilities/checkpoint/checkpoint_test.cc \
utilities/column_aware_encoding_exp.cc \
utilities/column_aware_encoding_test.cc \
utilities/date_tiered/date_tiered_test.cc \
utilities/document/document_db_test.cc \
utilities/document/json_document_test.cc \
utilities/geodb/geodb_test.cc \
utilities/lua/rocks_lua_test.cc \
utilities/memory/memory_test.cc \
utilities/merge_operators/string_append/stringappend_test.cc \
utilities/object_registry_test.cc \
utilities/option_change_migration/option_change_migration_test.cc \
utilities/options/options_util_test.cc \
utilities/redis/redis_lists_test.cc \
utilities/simulator_cache/sim_cache_test.cc \
utilities/spatialdb/spatial_db_test.cc \
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc \
utilities/transactions/optimistic_transaction_test.cc \
utilities/transactions/transaction_test.cc \

240
utilities/col_buf_decoder.cc
View File

@ -1,240 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "utilities/col_buf_decoder.h"
#include <cstring>
#include <string>
#include "port/port.h"
namespace rocksdb {
ColBufDecoder::~ColBufDecoder() {}
namespace {
inline uint64_t EncodeFixed64WithEndian(uint64_t val, bool big_endian,
size_t size) {
if (big_endian && port::kLittleEndian) {
val = EndianTransform(val, size);
} else if (!big_endian && !port::kLittleEndian) {
val = EndianTransform(val, size);
}
return val;
}
} // namespace
ColBufDecoder* ColBufDecoder::NewColBufDecoder(
const ColDeclaration& col_declaration) {
if (col_declaration.col_type == "FixedLength") {
return new FixedLengthColBufDecoder(
col_declaration.size, col_declaration.col_compression_type,
col_declaration.nullable, col_declaration.big_endian);
} else if (col_declaration.col_type == "VariableLength") {
return new VariableLengthColBufDecoder();
} else if (col_declaration.col_type == "VariableChunk") {
return new VariableChunkColBufDecoder(col_declaration.col_compression_type);
} else if (col_declaration.col_type == "LongFixedLength") {
return new LongFixedLengthColBufDecoder(col_declaration.size,
col_declaration.nullable);
}
// Unrecognized column type
return nullptr;
}
namespace {
void ReadVarint64(const char** src_ptr, uint64_t* val_ptr) {
const char* q = GetVarint64Ptr(*src_ptr, *src_ptr + 10, val_ptr);
assert(q != nullptr);
*src_ptr = q;
}
} // namespace
size_t FixedLengthColBufDecoder::Init(const char* src) {
remain_runs_ = 0;
last_val_ = 0;
// Dictionary initialization
dict_vec_.clear();
const char* orig_src = src;
if (col_compression_type_ == kColDict ||
col_compression_type_ == kColRleDict) {
const char* q;
uint64_t dict_size;
// Bypass limit
q = GetVarint64Ptr(src, src + 10, &dict_size);
assert(q != nullptr);
src = q;
uint64_t dict_key;
for (uint64_t i = 0; i < dict_size; ++i) {
// Bypass limit
ReadVarint64(&src, &dict_key);
dict_key = EncodeFixed64WithEndian(dict_key, big_endian_, size_);
dict_vec_.push_back(dict_key);
}
}
return src - orig_src;
}
size_t FixedLengthColBufDecoder::Decode(const char* src, char** dest) {
uint64_t read_val = 0;
const char* orig_src = src;
const char* src_limit = src + 20;
if (nullable_) {
bool not_null;
not_null = *src;
src += 1;
if (!not_null) {
return 1;
}
}
if (IsRunLength(col_compression_type_)) {
if (remain_runs_ == 0) {
const char* q;
run_val_ = 0;
if (col_compression_type_ == kColRle) {
memcpy(&run_val_, src, size_);
src += size_;
} else {
q = GetVarint64Ptr(src, src_limit, &run_val_);
assert(q != nullptr);
src = q;
}
q = GetVarint64Ptr(src, src_limit, &remain_runs_);
assert(q != nullptr);
src = q;
if (col_compression_type_ != kColRleDeltaVarint &&
col_compression_type_ != kColRleDict) {
run_val_ = EncodeFixed64WithEndian(run_val_, big_endian_, size_);
}
}
read_val = run_val_;
} else {
if (col_compression_type_ == kColNoCompression) {
memcpy(&read_val, src, size_);
src += size_;
} else {
// Assume a column does not exceed 8 bytes here
const char* q = GetVarint64Ptr(src, src_limit, &read_val);
assert(q != nullptr);
src = q;
}
if (col_compression_type_ != kColDeltaVarint &&
col_compression_type_ != kColDict) {
read_val = EncodeFixed64WithEndian(read_val, big_endian_, size_);
}
}
uint64_t write_val = read_val;
if (col_compression_type_ == kColDeltaVarint ||
col_compression_type_ == kColRleDeltaVarint) {
// does not support 64 bit
uint64_t mask = (write_val & 1) ? (~uint64_t(0)) : 0;
int64_t delta = (write_val >> 1) ^ mask;
write_val = last_val_ + delta;
uint64_t tmp = write_val;
write_val = EncodeFixed64WithEndian(write_val, big_endian_, size_);
last_val_ = tmp;
} else if (col_compression_type_ == kColRleDict ||
col_compression_type_ == kColDict) {
uint64_t dict_val = read_val;
assert(dict_val < dict_vec_.size());
write_val = dict_vec_[static_cast<size_t>(dict_val)];
}
// dest->append(reinterpret_cast<char*>(&write_val), size_);
memcpy(*dest, reinterpret_cast<char*>(&write_val), size_);
*dest += size_;
if (IsRunLength(col_compression_type_)) {
--remain_runs_;
}
return src - orig_src;
}
size_t LongFixedLengthColBufDecoder::Decode(const char* src, char** dest) {
if (nullable_) {
bool not_null;
not_null = *src;
src += 1;
if (!not_null) {
return 1;
}
}
memcpy(*dest, src, size_);
*dest += size_;
return size_ + 1;
}
size_t VariableLengthColBufDecoder::Decode(const char* src, char** dest) {
uint8_t len;
len = *src;
memcpy(dest, reinterpret_cast<char*>(&len), 1);
*dest += 1;
src += 1;
memcpy(*dest, src, len);
*dest += len;
return len + 1;
}
size_t VariableChunkColBufDecoder::Init(const char* src) {
// Dictionary initialization
dict_vec_.clear();
const char* orig_src = src;
if (col_compression_type_ == kColDict) {
const char* q;
uint64_t dict_size;
// Bypass limit
q = GetVarint64Ptr(src, src + 10, &dict_size);
assert(q != nullptr);
src = q;
uint64_t dict_key;
for (uint64_t i = 0; i < dict_size; ++i) {
// Bypass limit
ReadVarint64(&src, &dict_key);
dict_vec_.push_back(dict_key);
}
}
return src - orig_src;
}
size_t VariableChunkColBufDecoder::Decode(const char* src, char** dest) {
const char* orig_src = src;
uint64_t size = 0;
ReadVarint64(&src, &size);
int64_t full_chunks = size / 8;
uint64_t chunk_buf;
size_t chunk_size = 8;
for (int64_t i = 0; i < full_chunks + 1; ++i) {
chunk_buf = 0;
if (i == full_chunks) {
chunk_size = size % 8;
}
if (col_compression_type_ == kColDict) {
uint64_t dict_val;
ReadVarint64(&src, &dict_val);
assert(dict_val < dict_vec_.size());
chunk_buf = dict_vec_[static_cast<size_t>(dict_val)];
} else {
memcpy(&chunk_buf, src, chunk_size);
src += chunk_size;
}
memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8);
*dest += 8;
uint8_t mask = ((0xFF - 8) + chunk_size) & 0xFF;
memcpy(*dest, reinterpret_cast<char*>(&mask), 1);
*dest += 1;
}
return src - orig_src;
}
} // namespace rocksdb

119
utilities/col_buf_decoder.h
View File

@ -1,119 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <cstdio>
#include <cstring>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "util/coding.h"
#include "utilities/col_buf_encoder.h"
namespace rocksdb {
struct ColDeclaration;
// ColBufDecoder is a class to decode column buffers. It can be populated from a
// ColDeclaration. Before starting decoding, a Init() method should be called.
// Each time it takes a column value into Decode() method.
class ColBufDecoder {
public:
virtual ~ColBufDecoder() = 0;
virtual size_t Init(const char* /*src*/) { return 0; }
virtual size_t Decode(const char* src, char** dest) = 0;
static ColBufDecoder* NewColBufDecoder(const ColDeclaration& col_declaration);
protected:
std::string buffer_;
static inline bool IsRunLength(ColCompressionType type) {
return type == kColRle || type == kColRleVarint ||
type == kColRleDeltaVarint || type == kColRleDict;
}
};
class FixedLengthColBufDecoder : public ColBufDecoder {
public:
explicit FixedLengthColBufDecoder(
size_t size, ColCompressionType col_compression_type = kColNoCompression,
bool nullable = false, bool big_endian = false)
: size_(size),
col_compression_type_(col_compression_type),
nullable_(nullable),
big_endian_(big_endian),
remain_runs_(0),
run_val_(0),
last_val_(0) {}
size_t Init(const char* src) override;
size_t Decode(const char* src, char** dest) override;
~FixedLengthColBufDecoder() {}
private:
size_t size_;
ColCompressionType col_compression_type_;
bool nullable_;
bool big_endian_;
// for decoding
std::vector<uint64_t> dict_vec_;
uint64_t remain_runs_;
uint64_t run_val_;
uint64_t last_val_;
};
class LongFixedLengthColBufDecoder : public ColBufDecoder {
public:
LongFixedLengthColBufDecoder(size_t size, bool nullable)
: size_(size), nullable_(nullable) {}
size_t Decode(const char* src, char** dest) override;
~LongFixedLengthColBufDecoder() {}
private:
size_t size_;
bool nullable_;
};
class VariableLengthColBufDecoder : public ColBufDecoder {
public:
size_t Decode(const char* src, char** dest) override;
~VariableLengthColBufDecoder() {}
};
class VariableChunkColBufDecoder : public VariableLengthColBufDecoder {
public:
size_t Init(const char* src) override;
size_t Decode(const char* src, char** dest) override;
explicit VariableChunkColBufDecoder(ColCompressionType col_compression_type)
: col_compression_type_(col_compression_type) {}
VariableChunkColBufDecoder() : col_compression_type_(kColNoCompression) {}
private:
ColCompressionType col_compression_type_;
std::unordered_map<uint64_t, uint64_t> dictionary_;
std::vector<uint64_t> dict_vec_;
};
struct KVPairColBufDecoders {
std::vector<std::unique_ptr<ColBufDecoder>> key_col_bufs;
std::vector<std::unique_ptr<ColBufDecoder>> value_col_bufs;
std::unique_ptr<ColBufDecoder> value_checksum_buf;
explicit KVPairColBufDecoders(const KVPairColDeclarations& kvp_cd) {
for (auto kcd : *kvp_cd.key_col_declarations) {
key_col_bufs.emplace_back(
std::move(ColBufDecoder::NewColBufDecoder(kcd)));
}
for (auto vcd : *kvp_cd.value_col_declarations) {
value_col_bufs.emplace_back(
std::move(ColBufDecoder::NewColBufDecoder(vcd)));
}
value_checksum_buf.reset(
ColBufDecoder::NewColBufDecoder(*kvp_cd.value_checksum_declaration));
}
};
} // namespace rocksdb

210
utilities/col_buf_encoder.cc
View File

@ -1,210 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "utilities/col_buf_encoder.h"
#include <cstring>
#include <string>
#include "port/port.h"
namespace rocksdb {
ColBufEncoder::~ColBufEncoder() {}
namespace {
inline uint64_t DecodeFixed64WithEndian(uint64_t val, bool big_endian,
size_t size) {
if (big_endian && port::kLittleEndian) {
val = EndianTransform(val, size);
} else if (!big_endian && !port::kLittleEndian) {
val = EndianTransform(val, size);
}
return val;
}
} // namespace
const std::string &ColBufEncoder::GetData() { return buffer_; }
ColBufEncoder *ColBufEncoder::NewColBufEncoder(
const ColDeclaration &col_declaration) {
if (col_declaration.col_type == "FixedLength") {
return new FixedLengthColBufEncoder(
col_declaration.size, col_declaration.col_compression_type,
col_declaration.nullable, col_declaration.big_endian);
} else if (col_declaration.col_type == "VariableLength") {
return new VariableLengthColBufEncoder();
} else if (col_declaration.col_type == "VariableChunk") {
return new VariableChunkColBufEncoder(col_declaration.col_compression_type);
} else if (col_declaration.col_type == "LongFixedLength") {
return new LongFixedLengthColBufEncoder(col_declaration.size,
col_declaration.nullable);
}
// Unrecognized column type
return nullptr;
}
size_t FixedLengthColBufEncoder::Append(const char *buf) {
if (nullable_) {
if (buf == nullptr) {
buffer_.append(1, 0);
return 0;
} else {
buffer_.append(1, 1);
}
}
uint64_t read_val = 0;
memcpy(&read_val, buf, size_);
read_val = DecodeFixed64WithEndian(read_val, big_endian_, size_);
// Determine write value
uint64_t write_val = read_val;
if (col_compression_type_ == kColDeltaVarint ||
col_compression_type_ == kColRleDeltaVarint) {
int64_t delta = read_val - last_val_;
// Encode signed delta value
delta = (static_cast<uint64_t>(delta) << 1) ^ (delta >> 63);
write_val = delta;
last_val_ = read_val;
} else if (col_compression_type_ == kColDict ||
col_compression_type_ == kColRleDict) {
auto iter = dictionary_.find(read_val);
uint64_t dict_val;
if (iter == dictionary_.end()) {
// Add new entry to dictionary
dict_val = dictionary_.size();
dictionary_.insert(std::make_pair(read_val, dict_val));
dict_vec_.push_back(read_val);
} else {
dict_val = iter->second;
}
write_val = dict_val;
}
// Write into buffer
if (IsRunLength(col_compression_type_)) {
if (run_length_ == -1) {
// First element
run_val_ = write_val;
run_length_ = 1;
} else if (write_val != run_val_) {
// End of run
// Write run value
if (col_compression_type_ == kColRle) {
buffer_.append(reinterpret_cast<char *>(&run_val_), size_);
} else {
PutVarint64(&buffer_, run_val_);
}
// Write run length
PutVarint64(&buffer_, run_length_);
run_val_ = write_val;
run_length_ = 1;
} else {
run_length_++;
}
} else { // non run-length encodings
if (col_compression_type_ == kColNoCompression) {
buffer_.append(reinterpret_cast<char *>(&write_val), size_);
} else {
PutVarint64(&buffer_, write_val);
}
}
return size_;
}
void FixedLengthColBufEncoder::Finish() {
if (col_compression_type_ == kColDict ||
col_compression_type_ == kColRleDict) {
std::string header;
PutVarint64(&header, dict_vec_.size());
// Put dictionary in the header
for (auto item : dict_vec_) {
PutVarint64(&header, item);
}
buffer_ = header + buffer_;
}
if (IsRunLength(col_compression_type_)) {
// Finish last run value
if (col_compression_type_ == kColRle) {
buffer_.append(reinterpret_cast<char *>(&run_val_), size_);
} else {
PutVarint64(&buffer_, run_val_);
}
PutVarint64(&buffer_, run_length_);
}
}
size_t LongFixedLengthColBufEncoder::Append(const char *buf) {
if (nullable_) {
if (buf == nullptr) {
buffer_.append(1, 0);
return 0;
} else {
buffer_.append(1, 1);
}
}
buffer_.append(buf, size_);
return size_;
}
void LongFixedLengthColBufEncoder::Finish() {}
size_t VariableLengthColBufEncoder::Append(const char *buf) {
uint8_t length = 0;
length = *buf;
buffer_.append(buf, 1);
buf += 1;
buffer_.append(buf, length);
return length + 1;
}
void VariableLengthColBufEncoder::Finish() {}
size_t VariableChunkColBufEncoder::Append(const char *buf) {
const char *orig_buf = buf;
uint8_t mark = 0xFF;
size_t length = 0;
std::string tmp_buffer;
while (mark == 0xFF) {
uint64_t val;
memcpy(&val, buf, 8);
buf += 8;
mark = *buf;
buf += 1;
int8_t chunk_size = 8 - (0xFF - mark);
if (col_compression_type_ == kColDict) {
auto iter = dictionary_.find(val);
uint64_t dict_val;
if (iter == dictionary_.end()) {
dict_val = dictionary_.size();
dictionary_.insert(std::make_pair(val, dict_val));
dict_vec_.push_back(val);
} else {
dict_val = iter->second;
}
PutVarint64(&tmp_buffer, dict_val);
} else {
tmp_buffer.append(reinterpret_cast<char *>(&val), chunk_size);
}
length += chunk_size;
}
PutVarint64(&buffer_, length);
buffer_.append(tmp_buffer);
return buf - orig_buf;
}
void VariableChunkColBufEncoder::Finish() {
if (col_compression_type_ == kColDict) {
std::string header;
PutVarint64(&header, dict_vec_.size());
for (auto item : dict_vec_) {
PutVarint64(&header, item);
}
buffer_ = header + buffer_;
}
}
} // namespace rocksdb

219
utilities/col_buf_encoder.h
View File

@ -1,219 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <cstdio>
#include <cstring>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "util/coding.h"
namespace rocksdb {
enum ColCompressionType {
kColNoCompression,
kColRle,
kColVarint,
kColRleVarint,
kColDeltaVarint,
kColRleDeltaVarint,
kColDict,
kColRleDict
};
struct ColDeclaration;
// ColBufEncoder is a class to encode column buffers. It can be populated from a
// ColDeclaration. Each time it takes a column value into Append() method to
// encode the column and store it into an internal buffer. After all rows for
// this column are consumed, a Finish() should be called to add header and
// remaining data.
class ColBufEncoder {
public:
// Read a column, encode data and append into internal buffer.
virtual size_t Append(const char *buf) = 0;
virtual ~ColBufEncoder() = 0;
// Get the internal column buffer. Should only be called after Finish().
const std::string &GetData();
// Finish encoding. Add header and remaining data.
virtual void Finish() = 0;
// Populate a ColBufEncoder from ColDeclaration.
static ColBufEncoder *NewColBufEncoder(const ColDeclaration &col_declaration);
protected:
std::string buffer_;
static inline bool IsRunLength(ColCompressionType type) {
return type == kColRle || type == kColRleVarint ||
type == kColRleDeltaVarint || type == kColRleDict;
}
};
// Encoder for fixed length column buffer. In fixed length column buffer, the
// size of the column should not exceed 8 bytes.
// The following encodings are supported:
// Varint: Variable length integer. See util/coding.h for more details
// Rle (Run length encoding): encode a sequence of contiguous value as
// [run_value][run_length]. Can be combined with Varint
// Delta: Encode value to its delta with its adjacent entry. Use varint to
// possibly reduce stored bytes. Can be combined with Rle.
// Dictionary: Use a dictionary to record all possible values in the block and
// encode them with an ID started from 0. IDs are encoded as varint. A column
// with dictionary encoding will have a header to store all actual values,
// ordered by their dictionary value, and the data will be replaced by
// dictionary value. Can be combined with Rle.
class FixedLengthColBufEncoder : public ColBufEncoder {
public:
explicit FixedLengthColBufEncoder(
size_t size, ColCompressionType col_compression_type = kColNoCompression,
bool nullable = false, bool big_endian = false)
: size_(size),
col_compression_type_(col_compression_type),
nullable_(nullable),
big_endian_(big_endian),
last_val_(0),
run_length_(-1),
run_val_(0) {}
size_t Append(const char *buf) override;
void Finish() override;
~FixedLengthColBufEncoder() {}
private:
size_t size_;
ColCompressionType col_compression_type_;
// If set as true, the input value can be null (represented as nullptr). When
// nullable is true, use one more byte before actual value to indicate if the
// current value is null.
bool nullable_;
// If set as true, input value will be treated as big endian encoded.
bool big_endian_;
// for encoding
uint64_t last_val_;
int16_t run_length_;
uint64_t run_val_;
// Map to store dictionary for dictionary encoding
std::unordered_map<uint64_t, uint64_t> dictionary_;
// Vector of dictionary keys.
std::vector<uint64_t> dict_vec_;
};
// Long fixed length column buffer is a variant of fixed length buffer to hold
// fixed length buffer with more than 8 bytes. We do not support any special
// encoding schemes in LongFixedLengthColBufEncoder.
class LongFixedLengthColBufEncoder : public ColBufEncoder {
public:
LongFixedLengthColBufEncoder(size_t size, bool nullable)
: size_(size), nullable_(nullable) {}
size_t Append(const char *buf) override;
void Finish() override;
~LongFixedLengthColBufEncoder() {}
private:
size_t size_;
bool nullable_;
};
// Variable length column buffer holds a format of variable length column. In
// this format, a column is composed of one byte length k, followed by data with
// k bytes long data.
class VariableLengthColBufEncoder : public ColBufEncoder {
public:
size_t Append(const char *buf) override;
void Finish() override;
~VariableLengthColBufEncoder() {}
};
// Variable chunk column buffer holds another format of variable length column.
// In this format, a column contains multiple chunks of data, each of which is
// composed of 8 bytes long data, and one byte as a mask to indicate whether we
// have more data to come. If no more data coming, the mask is set as 0xFF. If
// the chunk is the last chunk and has only k valid bytes, the mask is set as
// 0xFF - (8 - k).
class VariableChunkColBufEncoder : public VariableLengthColBufEncoder {
public:
size_t Append(const char *buf) override;
void Finish() override;
explicit VariableChunkColBufEncoder(ColCompressionType col_compression_type)
: col_compression_type_(col_compression_type) {}
VariableChunkColBufEncoder() : col_compression_type_(kColNoCompression) {}
private:
ColCompressionType col_compression_type_;
// Map to store dictionary for dictionary encoding
std::unordered_map<uint64_t, uint64_t> dictionary_;
// Vector of dictionary keys.
std::vector<uint64_t> dict_vec_;
};
// ColDeclaration declares a column's type, algorithm of column-aware encoding,
// and other column data like endian and nullability.
struct ColDeclaration {
explicit ColDeclaration(
std::string _col_type,
ColCompressionType _col_compression_type = kColNoCompression,
size_t _size = 0, bool _nullable = false, bool _big_endian = false)
: col_type(_col_type),
col_compression_type(_col_compression_type),
size(_size),
nullable(_nullable),
big_endian(_big_endian) {}
std::string col_type;
ColCompressionType col_compression_type;
size_t size;
bool nullable;
bool big_endian;
};
// KVPairColDeclarations is a class to hold column declaration of columns in
// key and value.
struct KVPairColDeclarations {
std::vector<ColDeclaration> *key_col_declarations;
std::vector<ColDeclaration> *value_col_declarations;
ColDeclaration *value_checksum_declaration;
KVPairColDeclarations(std::vector<ColDeclaration> *_key_col_declarations,
std::vector<ColDeclaration> *_value_col_declarations,
ColDeclaration *_value_checksum_declaration)
: key_col_declarations(_key_col_declarations),
value_col_declarations(_value_col_declarations),
value_checksum_declaration(_value_checksum_declaration) {}
};
// Similar to KVPairDeclarations, KVPairColBufEncoders is used to hold column
// buffer encoders of all columns in key and value.
struct KVPairColBufEncoders {
std::vector<std::unique_ptr<ColBufEncoder>> key_col_bufs;
std::vector<std::unique_ptr<ColBufEncoder>> value_col_bufs;
std::unique_ptr<ColBufEncoder> value_checksum_buf;
explicit KVPairColBufEncoders(const KVPairColDeclarations &kvp_cd) {
for (auto kcd : *kvp_cd.key_col_declarations) {
key_col_bufs.emplace_back(
std::move(ColBufEncoder::NewColBufEncoder(kcd)));
}
for (auto vcd : *kvp_cd.value_col_declarations) {
value_col_bufs.emplace_back(
std::move(ColBufEncoder::NewColBufEncoder(vcd)));
}
value_checksum_buf.reset(
ColBufEncoder::NewColBufEncoder(*kvp_cd.value_checksum_declaration));
}
// Helper function to call Finish()
void Finish() {
for (auto &col_buf : key_col_bufs) {
col_buf->Finish();
}
for (auto &col_buf : value_col_bufs) {
col_buf->Finish();
}
value_checksum_buf->Finish();
}
};
} // namespace rocksdb

176
utilities/column_aware_encoding_exp.cc
View File

@ -1,176 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <cstdio>
#include <cstdlib>
#ifndef ROCKSDB_LITE
#ifdef GFLAGS
#include <inttypes.h>
#include <vector>
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "table/block_based_table_builder.h"
#include "table/block_based_table_reader.h"
#include "table/format.h"
#include "tools/sst_dump_tool_imp.h"
#include "util/compression.h"
#include "util/gflags_compat.h"
#include "util/stop_watch.h"
#include "utilities/col_buf_encoder.h"
#include "utilities/column_aware_encoding_util.h"
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
DEFINE_string(encoded_file, "", "file to store encoded data blocks");
DEFINE_string(decoded_file, "",
"file to store decoded data blocks after encoding");
DEFINE_string(format, "col", "Output Format. Can be 'row' or 'col'");
// TODO(jhli): option `col` should be removed and replaced by general
// column specifications.
DEFINE_string(index_type, "col", "Index type. Can be 'primary' or 'secondary'");
DEFINE_string(dump_file, "",
"Dump data blocks separated by columns in human-readable format");
DEFINE_bool(decode, false, "Deocde blocks after they are encoded");
DEFINE_bool(stat, false,
"Print column distribution statistics. Cannot decode in this mode");
DEFINE_string(compression_type, "kNoCompression",
"The compression algorithm used to compress data blocks");
namespace rocksdb {
class ColumnAwareEncodingExp {
public:
static void Run(const std::string& sst_file) {
bool decode = FLAGS_decode;
if (FLAGS_decoded_file.size() > 0) {
decode = true;
}
if (FLAGS_stat) {
decode = false;
}
ColumnAwareEncodingReader reader(sst_file);
std::vector<ColDeclaration>* key_col_declarations;
std::vector<ColDeclaration>* value_col_declarations;
ColDeclaration* value_checksum_declaration;
if (FLAGS_index_type == "primary") {
ColumnAwareEncodingReader::GetColDeclarationsPrimary(
&key_col_declarations, &value_col_declarations,
&value_checksum_declaration);
} else {
ColumnAwareEncodingReader::GetColDeclarationsSecondary(
&key_col_declarations, &value_col_declarations,
&value_checksum_declaration);
}
KVPairColDeclarations kvp_cd(key_col_declarations, value_col_declarations,
value_checksum_declaration);
if (!FLAGS_dump_file.empty()) {
std::vector<KVPairBlock> kv_pair_blocks;
reader.GetKVPairsFromDataBlocks(&kv_pair_blocks);
reader.DumpDataColumns(FLAGS_dump_file, kvp_cd, kv_pair_blocks);
return;
}
std::unordered_map<std::string, CompressionType> compressions = {
{"kNoCompression", CompressionType::kNoCompression},
{"kZlibCompression", CompressionType::kZlibCompression},
{"kZSTD", CompressionType::kZSTD}};
// Find Compression
CompressionType compression_type = compressions[FLAGS_compression_type];
EnvOptions env_options;
if (CompressionTypeSupported(compression_type)) {
fprintf(stdout, "[%s]\n", FLAGS_compression_type.c_str());
std::unique_ptr<WritableFile> encoded_out_file;
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
if (!FLAGS_encoded_file.empty()) {
env->NewWritableFile(FLAGS_encoded_file, &encoded_out_file,
env_options);
}
std::vector<KVPairBlock> kv_pair_blocks;
reader.GetKVPairsFromDataBlocks(&kv_pair_blocks);
std::vector<std::string> encoded_blocks;
StopWatchNano sw(env.get(), true);
if (FLAGS_format == "col") {
reader.EncodeBlocks(kvp_cd, encoded_out_file.get(), compression_type,
kv_pair_blocks, &encoded_blocks, FLAGS_stat);
} else { // row format
reader.EncodeBlocksToRowFormat(encoded_out_file.get(), compression_type,
kv_pair_blocks, &encoded_blocks);
}
if (encoded_out_file != nullptr) {
uint64_t size = 0;
env->GetFileSize(FLAGS_encoded_file, &size);
fprintf(stdout, "File size: %" PRIu64 "\n", size);
}
uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */);
fprintf(stdout, "Encode time: %" PRIu64 "\n", encode_time);
if (decode) {
std::unique_ptr<WritableFile> decoded_out_file;
if (!FLAGS_decoded_file.empty()) {
env->NewWritableFile(FLAGS_decoded_file, &decoded_out_file,
env_options);
}
sw.Start();
if (FLAGS_format == "col") {
reader.DecodeBlocks(kvp_cd, decoded_out_file.get(), &encoded_blocks);
} else {
reader.DecodeBlocksFromRowFormat(decoded_out_file.get(),
&encoded_blocks);
}
uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */);
fprintf(stdout, "Decode time: %" PRIu64 "\n", decode_time);
}
} else {
fprintf(stdout, "Unsupported compression type: %s.\n",
FLAGS_compression_type.c_str());
}
delete key_col_declarations;
delete value_col_declarations;
delete value_checksum_declaration;
}
};
} // namespace rocksdb
int main(int argc, char** argv) {
int arg_idx = ParseCommandLineFlags(&argc, &argv, true);
if (arg_idx >= argc) {
fprintf(stdout, "SST filename required.\n");
exit(1);
}
std::string sst_file(argv[arg_idx]);
if (FLAGS_format != "row" && FLAGS_format != "col") {
fprintf(stderr, "Format must be 'row' or 'col'\n");
exit(1);
}
if (FLAGS_index_type != "primary" && FLAGS_index_type != "secondary") {
fprintf(stderr, "Format must be 'primary' or 'secondary'\n");
exit(1);
}
rocksdb::ColumnAwareEncodingExp::Run(sst_file);
return 0;
}
#else
int main() {
fprintf(stderr, "Please install gflags to run rocksdb tools\n");
return 1;
}
#endif // GFLAGS
#else
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "Not supported in lite mode.\n");
return 1;
}
#endif // ROCKSDB_LITE

254
utilities/column_aware_encoding_test.cc
View File

@ -1,254 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#include <vector>
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/col_buf_decoder.h"
#include "utilities/col_buf_encoder.h"
namespace rocksdb {
class ColumnAwareEncodingTest : public testing::Test {
public:
ColumnAwareEncodingTest() {}
~ColumnAwareEncodingTest() {}
};
class ColumnAwareEncodingTestWithSize
: public ColumnAwareEncodingTest,
public testing::WithParamInterface<size_t> {
public:
ColumnAwareEncodingTestWithSize() {}
~ColumnAwareEncodingTestWithSize() {}
static std::vector<size_t> GetValues() { return {4, 8}; }
};
INSTANTIATE_TEST_CASE_P(
ColumnAwareEncodingTestWithSize, ColumnAwareEncodingTestWithSize,
::testing::ValuesIn(ColumnAwareEncodingTestWithSize::GetValues()));
TEST_P(ColumnAwareEncodingTestWithSize, NoCompressionEncodeDecode) {
size_t col_size = GetParam();
std::unique_ptr<ColBufEncoder> col_buf_encoder(
new FixedLengthColBufEncoder(col_size, kColNoCompression, false, true));
std::string str_buf;
uint64_t base_val = 0x0102030405060708;
uint64_t val = 0;
memcpy(&val, &base_val, col_size);
const int row_count = 4;
for (int i = 0; i < row_count; ++i) {
str_buf.append(reinterpret_cast<char*>(&val), col_size);
}
const char* str_buf_ptr = str_buf.c_str();
for (int i = 0; i < row_count; ++i) {
col_buf_encoder->Append(str_buf_ptr);
}
col_buf_encoder->Finish();
const std::string& encoded_data = col_buf_encoder->GetData();
// Check correctness of encoded string length
ASSERT_EQ(row_count * col_size, encoded_data.size());
const char* encoded_data_ptr = encoded_data.c_str();
uint64_t expected_encoded_val;
if (col_size == 8) {
expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
} else if (col_size == 4) {
expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
}
uint64_t encoded_val = 0;
for (int i = 0; i < row_count; ++i) {
memcpy(&encoded_val, encoded_data_ptr, col_size);
// Check correctness of encoded value
ASSERT_EQ(expected_encoded_val, encoded_val);
encoded_data_ptr += col_size;
}
std::unique_ptr<ColBufDecoder> col_buf_decoder(
new FixedLengthColBufDecoder(col_size, kColNoCompression, false, true));
encoded_data_ptr = encoded_data.c_str();
encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
char* decoded_data = new char[100];
char* decoded_data_base = decoded_data;
for (int i = 0; i < row_count; ++i) {
encoded_data_ptr +=
col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
}
// Check correctness of decoded string length
ASSERT_EQ(row_count * col_size, decoded_data - decoded_data_base);
decoded_data = decoded_data_base;
for (int i = 0; i < row_count; ++i) {
uint64_t decoded_val;
decoded_val = 0;
memcpy(&decoded_val, decoded_data, col_size);
// Check correctness of decoded value
ASSERT_EQ(val, decoded_val);
decoded_data += col_size;
}
delete[] decoded_data_base;
}
TEST_P(ColumnAwareEncodingTestWithSize, RleEncodeDecode) {
size_t col_size = GetParam();
std::unique_ptr<ColBufEncoder> col_buf_encoder(
new FixedLengthColBufEncoder(col_size, kColRle, false, true));
std::string str_buf;
uint64_t base_val = 0x0102030405060708;
uint64_t val = 0;
memcpy(&val, &base_val, col_size);
const int row_count = 4;
for (int i = 0; i < row_count; ++i) {
str_buf.append(reinterpret_cast<char*>(&val), col_size);
}
const char* str_buf_ptr = str_buf.c_str();
for (int i = 0; i < row_count; ++i) {
str_buf_ptr += col_buf_encoder->Append(str_buf_ptr);
}
col_buf_encoder->Finish();
const std::string& encoded_data = col_buf_encoder->GetData();
// Check correctness of encoded string length
ASSERT_EQ(col_size + 1, encoded_data.size());
const char* encoded_data_ptr = encoded_data.c_str();
uint64_t encoded_val = 0;
memcpy(&encoded_val, encoded_data_ptr, col_size);
uint64_t expected_encoded_val;
if (col_size == 8) {
expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
} else if (col_size == 4) {
expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
}
// Check correctness of encoded value
ASSERT_EQ(expected_encoded_val, encoded_val);
std::unique_ptr<ColBufDecoder> col_buf_decoder(
new FixedLengthColBufDecoder(col_size, kColRle, false, true));
char* decoded_data = new char[100];
char* decoded_data_base = decoded_data;
encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
for (int i = 0; i < row_count; ++i) {
encoded_data_ptr +=
col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
}
// Check correctness of decoded string length
ASSERT_EQ(decoded_data - decoded_data_base, row_count * col_size);
decoded_data = decoded_data_base;
for (int i = 0; i < row_count; ++i) {
uint64_t decoded_val;
decoded_val = 0;
memcpy(&decoded_val, decoded_data, col_size);
// Check correctness of decoded value
ASSERT_EQ(val, decoded_val);
decoded_data += col_size;
}
delete[] decoded_data_base;
}
TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
size_t col_size = GetParam();
int row_count = 4;
std::unique_ptr<ColBufEncoder> col_buf_encoder(
new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true));
std::string str_buf;
uint64_t base_val1 = port::kLittleEndian ? 0x0102030405060708 : 0x0807060504030201;
uint64_t base_val2 = port::kLittleEndian ? 0x0202030405060708 : 0x0807060504030202;
uint64_t val1 = 0, val2 = 0;
memcpy(&val1, &base_val1, col_size);
memcpy(&val2, &base_val2, col_size);
const char* str_buf_ptr;
for (int i = 0; i < row_count / 2; ++i) {
str_buf = std::string(reinterpret_cast<char*>(&val1), col_size);
str_buf_ptr = str_buf.c_str();
col_buf_encoder->Append(str_buf_ptr);
str_buf = std::string(reinterpret_cast<char*>(&val2), col_size);
str_buf_ptr = str_buf.c_str();
col_buf_encoder->Append(str_buf_ptr);
}
col_buf_encoder->Finish();
const std::string& encoded_data = col_buf_encoder->GetData();
// Check encoded string length
int varint_len = 0;
if (col_size == 8) {
varint_len = 9;
} else if (col_size == 4) {
varint_len = port::kLittleEndian ? 5 : 9;
}
// Check encoded string length: first value is original one (val - 0), the
// coming three are encoded as 1, -1, 1, so they should take 1 byte in varint.
ASSERT_EQ(varint_len + 3 * 1, encoded_data.size());
std::unique_ptr<ColBufDecoder> col_buf_decoder(
new FixedLengthColBufDecoder(col_size, kColDeltaVarint, false, true));
char* decoded_data = new char[100];
char* decoded_data_base = decoded_data;
const char* encoded_data_ptr = encoded_data.c_str();
encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
for (int i = 0; i < row_count; ++i) {
encoded_data_ptr +=
col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
}
// Check correctness of decoded string length
ASSERT_EQ(row_count * col_size, decoded_data - decoded_data_base);
decoded_data = decoded_data_base;
// Check correctness of decoded data
for (int i = 0; i < row_count / 2; ++i) {
uint64_t decoded_val = 0;
memcpy(&decoded_val, decoded_data, col_size);
ASSERT_EQ(val1, decoded_val);
decoded_data += col_size;
memcpy(&decoded_val, decoded_data, col_size);
ASSERT_EQ(val2, decoded_val);
decoded_data += col_size;
}
delete[] decoded_data_base;
}
TEST_F(ColumnAwareEncodingTest, ChunkBufEncodeDecode) {
std::unique_ptr<ColBufEncoder> col_buf_encoder(
new VariableChunkColBufEncoder(kColDict));
std::string buf("12345678\377\1\0\0\0\0\0\0\0\376", 18);
col_buf_encoder->Append(buf.c_str());
col_buf_encoder->Finish();
const std::string& encoded_data = col_buf_encoder->GetData();
const char* str_ptr = encoded_data.c_str();
std::unique_ptr<ColBufDecoder> col_buf_decoder(
new VariableChunkColBufDecoder(kColDict));
str_ptr += col_buf_decoder->Init(str_ptr);
char* decoded_data = new char[100];
char* decoded_data_base = decoded_data;
col_buf_decoder->Decode(str_ptr, &decoded_data);
for (size_t i = 0; i < buf.size(); ++i) {
ASSERT_EQ(buf[i], decoded_data_base[i]);
}
delete[] decoded_data_base;
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <cstdio>
int main() {
fprintf(stderr,
"SKIPPED as column aware encoding experiment is not enabled in "
"ROCKSDB_LITE\n");
}
#endif // ROCKSDB_LITE

491
utilities/column_aware_encoding_util.cc
View File

@ -1,491 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#include "utilities/column_aware_encoding_util.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <algorithm>
#include <utility>
#include <vector>
#include "include/rocksdb/comparator.h"
#include "include/rocksdb/slice.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "table/block_based_table_builder.h"
#include "table/block_based_table_factory.h"
#include "table/format.h"
#include "table/table_reader.h"
#include "util/cast_util.h"
#include "util/coding.h"
#include "utilities/col_buf_decoder.h"
#include "utilities/col_buf_encoder.h"
#include "port/port.h"
namespace rocksdb {
ColumnAwareEncodingReader::ColumnAwareEncodingReader(
const std::string& file_path)
: file_name_(file_path),
ioptions_(options_),
moptions_(options_),
internal_comparator_(BytewiseComparator()) {
InitTableReader(file_name_);
}
void ColumnAwareEncodingReader::InitTableReader(const std::string& file_path) {
std::unique_ptr<RandomAccessFile> file;
uint64_t file_size;
options_.env->NewRandomAccessFile(file_path, &file, soptions_);
options_.env->GetFileSize(file_path, &file_size);
file_.reset(new RandomAccessFileReader(std::move(file), file_path));
options_.comparator = &internal_comparator_;
options_.table_factory = std::make_shared<BlockBasedTableFactory>();
std::unique_ptr<TableReader> table_reader;
options_.table_factory->NewTableReader(
TableReaderOptions(ioptions_, moptions_.prefix_extractor.get(), soptions_,
internal_comparator_),
std::move(file_), file_size, &table_reader, /*enable_prefetch=*/false);
table_reader_.reset(static_cast_with_check<BlockBasedTable, TableReader>(
table_reader.release()));
}
void ColumnAwareEncodingReader::GetKVPairsFromDataBlocks(
std::vector<KVPairBlock>* kv_pair_blocks) {
table_reader_->GetKVPairsFromDataBlocks(kv_pair_blocks);
}
void ColumnAwareEncodingReader::DecodeBlocks(
const KVPairColDeclarations& kvp_col_declarations, WritableFile* out_file,
const std::vector<std::string>* blocks) {
char* decoded_content_base = new char[16384];
Options options;
ImmutableCFOptions ioptions(options);
for (auto& block : *blocks) {
KVPairColBufDecoders kvp_col_bufs(kvp_col_declarations);
auto& key_col_bufs = kvp_col_bufs.key_col_bufs;
auto& value_col_bufs = kvp_col_bufs.value_col_bufs;
auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf;
auto& slice_final_with_bit = block;
uint32_t format_version = 2;
BlockContents contents;
const char* content_ptr;
CompressionType type =
(CompressionType)slice_final_with_bit[slice_final_with_bit.size() - 1];
if (type != kNoCompression) {
UncompressionContext uncompression_ctx(type);
UncompressBlockContents(uncompression_ctx, slice_final_with_bit.c_str(),
slice_final_with_bit.size() - 1, &contents,
format_version, ioptions);
content_ptr = contents.data.data();
} else {
content_ptr = slice_final_with_bit.data();
}
size_t num_kv_pairs;
const char* header_content_ptr = content_ptr;
num_kv_pairs = static_cast<size_t>(DecodeFixed64(header_content_ptr));
header_content_ptr += sizeof(size_t);
size_t num_key_columns = key_col_bufs.size();
size_t num_value_columns = value_col_bufs.size();
std::vector<const char*> key_content_ptr(num_key_columns);
std::vector<const char*> value_content_ptr(num_value_columns);
const char* checksum_content_ptr;
size_t num_columns = num_key_columns + num_value_columns;
const char* col_content_ptr =
header_content_ptr + sizeof(size_t) * num_columns;
// Read headers
for (size_t i = 0; i < num_key_columns; ++i) {
key_content_ptr[i] = col_content_ptr;
key_content_ptr[i] += key_col_bufs[i]->Init(key_content_ptr[i]);
size_t offset;
offset = static_cast<size_t>(DecodeFixed64(header_content_ptr));
header_content_ptr += sizeof(size_t);
col_content_ptr += offset;
}
for (size_t i = 0; i < num_value_columns; ++i) {
value_content_ptr[i] = col_content_ptr;
value_content_ptr[i] += value_col_bufs[i]->Init(value_content_ptr[i]);
size_t offset;
offset = static_cast<size_t>(DecodeFixed64(header_content_ptr));
header_content_ptr += sizeof(size_t);
col_content_ptr += offset;
}
checksum_content_ptr = col_content_ptr;
checksum_content_ptr += value_checksum_buf->Init(checksum_content_ptr);
// Decode block
char* decoded_content = decoded_content_base;
for (size_t j = 0; j < num_kv_pairs; ++j) {
for (size_t i = 0; i < num_key_columns; ++i) {
key_content_ptr[i] +=
key_col_bufs[i]->Decode(key_content_ptr[i], &decoded_content);
}
for (size_t i = 0; i < num_value_columns; ++i) {
value_content_ptr[i] +=
value_col_bufs[i]->Decode(value_content_ptr[i], &decoded_content);
}
checksum_content_ptr +=
value_checksum_buf->Decode(checksum_content_ptr, &decoded_content);
}
size_t offset = decoded_content - decoded_content_base;
Slice output_content(decoded_content, offset);
if (out_file != nullptr) {
out_file->Append(output_content);
}
}
delete[] decoded_content_base;
}
void ColumnAwareEncodingReader::DecodeBlocksFromRowFormat(
WritableFile* out_file, const std::vector<std::string>* blocks) {
Options options;
ImmutableCFOptions ioptions(options);
for (auto& block : *blocks) {
auto& slice_final_with_bit = block;
uint32_t format_version = 2;
BlockContents contents;
std::string decoded_content;
CompressionType type =
(CompressionType)slice_final_with_bit[slice_final_with_bit.size() - 1];
if (type != kNoCompression) {
UncompressionContext uncompression_ctx(type);
UncompressBlockContents(uncompression_ctx, slice_final_with_bit.c_str(),
slice_final_with_bit.size() - 1, &contents,
format_version, ioptions);
decoded_content = std::string(contents.data.data(), contents.data.size());
} else {
decoded_content = std::move(slice_final_with_bit);
}
if (out_file != nullptr) {
out_file->Append(decoded_content);
}
}
}
void ColumnAwareEncodingReader::DumpDataColumns(
const std::string& filename,
const KVPairColDeclarations& kvp_col_declarations,
const std::vector<KVPairBlock>& kv_pair_blocks) {
KVPairColBufEncoders kvp_col_bufs(kvp_col_declarations);
auto& key_col_bufs = kvp_col_bufs.key_col_bufs;
auto& value_col_bufs = kvp_col_bufs.value_col_bufs;
auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf;
FILE* fp = fopen(filename.c_str(), "w");
size_t block_id = 1;
for (auto& kv_pairs : kv_pair_blocks) {
fprintf(fp, "---------------- Block: %-4" ROCKSDB_PRIszt " ----------------\n", block_id);
for (auto& kv_pair : kv_pairs) {
const auto& key = kv_pair.first;
const auto& value = kv_pair.second;
size_t value_offset = 0;
const char* key_ptr = key.data();
for (auto& buf : key_col_bufs) {
size_t col_size = buf->Append(key_ptr);
std::string tmp_buf(key_ptr, col_size);
Slice col(tmp_buf);
fprintf(fp, "%s ", col.ToString(true).c_str());
key_ptr += col_size;
}
fprintf(fp, "|");
const char* value_ptr = value.data();
for (auto& buf : value_col_bufs) {
size_t col_size = buf->Append(value_ptr);
std::string tmp_buf(value_ptr, col_size);
Slice col(tmp_buf);
fprintf(fp, " %s", col.ToString(true).c_str());
value_ptr += col_size;
value_offset += col_size;
}
if (value_offset < value.size()) {
size_t col_size = value_checksum_buf->Append(value_ptr);
std::string tmp_buf(value_ptr, col_size);
Slice col(tmp_buf);
fprintf(fp, "|%s", col.ToString(true).c_str());
} else {
value_checksum_buf->Append(nullptr);
}
fprintf(fp, "\n");
}
block_id++;
}
fclose(fp);
}
namespace {
void CompressDataBlock(const std::string& output_content, Slice* slice_final,
CompressionType* type, std::string* compressed_output) {
CompressionContext compression_ctx(*type);
uint32_t format_version = 2; // hard-coded version
*slice_final = CompressBlock(output_content, compression_ctx, type,
format_version, compressed_output);
}
} // namespace
void ColumnAwareEncodingReader::EncodeBlocksToRowFormat(
WritableFile* out_file, CompressionType compression_type,
const std::vector<KVPairBlock>& kv_pair_blocks,
std::vector<std::string>* blocks) {
std::string output_content;
for (auto& kv_pairs : kv_pair_blocks) {
output_content.clear();
std::string last_key;
size_t counter = 0;
const size_t block_restart_interval = 16;
for (auto& kv_pair : kv_pairs) {
const auto& key = kv_pair.first;
const auto& value = kv_pair.second;
Slice last_key_piece(last_key);
size_t shared = 0;
if (counter >= block_restart_interval) {
counter = 0;
} else {
const size_t min_length = std::min(last_key_piece.size(), key.size());
while ((shared < min_length) && last_key_piece[shared] == key[shared]) {
shared++;
}
}
const size_t non_shared = key.size() - shared;
output_content.append(key.c_str() + shared, non_shared);
output_content.append(value);
last_key.resize(shared);
last_key.append(key.data() + shared, non_shared);
counter++;
}
Slice slice_final;
auto type = compression_type;
std::string compressed_output;
CompressDataBlock(output_content, &slice_final, &type, &compressed_output);
if (out_file != nullptr) {
out_file->Append(slice_final);
}
// Add a bit in the end for decoding
std::string slice_final_with_bit(slice_final.data(), slice_final.size());
slice_final_with_bit.append(reinterpret_cast<char*>(&type), 1);
blocks->push_back(
std::string(slice_final_with_bit.data(), slice_final_with_bit.size()));
}
}
Status ColumnAwareEncodingReader::EncodeBlocks(
const KVPairColDeclarations& kvp_col_declarations, WritableFile* out_file,
CompressionType compression_type,
const std::vector<KVPairBlock>& kv_pair_blocks,
std::vector<std::string>* blocks, bool print_column_stat) {
std::vector<size_t> key_col_sizes(
kvp_col_declarations.key_col_declarations->size(), 0);
std::vector<size_t> value_col_sizes(
kvp_col_declarations.value_col_declarations->size(), 0);
size_t value_checksum_size = 0;
for (auto& kv_pairs : kv_pair_blocks) {
KVPairColBufEncoders kvp_col_bufs(kvp_col_declarations);
auto& key_col_bufs = kvp_col_bufs.key_col_bufs;
auto& value_col_bufs = kvp_col_bufs.value_col_bufs;
auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf;
size_t num_kv_pairs = 0;
for (auto& kv_pair : kv_pairs) {
const auto& key = kv_pair.first;
const auto& value = kv_pair.second;
size_t value_offset = 0;
num_kv_pairs++;
const char* key_ptr = key.data();
for (auto& buf : key_col_bufs) {
size_t col_size = buf->Append(key_ptr);
key_ptr += col_size;
}
const char* value_ptr = value.data();
for (auto& buf : value_col_bufs) {
size_t col_size = buf->Append(value_ptr);
value_ptr += col_size;
value_offset += col_size;
}
if (value_offset < value.size()) {
value_checksum_buf->Append(value_ptr);
} else {
value_checksum_buf->Append(nullptr);
}
}
kvp_col_bufs.Finish();
// Get stats
// Compress and write a block
if (print_column_stat) {
for (size_t i = 0; i < key_col_bufs.size(); ++i) {
Slice slice_final;
auto type = compression_type;
std::string compressed_output;
CompressDataBlock(key_col_bufs[i]->GetData(), &slice_final, &type,
&compressed_output);
out_file->Append(slice_final);
key_col_sizes[i] += slice_final.size();
}
for (size_t i = 0; i < value_col_bufs.size(); ++i) {
Slice slice_final;
auto type = compression_type;
std::string compressed_output;
CompressDataBlock(value_col_bufs[i]->GetData(), &slice_final, &type,
&compressed_output);
out_file->Append(slice_final);
value_col_sizes[i] += slice_final.size();
}
Slice slice_final;
auto type = compression_type;
std::string compressed_output;
CompressDataBlock(value_checksum_buf->GetData(), &slice_final, &type,
&compressed_output);
out_file->Append(slice_final);
value_checksum_size += slice_final.size();
} else {
std::string output_content;
// Write column sizes
PutFixed64(&output_content, num_kv_pairs);
for (auto& buf : key_col_bufs) {
size_t size = buf->GetData().size();
PutFixed64(&output_content, size);
}
for (auto& buf : value_col_bufs) {
size_t size = buf->GetData().size();
PutFixed64(&output_content, size);
}
// Write data
for (auto& buf : key_col_bufs) {
output_content.append(buf->GetData());
}
for (auto& buf : value_col_bufs) {
output_content.append(buf->GetData());
}
output_content.append(value_checksum_buf->GetData());
Slice slice_final;
auto type = compression_type;
std::string compressed_output;
CompressDataBlock(output_content, &slice_final, &type,
&compressed_output);
if (out_file != nullptr) {
out_file->Append(slice_final);
}
// Add a bit in the end for decoding
std::string slice_final_with_bit(slice_final.data(),
slice_final.size() + 1);
slice_final_with_bit[slice_final.size()] = static_cast<char>(type);
blocks->push_back(std::string(slice_final_with_bit.data(),
slice_final_with_bit.size()));
}
}
if (print_column_stat) {
size_t total_size = 0;
for (size_t i = 0; i < key_col_sizes.size(); ++i)
total_size += key_col_sizes[i];
for (size_t i = 0; i < value_col_sizes.size(); ++i)
total_size += value_col_sizes[i];
total_size += value_checksum_size;
for (size_t i = 0; i < key_col_sizes.size(); ++i)
printf("Key col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i, key_col_sizes[i],
100.0 * key_col_sizes[i] / total_size);
for (size_t i = 0; i < value_col_sizes.size(); ++i)
printf("Value col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i,
value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size);
printf("Value checksum size: %" ROCKSDB_PRIszt " percentage %lf%%\n", value_checksum_size,
100.0 * value_checksum_size / total_size);
}
return Status::OK();
}
void ColumnAwareEncodingReader::GetColDeclarationsPrimary(
std::vector<ColDeclaration>** key_col_declarations,
std::vector<ColDeclaration>** value_col_declarations,
ColDeclaration** value_checksum_declaration) {
*key_col_declarations = new std::vector<ColDeclaration>{
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4, false,
true),
ColDeclaration("FixedLength", ColCompressionType::kColRleDeltaVarint, 8,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)};
*value_col_declarations = new std::vector<ColDeclaration>{
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4),
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4),
ColDeclaration("FixedLength", ColCompressionType::kColRle, 1),
ColDeclaration("VariableLength"),
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 4),
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)};
*value_checksum_declaration = new ColDeclaration(
"LongFixedLength", ColCompressionType::kColNoCompression, 9,
true /* nullable */);
}
void ColumnAwareEncodingReader::GetColDeclarationsSecondary(
std::vector<ColDeclaration>** key_col_declarations,
std::vector<ColDeclaration>** value_col_declarations,
ColDeclaration** value_checksum_declaration) {
*key_col_declarations = new std::vector<ColDeclaration>{
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4, false,
true),
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColRleDeltaVarint, 8,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColRle, 1),
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 4,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
false, true),
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8, false,
true),
ColDeclaration("VariableChunk", ColCompressionType::kColNoCompression),
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)};
*value_col_declarations = new std::vector<ColDeclaration>();
*value_checksum_declaration = new ColDeclaration(
"LongFixedLength", ColCompressionType::kColNoCompression, 9,
true /* nullable */);
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

81
utilities/column_aware_encoding_util.h
View File

@ -1,81 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include <string>
#include <vector>
#include "db/dbformat.h"
#include "include/rocksdb/env.h"
#include "include/rocksdb/listener.h"
#include "include/rocksdb/options.h"
#include "include/rocksdb/status.h"
#include "options/cf_options.h"
#include "table/block_based_table_reader.h"
namespace rocksdb {
struct ColDeclaration;
struct KVPairColDeclarations;
class ColumnAwareEncodingReader {
public:
explicit ColumnAwareEncodingReader(const std::string& file_name);
void GetKVPairsFromDataBlocks(std::vector<KVPairBlock>* kv_pair_blocks);
void EncodeBlocksToRowFormat(WritableFile* out_file,
CompressionType compression_type,
const std::vector<KVPairBlock>& kv_pair_blocks,
std::vector<std::string>* blocks);
void DecodeBlocksFromRowFormat(WritableFile* out_file,
const std::vector<std::string>* blocks);
void DumpDataColumns(const std::string& filename,
const KVPairColDeclarations& kvp_col_declarations,
const std::vector<KVPairBlock>& kv_pair_blocks);
Status EncodeBlocks(const KVPairColDeclarations& kvp_col_declarations,
WritableFile* out_file, CompressionType compression_type,
const std::vector<KVPairBlock>& kv_pair_blocks,
std::vector<std::string>* blocks, bool print_column_stat);
void DecodeBlocks(const KVPairColDeclarations& kvp_col_declarations,
WritableFile* out_file,
const std::vector<std::string>* blocks);
static void GetColDeclarationsPrimary(
std::vector<ColDeclaration>** key_col_declarations,
std::vector<ColDeclaration>** value_col_declarations,
ColDeclaration** value_checksum_declaration);
static void GetColDeclarationsSecondary(
std::vector<ColDeclaration>** key_col_declarations,
std::vector<ColDeclaration>** value_col_declarations,
ColDeclaration** value_checksum_declaration);
private:
// Init the TableReader for the sst file
void InitTableReader(const std::string& file_path);
std::string file_name_;
EnvOptions soptions_;
Options options_;
Status init_result_;
std::unique_ptr<BlockBasedTable> table_reader_;
std::unique_ptr<RandomAccessFileReader> file_;
const ImmutableCFOptions ioptions_;
const MutableCFOptions moptions_;
InternalKeyComparator internal_comparator_;
std::unique_ptr<TableProperties> table_properties_;
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

399
utilities/date_tiered/date_tiered_db_impl.cc
View File

@ -1,399 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "utilities/date_tiered/date_tiered_db_impl.h"
#include <limits>
#include "db/db_impl.h"
#include "db/db_iter.h"
#include "db/write_batch_internal.h"
#include "monitoring/instrumented_mutex.h"
#include "options/options_helper.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/utilities/date_tiered_db.h"
#include "table/merging_iterator.h"
#include "util/coding.h"
#include "util/filename.h"
#include "util/string_util.h"
namespace rocksdb {
// Open the db inside DateTieredDBImpl because options needs pointer to its ttl
DateTieredDBImpl::DateTieredDBImpl(
DB* db, Options options,
const std::vector<ColumnFamilyDescriptor>& descriptors,
const std::vector<ColumnFamilyHandle*>& handles, int64_t ttl,
int64_t column_family_interval)
: db_(db),
cf_options_(ColumnFamilyOptions(options)),
ioptions_(ImmutableCFOptions(options)),
moptions_(MutableCFOptions(options)),
icomp_(cf_options_.comparator),
ttl_(ttl),
column_family_interval_(column_family_interval),
mutex_(options.statistics.get(), db->GetEnv(), DB_MUTEX_WAIT_MICROS,
options.use_adaptive_mutex) {
latest_timebound_ = std::numeric_limits<int64_t>::min();
for (size_t i = 0; i < handles.size(); ++i) {
const auto& name = descriptors[i].name;
int64_t timestamp = 0;
try {
timestamp = ParseUint64(name);
} catch (const std::invalid_argument&) {
// Bypass unrelated column family, e.g. default
db_->DestroyColumnFamilyHandle(handles[i]);
continue;
}
if (timestamp > latest_timebound_) {
latest_timebound_ = timestamp;
}
handle_map_.insert(std::make_pair(timestamp, handles[i]));
}
}
DateTieredDBImpl::~DateTieredDBImpl() {
for (auto handle : handle_map_) {
db_->DestroyColumnFamilyHandle(handle.second);
}
delete db_;
db_ = nullptr;
}
Status DateTieredDB::Open(const Options& options, const std::string& dbname,
DateTieredDB** dbptr, int64_t ttl,
int64_t column_family_interval, bool read_only) {
DBOptions db_options(options);
ColumnFamilyOptions cf_options(options);
std::vector<ColumnFamilyDescriptor> descriptors;
std::vector<ColumnFamilyHandle*> handles;
DB* db;
Status s;
// Get column families
std::vector<std::string> column_family_names;
s = DB::ListColumnFamilies(db_options, dbname, &column_family_names);
if (!s.ok()) {
// No column family found. Use default
s = DB::Open(options, dbname, &db);
if (!s.ok()) {
return s;
}
} else {
for (auto name : column_family_names) {
descriptors.emplace_back(ColumnFamilyDescriptor(name, cf_options));
}
// Open database
if (read_only) {
s = DB::OpenForReadOnly(db_options, dbname, descriptors, &handles, &db);
} else {
s = DB::Open(db_options, dbname, descriptors, &handles, &db);
}
}
if (s.ok()) {
*dbptr = new DateTieredDBImpl(db, options, descriptors, handles, ttl,
column_family_interval);
}
return s;
}
// Checks if the string is stale or not according to TTl provided
bool DateTieredDBImpl::IsStale(int64_t keytime, int64_t ttl, Env* env) {
if (ttl <= 0) {
// Data is fresh if TTL is non-positive
return false;
}
int64_t curtime;
if (!env->GetCurrentTime(&curtime).ok()) {
// Treat the data as fresh if could not get current time
return false;
}
return curtime >= keytime + ttl;
}
// Drop column family when all data in that column family is expired
// TODO(jhli): Can be made a background job
Status DateTieredDBImpl::DropObsoleteColumnFamilies() {
int64_t curtime;
Status s;
s = db_->GetEnv()->GetCurrentTime(&curtime);
if (!s.ok()) {
return s;
}
{
InstrumentedMutexLock l(&mutex_);
auto iter = handle_map_.begin();
while (iter != handle_map_.end()) {
if (iter->first <= curtime - ttl_) {
s = db_->DropColumnFamily(iter->second);
if (!s.ok()) {
return s;
}
delete iter->second;
iter = handle_map_.erase(iter);
} else {
break;
}
}
}
return Status::OK();
}
// Get timestamp from user key
Status DateTieredDBImpl::GetTimestamp(const Slice& key, int64_t* result) {
if (key.size() < kTSLength) {
return Status::Corruption("Bad timestamp in key");
}
const char* pos = key.data() + key.size() - 8;
int64_t timestamp = 0;
if (port::kLittleEndian) {
int bytes_to_fill = 8;
for (int i = 0; i < bytes_to_fill; ++i) {
timestamp |= (static_cast<uint64_t>(static_cast<unsigned char>(pos[i]))
<< ((bytes_to_fill - i - 1) << 3));
}
} else {
memcpy(&timestamp, pos, sizeof(timestamp));
}
*result = timestamp;
return Status::OK();
}
Status DateTieredDBImpl::CreateColumnFamily(
ColumnFamilyHandle** column_family) {
int64_t curtime;
Status s;
mutex_.AssertHeld();
s = db_->GetEnv()->GetCurrentTime(&curtime);
if (!s.ok()) {
return s;
}
int64_t new_timebound;
if (handle_map_.empty()) {
new_timebound = curtime + column_family_interval_;
} else {
new_timebound =
latest_timebound_ +
((curtime - latest_timebound_) / column_family_interval_ + 1) *
column_family_interval_;
}
std::string cf_name = ToString(new_timebound);
latest_timebound_ = new_timebound;
s = db_->CreateColumnFamily(cf_options_, cf_name, column_family);
if (s.ok()) {
handle_map_.insert(std::make_pair(new_timebound, *column_family));
}
return s;
}
Status DateTieredDBImpl::FindColumnFamily(int64_t keytime,
ColumnFamilyHandle** column_family,
bool create_if_missing) {
*column_family = nullptr;
{
InstrumentedMutexLock l(&mutex_);
auto iter = handle_map_.upper_bound(keytime);
if (iter == handle_map_.end()) {
if (!create_if_missing) {
return Status::NotFound();
} else {
return CreateColumnFamily(column_family);
}
}
// Move to previous element to get the appropriate time window
*column_family = iter->second;
}
return Status::OK();
}
Status DateTieredDBImpl::Put(const WriteOptions& options, const Slice& key,
const Slice& val) {
int64_t timestamp = 0;
Status s;
s = GetTimestamp(key, &timestamp);
if (!s.ok()) {
return s;
}
DropObsoleteColumnFamilies();
// Prune request to obsolete data
if (IsStale(timestamp, ttl_, db_->GetEnv())) {
return Status::InvalidArgument();
}
// Decide column family (i.e. the time window) to put into
ColumnFamilyHandle* column_family;
s = FindColumnFamily(timestamp, &column_family, true /*create_if_missing*/);
if (!s.ok()) {
return s;
}
// Efficiently put with WriteBatch
WriteBatch batch;
batch.Put(column_family, key, val);
return Write(options, &batch);
}
Status DateTieredDBImpl::Get(const ReadOptions& options, const Slice& key,
std::string* value) {
int64_t timestamp = 0;
Status s;
s = GetTimestamp(key, &timestamp);
if (!s.ok()) {
return s;
}
// Prune request to obsolete data
if (IsStale(timestamp, ttl_, db_->GetEnv())) {
return Status::NotFound();
}
// Decide column family to get from
ColumnFamilyHandle* column_family;
s = FindColumnFamily(timestamp, &column_family, false /*create_if_missing*/);
if (!s.ok()) {
return s;
}
if (column_family == nullptr) {
// Cannot find column family
return Status::NotFound();
}
// Get value with key
return db_->Get(options, column_family, key, value);
}
bool DateTieredDBImpl::KeyMayExist(const ReadOptions& options, const Slice& key,
std::string* value, bool* value_found) {
int64_t timestamp = 0;
Status s;
s = GetTimestamp(key, &timestamp);
if (!s.ok()) {
// Cannot get current time
return false;
}
// Decide column family to get from
ColumnFamilyHandle* column_family;
s = FindColumnFamily(timestamp, &column_family, false /*create_if_missing*/);
if (!s.ok() || column_family == nullptr) {
// Cannot find column family
return false;
}
if (IsStale(timestamp, ttl_, db_->GetEnv())) {
return false;
}
return db_->KeyMayExist(options, column_family, key, value, value_found);
}
Status DateTieredDBImpl::Delete(const WriteOptions& options, const Slice& key) {
int64_t timestamp = 0;
Status s;
s = GetTimestamp(key, &timestamp);
if (!s.ok()) {
return s;
}
DropObsoleteColumnFamilies();
// Prune request to obsolete data
if (IsStale(timestamp, ttl_, db_->GetEnv())) {
return Status::NotFound();
}
// Decide column family to get from
ColumnFamilyHandle* column_family;
s = FindColumnFamily(timestamp, &column_family, false /*create_if_missing*/);
if (!s.ok()) {
return s;
}
if (column_family == nullptr) {
// Cannot find column family
return Status::NotFound();
}
// Get value with key
return db_->Delete(options, column_family, key);
}
Status DateTieredDBImpl::Merge(const WriteOptions& options, const Slice& key,
const Slice& value) {
// Decide column family to get from
int64_t timestamp = 0;
Status s;
s = GetTimestamp(key, &timestamp);
if (!s.ok()) {
// Cannot get current time
return s;
}
ColumnFamilyHandle* column_family;
s = FindColumnFamily(timestamp, &column_family, true /*create_if_missing*/);
if (!s.ok()) {
return s;
}
WriteBatch batch;
batch.Merge(column_family, key, value);
return Write(options, &batch);
}
Status DateTieredDBImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
class Handler : public WriteBatch::Handler {
public:
explicit Handler() {}
WriteBatch updates_ttl;
Status batch_rewrite_status;
virtual Status PutCF(uint32_t column_family_id, const Slice& key,
const Slice& value) override {
WriteBatchInternal::Put(&updates_ttl, column_family_id, key, value);
return Status::OK();
}
virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
const Slice& value) override {
WriteBatchInternal::Merge(&updates_ttl, column_family_id, key, value);
return Status::OK();
}
virtual Status DeleteCF(uint32_t column_family_id,
const Slice& key) override {
WriteBatchInternal::Delete(&updates_ttl, column_family_id, key);
return Status::OK();
}
virtual void LogData(const Slice& blob) override {
updates_ttl.PutLogData(blob);
}
};
Handler handler;
updates->Iterate(&handler);
if (!handler.batch_rewrite_status.ok()) {
return handler.batch_rewrite_status;
} else {
return db_->Write(opts, &(handler.updates_ttl));
}
}
Iterator* DateTieredDBImpl::NewIterator(const ReadOptions& opts) {
if (handle_map_.empty()) {
return NewEmptyIterator();
}
DBImpl* db_impl = reinterpret_cast<DBImpl*>(db_);
auto db_iter = NewArenaWrappedDbIterator(
db_impl->GetEnv(), opts, ioptions_, moptions_, kMaxSequenceNumber,
cf_options_.max_sequential_skip_in_iterations, 0,
nullptr /*read_callback*/);
auto arena = db_iter->GetArena();
MergeIteratorBuilder builder(&icomp_, arena);
for (auto& item : handle_map_) {
auto handle = item.second;
builder.AddIterator(db_impl->NewInternalIterator(
arena, db_iter->GetRangeDelAggregator(), kMaxSequenceNumber, handle));
}
auto internal_iter = builder.Finish();
db_iter->SetIterUnderDBIter(internal_iter);
return db_iter;
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

93
utilities/date_tiered/date_tiered_db_impl.h
View File

@ -1,93 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include <map>
#include <string>
#include <vector>
#include "monitoring/instrumented_mutex.h"
#include "options/cf_options.h"
#include "rocksdb/db.h"
#include "rocksdb/utilities/date_tiered_db.h"
namespace rocksdb {
// Implementation of DateTieredDB.
class DateTieredDBImpl : public DateTieredDB {
public:
DateTieredDBImpl(DB* db, Options options,
const std::vector<ColumnFamilyDescriptor>& descriptors,
const std::vector<ColumnFamilyHandle*>& handles, int64_t ttl,
int64_t column_family_interval);
virtual ~DateTieredDBImpl();
Status Put(const WriteOptions& options, const Slice& key,
const Slice& val) override;
Status Get(const ReadOptions& options, const Slice& key,
std::string* value) override;
Status Delete(const WriteOptions& options, const Slice& key) override;
bool KeyMayExist(const ReadOptions& options, const Slice& key,
std::string* value, bool* value_found = nullptr) override;
Status Merge(const WriteOptions& options, const Slice& key,
const Slice& value) override;
Iterator* NewIterator(const ReadOptions& opts) override;
Status DropObsoleteColumnFamilies() override;
// Extract timestamp from key.
static Status GetTimestamp(const Slice& key, int64_t* result);
private:
// Base database object
DB* db_;
const ColumnFamilyOptions cf_options_;
const ImmutableCFOptions ioptions_;
const MutableCFOptions moptions_;
const InternalKeyComparator icomp_;
// Storing all column family handles for time series data.
std::vector<ColumnFamilyHandle*> handles_;
// Manages a mapping from a column family's maximum timestamp to its handle.
std::map<int64_t, ColumnFamilyHandle*> handle_map_;
// A time-to-live value to indicate when the data should be removed.
int64_t ttl_;
// An variable to indicate the time range of a column family.
int64_t column_family_interval_;
// Indicate largest maximum timestamp of a column family.
int64_t latest_timebound_;
// Mutex to protect handle_map_ operations.
InstrumentedMutex mutex_;
// Internal method to execute Put and Merge in batch.
Status Write(const WriteOptions& opts, WriteBatch* updates);
Status CreateColumnFamily(ColumnFamilyHandle** column_family);
Status FindColumnFamily(int64_t keytime, ColumnFamilyHandle** column_family,
bool create_if_missing);
static bool IsStale(int64_t keytime, int64_t ttl, Env* env);
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

469
utilities/date_tiered/date_tiered_test.cc
View File

@ -1,469 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#ifndef OS_WIN
#include <unistd.h>
#endif
#include <map>
#include <memory>
#include "rocksdb/compaction_filter.h"
#include "rocksdb/utilities/date_tiered_db.h"
#include "port/port.h"
#include "util/logging.h"
#include "util/string_util.h"
#include "util/testharness.h"
namespace rocksdb {
namespace {
typedef std::map<std::string, std::string> KVMap;
}
class SpecialTimeEnv : public EnvWrapper {
public:
explicit SpecialTimeEnv(Env* base) : EnvWrapper(base) {
base->GetCurrentTime(&current_time_);
}
void Sleep(int64_t sleep_time) { current_time_ += sleep_time; }
virtual Status GetCurrentTime(int64_t* current_time) override {
*current_time = current_time_;
return Status::OK();
}
private:
int64_t current_time_ = 0;
};
class DateTieredTest : public testing::Test {
public:
DateTieredTest() {
env_.reset(new SpecialTimeEnv(Env::Default()));
dbname_ = test::PerThreadDBPath("date_tiered");
options_.create_if_missing = true;
options_.env = env_.get();
date_tiered_db_.reset(nullptr);
DestroyDB(dbname_, Options());
}
~DateTieredTest() {
CloseDateTieredDB();
DestroyDB(dbname_, Options());
}
void OpenDateTieredDB(int64_t ttl, int64_t column_family_interval,
bool read_only = false) {
ASSERT_TRUE(date_tiered_db_.get() == nullptr);
DateTieredDB* date_tiered_db = nullptr;
ASSERT_OK(DateTieredDB::Open(options_, dbname_, &date_tiered_db, ttl,
column_family_interval, read_only));
date_tiered_db_.reset(date_tiered_db);
}
void CloseDateTieredDB() { date_tiered_db_.reset(nullptr); }
Status AppendTimestamp(std::string* key) {
char ts[8];
int bytes_to_fill = 8;
int64_t timestamp_value = 0;
Status s = env_->GetCurrentTime(&timestamp_value);
if (!s.ok()) {
return s;
}
if (port::kLittleEndian) {
for (int i = 0; i < bytes_to_fill; ++i) {
ts[i] = (timestamp_value >> ((bytes_to_fill - i - 1) << 3)) & 0xFF;
}
} else {
memcpy(ts, static_cast<void*>(&timestamp_value), bytes_to_fill);
}
key->append(ts, 8);
return Status::OK();
}
// Populates and returns a kv-map
void MakeKVMap(int64_t num_entries, KVMap* kvmap) {
kvmap->clear();
int digits = 1;
for (int64_t dummy = num_entries; dummy /= 10; ++digits) {
}
int digits_in_i = 1;
for (int64_t i = 0; i < num_entries; i++) {
std::string key = "key";
std::string value = "value";
if (i % 10 == 0) {
digits_in_i++;
}
for (int j = digits_in_i; j < digits; j++) {
key.append("0");
value.append("0");
}
AppendNumberTo(&key, i);
AppendNumberTo(&value, i);
ASSERT_OK(AppendTimestamp(&key));
(*kvmap)[key] = value;
}
// check all insertions done
ASSERT_EQ(num_entries, static_cast<int64_t>(kvmap->size()));
}
size_t GetColumnFamilyCount() {
DBOptions db_options(options_);
std::vector<std::string> cf;
DB::ListColumnFamilies(db_options, dbname_, &cf);
return cf.size();
}
void Sleep(int64_t sleep_time) { env_->Sleep(sleep_time); }
static const int64_t kSampleSize_ = 100;
std::string dbname_;
std::unique_ptr<DateTieredDB> date_tiered_db_;
std::unique_ptr<SpecialTimeEnv> env_;
KVMap kvmap_;
private:
Options options_;
KVMap::iterator kv_it_;
const std::string kNewValue_ = "new_value";
std::unique_ptr<CompactionFilter> test_comp_filter_;
};
// Puts a set of values and checks its presence using Get during ttl
TEST_F(DateTieredTest, KeyLifeCycle) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(2, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
// Put data in database
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
Sleep(1);
// T=1, keys should still reside in database
for (auto& kv : map_insert) {
std::string value;
ASSERT_OK(date_tiered_db_->Get(ropts, kv.first, &value));
ASSERT_EQ(value, kv.second);
}
Sleep(1);
// T=2, keys should not be retrieved
for (auto& kv : map_insert) {
std::string value;
auto s = date_tiered_db_->Get(ropts, kv.first, &value);
ASSERT_TRUE(s.IsNotFound());
}
CloseDateTieredDB();
}
TEST_F(DateTieredTest, DeleteTest) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(2, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
// Put data in database
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
Sleep(1);
// Delete keys when they are not obsolete
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Delete(wopts, kv.first));
}
// Key should not be found
for (auto& kv : map_insert) {
std::string value;
auto s = date_tiered_db_->Get(ropts, kv.first, &value);
ASSERT_TRUE(s.IsNotFound());
}
}
TEST_F(DateTieredTest, KeyMayExistTest) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(2, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
// Put data in database
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
Sleep(1);
// T=1, keys should still reside in database
for (auto& kv : map_insert) {
std::string value;
ASSERT_TRUE(date_tiered_db_->KeyMayExist(ropts, kv.first, &value));
ASSERT_EQ(value, kv.second);
}
}
// Database open and close should not affect
TEST_F(DateTieredTest, MultiOpen) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(4, 4);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
// Put data in database
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
CloseDateTieredDB();
Sleep(1);
OpenDateTieredDB(2, 2);
// T=1, keys should still reside in database
for (auto& kv : map_insert) {
std::string value;
ASSERT_OK(date_tiered_db_->Get(ropts, kv.first, &value));
ASSERT_EQ(value, kv.second);
}
Sleep(1);
// T=2, keys should not be retrieved
for (auto& kv : map_insert) {
std::string value;
auto s = date_tiered_db_->Get(ropts, kv.first, &value);
ASSERT_TRUE(s.IsNotFound());
}
CloseDateTieredDB();
}
// If the key in Put() is obsolete, the data should not be written into database
TEST_F(DateTieredTest, InsertObsoleteDate) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(2, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
Sleep(2);
// T=2, keys put into database are already obsolete
// Put data in database. Operations should not return OK
for (auto& kv : map_insert) {
auto s = date_tiered_db_->Put(wopts, kv.first, kv.second);
ASSERT_TRUE(s.IsInvalidArgument());
}
// Data should not be found in database
for (auto& kv : map_insert) {
std::string value;
auto s = date_tiered_db_->Get(ropts, kv.first, &value);
ASSERT_TRUE(s.IsNotFound());
}
CloseDateTieredDB();
}
// Resets the timestamp of a set of kvs by updating them and checks that they
// are not deleted according to the old timestamp
TEST_F(DateTieredTest, ColumnFamilyCounts) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(4, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Only default column family
ASSERT_EQ(1, GetColumnFamilyCount());
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
// A time series column family is created
ASSERT_EQ(2, GetColumnFamilyCount());
Sleep(2);
KVMap map_insert2;
MakeKVMap(kSampleSize_, &map_insert2);
for (auto& kv : map_insert2) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
// Another time series column family is created
ASSERT_EQ(3, GetColumnFamilyCount());
Sleep(4);
// Data should not be found in database
for (auto& kv : map_insert) {
std::string value;
auto s = date_tiered_db_->Get(ropts, kv.first, &value);
ASSERT_TRUE(s.IsNotFound());
}
// Explicitly drop obsolete column families
date_tiered_db_->DropObsoleteColumnFamilies();
// The first column family is deleted from database
ASSERT_EQ(2, GetColumnFamilyCount());
CloseDateTieredDB();
}
// Puts a set of values and checks its presence using iterator during ttl
TEST_F(DateTieredTest, IteratorLifeCycle) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(2, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
// Create key value pairs to insert
KVMap map_insert;
MakeKVMap(kSampleSize_, &map_insert);
Iterator* dbiter;
// Put data in database
for (auto& kv : map_insert) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
Sleep(1);
ASSERT_EQ(2, GetColumnFamilyCount());
// T=1, keys should still reside in database
dbiter = date_tiered_db_->NewIterator(ropts);
dbiter->SeekToFirst();
for (auto& kv : map_insert) {
ASSERT_TRUE(dbiter->Valid());
ASSERT_EQ(0, dbiter->value().compare(kv.second));
dbiter->Next();
}
delete dbiter;
Sleep(4);
// T=5, keys should not be retrieved
for (auto& kv : map_insert) {
std::string value;
auto s = date_tiered_db_->Get(ropts, kv.first, &value);
ASSERT_TRUE(s.IsNotFound());
}
// Explicitly drop obsolete column families
date_tiered_db_->DropObsoleteColumnFamilies();
// Only default column family
ASSERT_EQ(1, GetColumnFamilyCount());
// Empty iterator
dbiter = date_tiered_db_->NewIterator(ropts);
dbiter->Seek(map_insert.begin()->first);
ASSERT_FALSE(dbiter->Valid());
delete dbiter;
CloseDateTieredDB();
}
// Iterator should be able to merge data from multiple column families
TEST_F(DateTieredTest, IteratorMerge) {
WriteOptions wopts;
ReadOptions ropts;
// T=0, open the database and insert data
OpenDateTieredDB(4, 2);
ASSERT_TRUE(date_tiered_db_.get() != nullptr);
Iterator* dbiter;
// Put data in database
KVMap map_insert1;
MakeKVMap(kSampleSize_, &map_insert1);
for (auto& kv : map_insert1) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
ASSERT_EQ(2, GetColumnFamilyCount());
Sleep(2);
// Put more data
KVMap map_insert2;
MakeKVMap(kSampleSize_, &map_insert2);
for (auto& kv : map_insert2) {
ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
}
// Multiple column families for time series data
ASSERT_EQ(3, GetColumnFamilyCount());
// Iterator should be able to merge data from different column families
dbiter = date_tiered_db_->NewIterator(ropts);
dbiter->SeekToFirst();
KVMap::iterator iter1 = map_insert1.begin();
KVMap::iterator iter2 = map_insert2.begin();
for (; iter1 != map_insert1.end() && iter2 != map_insert2.end();
iter1++, iter2++) {
ASSERT_TRUE(dbiter->Valid());
ASSERT_EQ(0, dbiter->value().compare(iter1->second));
dbiter->Next();
ASSERT_TRUE(dbiter->Valid());
ASSERT_EQ(0, dbiter->value().compare(iter2->second));
dbiter->Next();
}
delete dbiter;
CloseDateTieredDB();
}
} // namespace rocksdb
// A black-box test for the DateTieredDB around rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "SKIPPED as DateTieredDB is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

1207
utilities/document/document_db.cc
View File

File diff suppressed because it is too large Load Diff

338
utilities/document/document_db_test.cc
View File

@ -1,338 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include <algorithm>
#include "rocksdb/utilities/json_document.h"
#include "rocksdb/utilities/document_db.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
class DocumentDBTest : public testing::Test {
public:
DocumentDBTest() {
dbname_ = test::PerThreadDBPath("document_db_test");
DestroyDB(dbname_, Options());
}
~DocumentDBTest() {
delete db_;
DestroyDB(dbname_, Options());
}
void AssertCursorIDs(Cursor* cursor, std::vector<int64_t> expected) {
std::vector<int64_t> got;
while (cursor->Valid()) {
ASSERT_TRUE(cursor->Valid());
ASSERT_TRUE(cursor->document().Contains("_id"));
got.push_back(cursor->document()["_id"].GetInt64());
cursor->Next();
}
std::sort(expected.begin(), expected.end());
std::sort(got.begin(), got.end());
ASSERT_TRUE(got == expected);
}
// converts ' to ", so that we don't have to escape " all over the place
std::string ConvertQuotes(const std::string& input) {
std::string output;
for (auto x : input) {
if (x == '\'') {
output.push_back('\"');
} else {
output.push_back(x);
}
}
return output;
}
void CreateIndexes(std::vector<DocumentDB::IndexDescriptor> indexes) {
for (auto i : indexes) {
ASSERT_OK(db_->CreateIndex(WriteOptions(), i));
}
}
JSONDocument* Parse(const std::string& doc) {
return JSONDocument::ParseJSON(ConvertQuotes(doc).c_str());
}
std::string dbname_;
DocumentDB* db_;
};
TEST_F(DocumentDBTest, SimpleQueryTest) {
DocumentDBOptions options;
DocumentDB::IndexDescriptor index;
index.description = Parse("{\"name\": 1}");
index.name = "name_index";
ASSERT_OK(DocumentDB::Open(options, dbname_, {}, &db_));
CreateIndexes({index});
delete db_;
db_ = nullptr;
// now there is index present
ASSERT_OK(DocumentDB::Open(options, dbname_, {index}, &db_));
assert(db_ != nullptr);
delete index.description;
std::vector<std::string> json_objects = {
"{\"_id\': 1, \"name\": \"One\"}", "{\"_id\": 2, \"name\": \"Two\"}",
"{\"_id\": 3, \"name\": \"Three\"}", "{\"_id\": 4, \"name\": \"Four\"}"};
for (auto& json : json_objects) {
std::unique_ptr<JSONDocument> document(Parse(json));
ASSERT_TRUE(document.get() != nullptr);
ASSERT_OK(db_->Insert(WriteOptions(), *document));
}
// inserting a document with existing primary key should return failure
{
std::unique_ptr<JSONDocument> document(Parse(json_objects[0]));
ASSERT_TRUE(document.get() != nullptr);
Status s = db_->Insert(WriteOptions(), *document);
ASSERT_TRUE(s.IsInvalidArgument());
}
// find equal to "Two"
{
std::unique_ptr<JSONDocument> query(
Parse("[{'$filter': {'name': 'Two', '$index': 'name_index'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {2});
}
// find less than "Three"
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'name': {'$lt': 'Three'}, '$index': "
"'name_index'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {1, 4});
}
// find less than "Three" without index
{
std::unique_ptr<JSONDocument> query(
Parse("[{'$filter': {'name': {'$lt': 'Three'} }}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {1, 4});
}
// remove less or equal to "Three"
{
std::unique_ptr<JSONDocument> query(
Parse("{'name': {'$lte': 'Three'}, '$index': 'name_index'}"));
ASSERT_OK(db_->Remove(ReadOptions(), WriteOptions(), *query));
}
// find all -- only "Two" left, everything else should be deleted
{
std::unique_ptr<JSONDocument> query(Parse("[]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {2});
}
}
TEST_F(DocumentDBTest, ComplexQueryTest) {
DocumentDBOptions options;
DocumentDB::IndexDescriptor priority_index;
priority_index.description = Parse("{'priority': 1}");
priority_index.name = "priority";
DocumentDB::IndexDescriptor job_name_index;
job_name_index.description = Parse("{'job_name': 1}");
job_name_index.name = "job_name";
DocumentDB::IndexDescriptor progress_index;
progress_index.description = Parse("{'progress': 1}");
progress_index.name = "progress";
ASSERT_OK(DocumentDB::Open(options, dbname_, {}, &db_));
CreateIndexes({priority_index, progress_index});
delete priority_index.description;
delete progress_index.description;
std::vector<std::string> json_objects = {
"{'_id': 1, 'job_name': 'play', 'priority': 10, 'progress': 14.2}",
"{'_id': 2, 'job_name': 'white', 'priority': 2, 'progress': 45.1}",
"{'_id': 3, 'job_name': 'straw', 'priority': 5, 'progress': 83.2}",
"{'_id': 4, 'job_name': 'temporary', 'priority': 3, 'progress': 14.9}",
"{'_id': 5, 'job_name': 'white', 'priority': 4, 'progress': 44.2}",
"{'_id': 6, 'job_name': 'tea', 'priority': 1, 'progress': 12.4}",
"{'_id': 7, 'job_name': 'delete', 'priority': 2, 'progress': 77.54}",
"{'_id': 8, 'job_name': 'rock', 'priority': 3, 'progress': 93.24}",
"{'_id': 9, 'job_name': 'steady', 'priority': 3, 'progress': 9.1}",
"{'_id': 10, 'job_name': 'white', 'priority': 1, 'progress': 61.4}",
"{'_id': 11, 'job_name': 'who', 'priority': 4, 'progress': 39.41}",
"{'_id': 12, 'job_name': 'who', 'priority': -1, 'progress': 39.42}",
"{'_id': 13, 'job_name': 'who', 'priority': -2, 'progress': 39.42}", };
// add index on the fly!
CreateIndexes({job_name_index});
delete job_name_index.description;
for (auto& json : json_objects) {
std::unique_ptr<JSONDocument> document(Parse(json));
ASSERT_TRUE(document != nullptr);
ASSERT_OK(db_->Insert(WriteOptions(), *document));
}
// 2 < priority < 4 AND progress > 10.0, index priority
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'priority': {'$lt': 4, '$gt': 2}, 'progress': {'$gt': "
"10.0}, '$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {4, 8});
}
// -1 <= priority <= 1, index priority
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'priority': {'$lte': 1, '$gte': -1},"
" '$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {6, 10, 12});
}
// 2 < priority < 4 AND progress > 10.0, index progress
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'priority': {'$lt': 4, '$gt': 2}, 'progress': {'$gt': "
"10.0}, '$index': 'progress'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {4, 8});
}
// job_name == 'white' AND priority >= 2, index job_name
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'job_name': 'white', 'priority': {'$gte': "
"2}, '$index': 'job_name'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {2, 5});
}
// 35.0 <= progress < 65.5, index progress
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'progress': {'$gt': 5.0, '$gte': 35.0, '$lt': 65.5}, "
"'$index': 'progress'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {2, 5, 10, 11, 12, 13});
}
// 2 < priority <= 4, index priority
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'priority': {'$gt': 2, '$lt': 8, '$lte': 4}, "
"'$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {4, 5, 8, 9, 11});
}
// Delete all whose progress is bigger than 50%
{
std::unique_ptr<JSONDocument> query(
Parse("{'progress': {'$gt': 50.0}, '$index': 'progress'}"));
ASSERT_OK(db_->Remove(ReadOptions(), WriteOptions(), *query));
}
// 2 < priority < 6, index priority
{
std::unique_ptr<JSONDocument> query(Parse(
"[{'$filter': {'priority': {'$gt': 2, '$lt': 6}, "
"'$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
AssertCursorIDs(cursor.get(), {4, 5, 9, 11});
}
// update set priority to 10 where job_name is 'white'
{
std::unique_ptr<JSONDocument> query(Parse("{'job_name': 'white'}"));
std::unique_ptr<JSONDocument> update(Parse("{'$set': {'priority': 10}}"));
ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update));
}
// update twice: set priority to 15 where job_name is 'white'
{
std::unique_ptr<JSONDocument> query(Parse("{'job_name': 'white'}"));
std::unique_ptr<JSONDocument> update(Parse("{'$set': {'priority': 10},"
"'$set': {'priority': 15}}"));
ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update));
}
// update twice: set priority to 15 and
// progress to 40 where job_name is 'white'
{
std::unique_ptr<JSONDocument> query(Parse("{'job_name': 'white'}"));
std::unique_ptr<JSONDocument> update(
Parse("{'$set': {'priority': 10, 'progress': 35},"
"'$set': {'priority': 15, 'progress': 40}}"));
ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update));
}
// priority < 0
{
std::unique_ptr<JSONDocument> query(
Parse("[{'$filter': {'priority': {'$lt': 0}, '$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
ASSERT_OK(cursor->status());
AssertCursorIDs(cursor.get(), {12, 13});
}
// -2 < priority < 0
{
std::unique_ptr<JSONDocument> query(
Parse("[{'$filter': {'priority': {'$gt': -2, '$lt': 0},"
" '$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
ASSERT_OK(cursor->status());
AssertCursorIDs(cursor.get(), {12});
}
// -2 <= priority < 0
{
std::unique_ptr<JSONDocument> query(
Parse("[{'$filter': {'priority': {'$gte': -2, '$lt': 0},"
" '$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
ASSERT_OK(cursor->status());
AssertCursorIDs(cursor.get(), {12, 13});
}
// 4 < priority
{
std::unique_ptr<JSONDocument> query(
Parse("[{'$filter': {'priority': {'$gt': 4}, '$index': 'priority'}}]"));
std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
ASSERT_OK(cursor->status());
AssertCursorIDs(cursor.get(), {1, 2, 5});
}
Status s = db_->DropIndex("doesnt-exist");
ASSERT_TRUE(!s.ok());
ASSERT_OK(db_->DropIndex("priority"));
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "SKIPPED as DocumentDB is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

610
utilities/document/json_document.cc
View File

@ -1,610 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "rocksdb/utilities/json_document.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <assert.h>
#include <inttypes.h>
#include <string.h>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "third-party/fbson/FbsonDocument.h"
#include "third-party/fbson/FbsonJsonParser.h"
#include "third-party/fbson/FbsonUtil.h"
#include "util/coding.h"
using std::placeholders::_1;
namespace {
size_t ObjectNumElem(const fbson::ObjectVal& objectVal) {
size_t size = 0;
for (auto keyValuePair : objectVal) {
(void)keyValuePair;
++size;
}
return size;
}
template <typename Func>
void InitJSONDocument(std::unique_ptr<char[]>* data,
fbson::FbsonValue** value,
Func f) {
// TODO(stash): maybe add function to FbsonDocument to avoid creating array?
fbson::FbsonWriter writer;
bool res __attribute__((__unused__)) = writer.writeStartArray();
assert(res);
uint32_t bytesWritten __attribute__((__unused__));
bytesWritten = f(writer);
assert(bytesWritten != 0);
res = writer.writeEndArray();
assert(res);
char* buf = new char[writer.getOutput()->getSize()];
memcpy(buf, writer.getOutput()->getBuffer(), writer.getOutput()->getSize());
*value = ((fbson::FbsonDocument *)buf)->getValue();
assert((*value)->isArray());
assert(((fbson::ArrayVal*)*value)->numElem() == 1);
*value = ((fbson::ArrayVal*)*value)->get(0);
data->reset(buf);
}
void InitString(std::unique_ptr<char[]>* data,
fbson::FbsonValue** value,
const std::string& s) {
InitJSONDocument(data, value, std::bind(
[](fbson::FbsonWriter& writer, const std::string& str) -> uint32_t {
bool res __attribute__((__unused__)) = writer.writeStartString();
assert(res);
auto bytesWritten = writer.writeString(str.c_str(),
static_cast<uint32_t>(str.length()));
res = writer.writeEndString();
assert(res);
// If the string is empty, then bytesWritten == 0, and assert in
// InitJsonDocument will fail.
return bytesWritten + static_cast<uint32_t>(str.empty());
},
_1, s));
}
bool IsNumeric(fbson::FbsonValue* value) {
return value->isInt8() || value->isInt16() ||
value->isInt32() || value->isInt64();
}
int64_t GetInt64ValFromFbsonNumericType(fbson::FbsonValue* value) {
switch (value->type()) {
case fbson::FbsonType::T_Int8:
return reinterpret_cast<fbson::Int8Val*>(value)->val();
case fbson::FbsonType::T_Int16:
return reinterpret_cast<fbson::Int16Val*>(value)->val();
case fbson::FbsonType::T_Int32:
return reinterpret_cast<fbson::Int32Val*>(value)->val();
case fbson::FbsonType::T_Int64:
return reinterpret_cast<fbson::Int64Val*>(value)->val();
default:
assert(false);
}
return 0;
}
bool IsComparable(fbson::FbsonValue* left, fbson::FbsonValue* right) {
if (left->type() == right->type()) {
return true;
}
if (IsNumeric(left) && IsNumeric(right)) {
return true;
}
return false;
}
void CreateArray(std::unique_ptr<char[]>* data, fbson::FbsonValue** value) {
fbson::FbsonWriter writer;
bool res __attribute__((__unused__)) = writer.writeStartArray();
assert(res);
res = writer.writeEndArray();
assert(res);
data->reset(new char[writer.getOutput()->getSize()]);
memcpy(data->get(),
writer.getOutput()->getBuffer(),
writer.getOutput()->getSize());
*value = reinterpret_cast<fbson::FbsonDocument*>(data->get())->getValue();
}
void CreateObject(std::unique_ptr<char[]>* data, fbson::FbsonValue** value) {
fbson::FbsonWriter writer;
bool res __attribute__((__unused__)) = writer.writeStartObject();
assert(res);
res = writer.writeEndObject();
assert(res);
data->reset(new char[writer.getOutput()->getSize()]);
memcpy(data->get(),
writer.getOutput()->getBuffer(),
writer.getOutput()->getSize());
*value = reinterpret_cast<fbson::FbsonDocument*>(data->get())->getValue();
}
} // namespace
namespace rocksdb {
// TODO(stash): find smth easier
JSONDocument::JSONDocument() {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeNull, _1));
}
JSONDocument::JSONDocument(bool b) {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeBool, _1, b));
}
JSONDocument::JSONDocument(double d) {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeDouble, _1, d));
}
JSONDocument::JSONDocument(int8_t i) {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeInt8, _1, i));
}
JSONDocument::JSONDocument(int16_t i) {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeInt16, _1, i));
}
JSONDocument::JSONDocument(int32_t i) {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeInt32, _1, i));
}
JSONDocument::JSONDocument(int64_t i) {
InitJSONDocument(&data_,
&value_,
std::bind(&fbson::FbsonWriter::writeInt64, _1, i));
}
JSONDocument::JSONDocument(const std::string& s) {
InitString(&data_, &value_, s);
}
JSONDocument::JSONDocument(const char* s) : JSONDocument(std::string(s)) {
}
void JSONDocument::InitFromValue(const fbson::FbsonValue* val) {
data_.reset(new char[val->numPackedBytes()]);
memcpy(data_.get(), val, val->numPackedBytes());
value_ = reinterpret_cast<fbson::FbsonValue*>(data_.get());
}
// Private constructor
JSONDocument::JSONDocument(fbson::FbsonValue* val, bool makeCopy) {
if (makeCopy) {
InitFromValue(val);
} else {
value_ = val;
}
}
JSONDocument::JSONDocument(Type _type) {
// TODO(icanadi) make all of this better by using templates
switch (_type) {
case kNull:
InitJSONDocument(&data_, &value_,
std::bind(&fbson::FbsonWriter::writeNull, _1));
break;
case kObject:
CreateObject(&data_, &value_);
break;
case kBool:
InitJSONDocument(&data_, &value_,
std::bind(&fbson::FbsonWriter::writeBool, _1, false));
break;
case kDouble:
InitJSONDocument(&data_, &value_,
std::bind(&fbson::FbsonWriter::writeDouble, _1, 0.));
break;
case kArray:
CreateArray(&data_, &value_);
break;
case kInt64:
InitJSONDocument(&data_, &value_,
std::bind(&fbson::FbsonWriter::writeInt64, _1, 0));
break;
case kString:
InitString(&data_, &value_, "");
break;
default:
assert(false);
}
}
JSONDocument::JSONDocument(const JSONDocument& jsonDocument) {
if (jsonDocument.IsOwner()) {
InitFromValue(jsonDocument.value_);
} else {
value_ = jsonDocument.value_;
}
}
JSONDocument::JSONDocument(JSONDocument&& jsonDocument) {
value_ = jsonDocument.value_;
data_.swap(jsonDocument.data_);
}
JSONDocument& JSONDocument::operator=(JSONDocument jsonDocument) {
value_ = jsonDocument.value_;
data_.swap(jsonDocument.data_);
return *this;
}
JSONDocument::Type JSONDocument::type() const {
switch (value_->type()) {
case fbson::FbsonType::T_Null:
return JSONDocument::kNull;
case fbson::FbsonType::T_True:
case fbson::FbsonType::T_False:
return JSONDocument::kBool;
case fbson::FbsonType::T_Int8:
case fbson::FbsonType::T_Int16:
case fbson::FbsonType::T_Int32:
case fbson::FbsonType::T_Int64:
return JSONDocument::kInt64;
case fbson::FbsonType::T_Double:
return JSONDocument::kDouble;
case fbson::FbsonType::T_String:
return JSONDocument::kString;
case fbson::FbsonType::T_Object:
return JSONDocument::kObject;
case fbson::FbsonType::T_Array:
return JSONDocument::kArray;
case fbson::FbsonType::T_Binary:
default:
assert(false);
}
return JSONDocument::kNull;
}
bool JSONDocument::Contains(const std::string& key) const {
assert(IsObject());
auto objectVal = reinterpret_cast<fbson::ObjectVal*>(value_);
return objectVal->find(key.c_str()) != nullptr;
}
JSONDocument JSONDocument::operator[](const std::string& key) const {
assert(IsObject());
auto objectVal = reinterpret_cast<fbson::ObjectVal*>(value_);
auto foundValue = objectVal->find(key.c_str());
assert(foundValue != nullptr);
// No need to save paths in const objects
JSONDocument ans(foundValue, false);
return ans;
}
size_t JSONDocument::Count() const {
assert(IsObject() || IsArray());
if (IsObject()) {
// TODO(stash): add to fbson?
const fbson::ObjectVal& objectVal =
*reinterpret_cast<fbson::ObjectVal*>(value_);
return ObjectNumElem(objectVal);
} else if (IsArray()) {
auto arrayVal = reinterpret_cast<fbson::ArrayVal*>(value_);
return arrayVal->numElem();
}
assert(false);
return 0;
}
JSONDocument JSONDocument::operator[](size_t i) const {
assert(IsArray());
auto arrayVal = reinterpret_cast<fbson::ArrayVal*>(value_);
auto foundValue = arrayVal->get(static_cast<int>(i));
JSONDocument ans(foundValue, false);
return ans;
}
bool JSONDocument::IsNull() const {
return value_->isNull();
}
bool JSONDocument::IsArray() const {
return value_->isArray();
}
bool JSONDocument::IsBool() const {
return value_->isTrue() || value_->isFalse();
}
bool JSONDocument::IsDouble() const {
return value_->isDouble();
}
bool JSONDocument::IsInt64() const {
return value_->isInt8() || value_->isInt16() ||
value_->isInt32() || value_->isInt64();
}
bool JSONDocument::IsObject() const {
return value_->isObject();
}
bool JSONDocument::IsString() const {
return value_->isString();
}
bool JSONDocument::GetBool() const {
assert(IsBool());
return value_->isTrue();
}
double JSONDocument::GetDouble() const {
assert(IsDouble());
return ((fbson::DoubleVal*)value_)->val();
}
int64_t JSONDocument::GetInt64() const {
assert(IsInt64());
return GetInt64ValFromFbsonNumericType(value_);
}
std::string JSONDocument::GetString() const {
assert(IsString());
fbson::StringVal* stringVal = (fbson::StringVal*)value_;
return std::string(stringVal->getBlob(), stringVal->getBlobLen());
}
namespace {
// FbsonValue can be int8, int16, int32, int64
bool CompareNumeric(fbson::FbsonValue* left, fbson::FbsonValue* right) {
assert(IsNumeric(left) && IsNumeric(right));
return GetInt64ValFromFbsonNumericType(left) ==
GetInt64ValFromFbsonNumericType(right);
}
bool CompareSimpleTypes(fbson::FbsonValue* left, fbson::FbsonValue* right) {
if (IsNumeric(left)) {
return CompareNumeric(left, right);
}
if (left->numPackedBytes() != right->numPackedBytes()) {
return false;
}
return memcmp(left, right, left->numPackedBytes()) == 0;
}
bool CompareFbsonValue(fbson::FbsonValue* left, fbson::FbsonValue* right) {
if (!IsComparable(left, right)) {
return false;
}
switch (left->type()) {
case fbson::FbsonType::T_True:
case fbson::FbsonType::T_False:
case fbson::FbsonType::T_Null:
return true;
case fbson::FbsonType::T_Int8:
case fbson::FbsonType::T_Int16:
case fbson::FbsonType::T_Int32:
case fbson::FbsonType::T_Int64:
return CompareNumeric(left, right);
case fbson::FbsonType::T_String:
case fbson::FbsonType::T_Double:
return CompareSimpleTypes(left, right);
case fbson::FbsonType::T_Object:
{
auto leftObject = reinterpret_cast<fbson::ObjectVal*>(left);
auto rightObject = reinterpret_cast<fbson::ObjectVal*>(right);
if (ObjectNumElem(*leftObject) != ObjectNumElem(*rightObject)) {
return false;
}
for (auto && keyValue : *leftObject) {
std::string str(keyValue.getKeyStr(), keyValue.klen());
if (rightObject->find(str.c_str()) == nullptr) {
return false;
}
if (!CompareFbsonValue(keyValue.value(),
rightObject->find(str.c_str()))) {
return false;
}
}
return true;
}
case fbson::FbsonType::T_Array:
{
auto leftArr = reinterpret_cast<fbson::ArrayVal*>(left);
auto rightArr = reinterpret_cast<fbson::ArrayVal*>(right);
if (leftArr->numElem() != rightArr->numElem()) {
return false;
}
for (int i = 0; i < static_cast<int>(leftArr->numElem()); ++i) {
if (!CompareFbsonValue(leftArr->get(i), rightArr->get(i))) {
return false;
}
}
return true;
}
default:
assert(false);
}
return false;
}
} // namespace
bool JSONDocument::operator==(const JSONDocument& rhs) const {
return CompareFbsonValue(value_, rhs.value_);
}
bool JSONDocument::operator!=(const JSONDocument& rhs) const {
return !(*this == rhs);
}
JSONDocument JSONDocument::Copy() const {
return JSONDocument(value_, true);
}
bool JSONDocument::IsOwner() const {
return data_.get() != nullptr;
}
std::string JSONDocument::DebugString() const {
fbson::FbsonToJson fbsonToJson;
return fbsonToJson.json(value_);
}
JSONDocument::ItemsIteratorGenerator JSONDocument::Items() const {
assert(IsObject());
return ItemsIteratorGenerator(*(reinterpret_cast<fbson::ObjectVal*>(value_)));
}
// TODO(icanadi) (perf) allocate objects with arena
JSONDocument* JSONDocument::ParseJSON(const char* json) {
fbson::FbsonJsonParser parser;
if (!parser.parse(json)) {
return nullptr;
}
auto fbsonVal = fbson::FbsonDocument::createValue(
parser.getWriter().getOutput()->getBuffer(),
static_cast<uint32_t>(parser.getWriter().getOutput()->getSize()));
if (fbsonVal == nullptr) {
return nullptr;
}
return new JSONDocument(fbsonVal, true);
}
void JSONDocument::Serialize(std::string* dst) const {
// first byte is reserved for header
// currently, header is only version number. that will help us provide
// backwards compatility. we might also store more information here if
// necessary
dst->push_back(kSerializationFormatVersion);
dst->push_back(FBSON_VER);
dst->append(reinterpret_cast<char*>(value_), value_->numPackedBytes());
}
const char JSONDocument::kSerializationFormatVersion = 2;
JSONDocument* JSONDocument::Deserialize(const Slice& src) {
Slice input(src);
if (src.size() == 0) {
return nullptr;
}
char header = input[0];
if (header == 1) {
assert(false);
}
input.remove_prefix(1);
auto value = fbson::FbsonDocument::createValue(input.data(),
static_cast<uint32_t>(input.size()));
if (value == nullptr) {
return nullptr;
}
return new JSONDocument(value, true);
}
class JSONDocument::const_item_iterator::Impl {
public:
typedef fbson::ObjectVal::const_iterator It;
explicit Impl(It it) : it_(it) {}
const char* getKeyStr() const {
return it_->getKeyStr();
}
uint8_t klen() const {
return it_->klen();
}
It& operator++() {
return ++it_;
}
bool operator!=(const Impl& other) {
return it_ != other.it_;
}
fbson::FbsonValue* value() const {
return it_->value();
}
private:
It it_;
};
JSONDocument::const_item_iterator::const_item_iterator(Impl* impl)
: it_(impl) {}
JSONDocument::const_item_iterator::const_item_iterator(const_item_iterator&& a)
: it_(std::move(a.it_)) {}
JSONDocument::const_item_iterator&
JSONDocument::const_item_iterator::operator++() {
++(*it_);
return *this;
}
bool JSONDocument::const_item_iterator::operator!=(
const const_item_iterator& other) {
return *it_ != *(other.it_);
}
JSONDocument::const_item_iterator::~const_item_iterator() {
}
JSONDocument::const_item_iterator::value_type
JSONDocument::const_item_iterator::operator*() {
return JSONDocument::const_item_iterator::value_type(std::string(it_->getKeyStr(), it_->klen()),
JSONDocument(it_->value(), false));
}
JSONDocument::ItemsIteratorGenerator::ItemsIteratorGenerator(
const fbson::ObjectVal& object)
: object_(object) {}
JSONDocument::const_item_iterator
JSONDocument::ItemsIteratorGenerator::begin() const {
return const_item_iterator(new const_item_iterator::Impl(object_.begin()));
}
JSONDocument::const_item_iterator
JSONDocument::ItemsIteratorGenerator::end() const {
return const_item_iterator(new const_item_iterator::Impl(object_.end()));
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

120
utilities/document/json_document_builder.cc
View File

@ -1,120 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include <assert.h>
#include <limits>
#include <stdint.h>
#include "rocksdb/utilities/json_document.h"
#include "third-party/fbson/FbsonWriter.h"
namespace rocksdb {
JSONDocumentBuilder::JSONDocumentBuilder()
: writer_(new fbson::FbsonWriter()) {
}
JSONDocumentBuilder::JSONDocumentBuilder(fbson::FbsonOutStream* out)
: writer_(new fbson::FbsonWriter(*out)) {
}
void JSONDocumentBuilder::Reset() {
writer_->reset();
}
bool JSONDocumentBuilder::WriteStartArray() {
return writer_->writeStartArray();
}
bool JSONDocumentBuilder::WriteEndArray() {
return writer_->writeEndArray();
}
bool JSONDocumentBuilder::WriteStartObject() {
return writer_->writeStartObject();
}
bool JSONDocumentBuilder::WriteEndObject() {
return writer_->writeEndObject();
}
bool JSONDocumentBuilder::WriteKeyValue(const std::string& key,
const JSONDocument& value) {
assert(key.size() <= std::numeric_limits<uint8_t>::max());
size_t bytesWritten = writer_->writeKey(key.c_str(),
static_cast<uint8_t>(key.size()));
if (bytesWritten == 0) {
return false;
}
return WriteJSONDocument(value);
}
bool JSONDocumentBuilder::WriteJSONDocument(const JSONDocument& value) {
switch (value.type()) {
case JSONDocument::kNull:
return writer_->writeNull() != 0;
case JSONDocument::kInt64:
return writer_->writeInt64(value.GetInt64());
case JSONDocument::kDouble:
return writer_->writeDouble(value.GetDouble());
case JSONDocument::kBool:
return writer_->writeBool(value.GetBool());
case JSONDocument::kString:
{
bool res = writer_->writeStartString();
if (!res) {
return false;
}
const std::string& str = value.GetString();
res = writer_->writeString(str.c_str(),
static_cast<uint32_t>(str.size()));
if (!res) {
return false;
}
return writer_->writeEndString();
}
case JSONDocument::kArray:
{
bool res = WriteStartArray();
if (!res) {
return false;
}
for (size_t i = 0; i < value.Count(); ++i) {
res = WriteJSONDocument(value[i]);
if (!res) {
return false;
}
}
return WriteEndArray();
}
case JSONDocument::kObject:
{
bool res = WriteStartObject();
if (!res) {
return false;
}
for (auto keyValue : value.Items()) {
WriteKeyValue(keyValue.first, keyValue.second);
}
return WriteEndObject();
}
default:
assert(false);
}
return false;
}
JSONDocument JSONDocumentBuilder::GetJSONDocument() {
fbson::FbsonValue* value =
fbson::FbsonDocument::createValue(writer_->getOutput()->getBuffer(),
static_cast<uint32_t>(writer_->getOutput()->getSize()));
return JSONDocument(value, true);
}
JSONDocumentBuilder::~JSONDocumentBuilder() {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

343
utilities/document/json_document_test.cc
View File

@ -1,343 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include <map>
#include <set>
#include <string>
#include "rocksdb/utilities/json_document.h"
#include "util/testutil.h"
#include "util/testharness.h"
namespace rocksdb {
namespace {
void AssertField(const JSONDocument& json, const std::string& field) {
ASSERT_TRUE(json.Contains(field));
ASSERT_TRUE(json[field].IsNull());
}
void AssertField(const JSONDocument& json, const std::string& field,
const std::string& expected) {
ASSERT_TRUE(json.Contains(field));
ASSERT_TRUE(json[field].IsString());
ASSERT_EQ(expected, json[field].GetString());
}
void AssertField(const JSONDocument& json, const std::string& field,
int64_t expected) {
ASSERT_TRUE(json.Contains(field));
ASSERT_TRUE(json[field].IsInt64());
ASSERT_EQ(expected, json[field].GetInt64());
}
void AssertField(const JSONDocument& json, const std::string& field,
bool expected) {
ASSERT_TRUE(json.Contains(field));
ASSERT_TRUE(json[field].IsBool());
ASSERT_EQ(expected, json[field].GetBool());
}
void AssertField(const JSONDocument& json, const std::string& field,
double expected) {
ASSERT_TRUE(json.Contains(field));
ASSERT_TRUE(json[field].IsDouble());
ASSERT_DOUBLE_EQ(expected, json[field].GetDouble());
}
} // namespace
class JSONDocumentTest : public testing::Test {
public:
JSONDocumentTest()
: rnd_(101)
{}
void AssertSampleJSON(const JSONDocument& json) {
AssertField(json, "title", std::string("json"));
AssertField(json, "type", std::string("object"));
// properties
ASSERT_TRUE(json.Contains("properties"));
ASSERT_TRUE(json["properties"].Contains("flags"));
ASSERT_TRUE(json["properties"]["flags"].IsArray());
ASSERT_EQ(3u, json["properties"]["flags"].Count());
ASSERT_TRUE(json["properties"]["flags"][0].IsInt64());
ASSERT_EQ(10, json["properties"]["flags"][0].GetInt64());
ASSERT_TRUE(json["properties"]["flags"][1].IsString());
ASSERT_EQ("parse", json["properties"]["flags"][1].GetString());
ASSERT_TRUE(json["properties"]["flags"][2].IsObject());
AssertField(json["properties"]["flags"][2], "tag", std::string("no"));
AssertField(json["properties"]["flags"][2], std::string("status"));
AssertField(json["properties"], "age", 110.5e-4);
AssertField(json["properties"], "depth", static_cast<int64_t>(-10));
// test iteration
std::set<std::string> expected({"flags", "age", "depth"});
for (auto item : json["properties"].Items()) {
auto iter = expected.find(item.first);
ASSERT_TRUE(iter != expected.end());
expected.erase(iter);
}
ASSERT_EQ(0U, expected.size());
ASSERT_TRUE(json.Contains("latlong"));
ASSERT_TRUE(json["latlong"].IsArray());
ASSERT_EQ(2u, json["latlong"].Count());
ASSERT_TRUE(json["latlong"][0].IsDouble());
ASSERT_EQ(53.25, json["latlong"][0].GetDouble());
ASSERT_TRUE(json["latlong"][1].IsDouble());
ASSERT_EQ(43.75, json["latlong"][1].GetDouble());
AssertField(json, "enabled", true);
}
const std::string kSampleJSON =
"{ \"title\" : \"json\", \"type\" : \"object\", \"properties\" : { "
"\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": null}], "
"\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], "
"\"enabled\": true }";
const std::string kSampleJSONDifferent =
"{ \"title\" : \"json\", \"type\" : \"object\", \"properties\" : { "
"\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": 2}], "
"\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], "
"\"enabled\": true }";
Random rnd_;
};
TEST_F(JSONDocumentTest, MakeNullTest) {
JSONDocument x;
ASSERT_TRUE(x.IsNull());
ASSERT_TRUE(x.IsOwner());
ASSERT_TRUE(!x.IsBool());
}
TEST_F(JSONDocumentTest, MakeBoolTest) {
{
JSONDocument x(true);
ASSERT_TRUE(x.IsOwner());
ASSERT_TRUE(x.IsBool());
ASSERT_TRUE(!x.IsInt64());
ASSERT_EQ(x.GetBool(), true);
}
{
JSONDocument x(false);
ASSERT_TRUE(x.IsOwner());
ASSERT_TRUE(x.IsBool());
ASSERT_TRUE(!x.IsInt64());
ASSERT_EQ(x.GetBool(), false);
}
}
TEST_F(JSONDocumentTest, MakeInt64Test) {
JSONDocument x(static_cast<int64_t>(16));
ASSERT_TRUE(x.IsInt64());
ASSERT_TRUE(x.IsInt64());
ASSERT_TRUE(!x.IsBool());
ASSERT_TRUE(x.IsOwner());
ASSERT_EQ(x.GetInt64(), 16);
}
TEST_F(JSONDocumentTest, MakeStringTest) {
JSONDocument x("string");
ASSERT_TRUE(x.IsOwner());
ASSERT_TRUE(x.IsString());
ASSERT_TRUE(!x.IsBool());
ASSERT_EQ(x.GetString(), "string");
}
TEST_F(JSONDocumentTest, MakeDoubleTest) {
JSONDocument x(5.6);
ASSERT_TRUE(x.IsOwner());
ASSERT_TRUE(x.IsDouble());
ASSERT_TRUE(!x.IsBool());
ASSERT_EQ(x.GetDouble(), 5.6);
}
TEST_F(JSONDocumentTest, MakeByTypeTest) {
{
JSONDocument x(JSONDocument::kNull);
ASSERT_TRUE(x.IsNull());
}
{
JSONDocument x(JSONDocument::kBool);
ASSERT_TRUE(x.IsBool());
}
{
JSONDocument x(JSONDocument::kString);
ASSERT_TRUE(x.IsString());
}
{
JSONDocument x(JSONDocument::kInt64);
ASSERT_TRUE(x.IsInt64());
}
{
JSONDocument x(JSONDocument::kDouble);
ASSERT_TRUE(x.IsDouble());
}
{
JSONDocument x(JSONDocument::kObject);
ASSERT_TRUE(x.IsObject());
}
{
JSONDocument x(JSONDocument::kArray);
ASSERT_TRUE(x.IsArray());
}
}
TEST_F(JSONDocumentTest, Parsing) {
std::unique_ptr<JSONDocument> parsed_json(
JSONDocument::ParseJSON(kSampleJSON.c_str()));
ASSERT_TRUE(parsed_json->IsOwner());
ASSERT_TRUE(parsed_json != nullptr);
AssertSampleJSON(*parsed_json);
// test deep copying
JSONDocument copied_json_document(*parsed_json);
AssertSampleJSON(copied_json_document);
ASSERT_TRUE(copied_json_document == *parsed_json);
std::unique_ptr<JSONDocument> parsed_different_sample(
JSONDocument::ParseJSON(kSampleJSONDifferent.c_str()));
ASSERT_TRUE(parsed_different_sample != nullptr);
ASSERT_TRUE(!(*parsed_different_sample == copied_json_document));
// parse error
const std::string kFaultyJSON =
kSampleJSON.substr(0, kSampleJSON.size() - 10);
ASSERT_TRUE(JSONDocument::ParseJSON(kFaultyJSON.c_str()) == nullptr);
}
TEST_F(JSONDocumentTest, Serialization) {
std::unique_ptr<JSONDocument> parsed_json(
JSONDocument::ParseJSON(kSampleJSON.c_str()));
ASSERT_TRUE(parsed_json != nullptr);
ASSERT_TRUE(parsed_json->IsOwner());
std::string serialized;
parsed_json->Serialize(&serialized);
std::unique_ptr<JSONDocument> deserialized_json(
JSONDocument::Deserialize(Slice(serialized)));
ASSERT_TRUE(deserialized_json != nullptr);
AssertSampleJSON(*deserialized_json);
// deserialization failure
ASSERT_TRUE(JSONDocument::Deserialize(
Slice(serialized.data(), serialized.size() - 10)) == nullptr);
}
TEST_F(JSONDocumentTest, OperatorEqualsTest) {
// kNull
ASSERT_TRUE(JSONDocument() == JSONDocument());
// kBool
ASSERT_TRUE(JSONDocument(false) != JSONDocument());
ASSERT_TRUE(JSONDocument(false) == JSONDocument(false));
ASSERT_TRUE(JSONDocument(true) == JSONDocument(true));
ASSERT_TRUE(JSONDocument(false) != JSONDocument(true));
// kString
ASSERT_TRUE(JSONDocument("test") != JSONDocument());
ASSERT_TRUE(JSONDocument("test") == JSONDocument("test"));
// kInt64
ASSERT_TRUE(JSONDocument(static_cast<int64_t>(15)) != JSONDocument());
ASSERT_TRUE(JSONDocument(static_cast<int64_t>(15)) !=
JSONDocument(static_cast<int64_t>(14)));
ASSERT_TRUE(JSONDocument(static_cast<int64_t>(15)) ==
JSONDocument(static_cast<int64_t>(15)));
std::unique_ptr<JSONDocument> arrayWithInt8Doc(
JSONDocument::ParseJSON("[8]"));
ASSERT_TRUE(arrayWithInt8Doc != nullptr);
ASSERT_TRUE(arrayWithInt8Doc->IsArray());
ASSERT_TRUE((*arrayWithInt8Doc)[0].IsInt64());
ASSERT_TRUE((*arrayWithInt8Doc)[0] == JSONDocument(static_cast<int64_t>(8)));
std::unique_ptr<JSONDocument> arrayWithInt16Doc(
JSONDocument::ParseJSON("[512]"));
ASSERT_TRUE(arrayWithInt16Doc != nullptr);
ASSERT_TRUE(arrayWithInt16Doc->IsArray());
ASSERT_TRUE((*arrayWithInt16Doc)[0].IsInt64());
ASSERT_TRUE((*arrayWithInt16Doc)[0] ==
JSONDocument(static_cast<int64_t>(512)));
std::unique_ptr<JSONDocument> arrayWithInt32Doc(
JSONDocument::ParseJSON("[1000000]"));
ASSERT_TRUE(arrayWithInt32Doc != nullptr);
ASSERT_TRUE(arrayWithInt32Doc->IsArray());
ASSERT_TRUE((*arrayWithInt32Doc)[0].IsInt64());
ASSERT_TRUE((*arrayWithInt32Doc)[0] ==
JSONDocument(static_cast<int64_t>(1000000)));
// kDouble
ASSERT_TRUE(JSONDocument(15.) != JSONDocument());
ASSERT_TRUE(JSONDocument(15.) != JSONDocument(14.));
ASSERT_TRUE(JSONDocument(15.) == JSONDocument(15.));
}
TEST_F(JSONDocumentTest, JSONDocumentBuilderTest) {
std::unique_ptr<JSONDocument> parsedArray(
JSONDocument::ParseJSON("[1, [123, \"a\", \"b\"], {\"b\":\"c\"}]"));
ASSERT_TRUE(parsedArray != nullptr);
JSONDocumentBuilder builder;
ASSERT_TRUE(builder.WriteStartArray());
ASSERT_TRUE(builder.WriteJSONDocument(1));
ASSERT_TRUE(builder.WriteStartArray());
ASSERT_TRUE(builder.WriteJSONDocument(123));
ASSERT_TRUE(builder.WriteJSONDocument("a"));
ASSERT_TRUE(builder.WriteJSONDocument("b"));
ASSERT_TRUE(builder.WriteEndArray());
ASSERT_TRUE(builder.WriteStartObject());
ASSERT_TRUE(builder.WriteKeyValue("b", "c"));
ASSERT_TRUE(builder.WriteEndObject());
ASSERT_TRUE(builder.WriteEndArray());
ASSERT_TRUE(*parsedArray == builder.GetJSONDocument());
}
TEST_F(JSONDocumentTest, OwnershipTest) {
std::unique_ptr<JSONDocument> parsed(
JSONDocument::ParseJSON(kSampleJSON.c_str()));
ASSERT_TRUE(parsed != nullptr);
ASSERT_TRUE(parsed->IsOwner());
// Copy constructor from owner -> owner
JSONDocument copy_constructor(*parsed);
ASSERT_TRUE(copy_constructor.IsOwner());
// Copy constructor from non-owner -> non-owner
JSONDocument non_owner((*parsed)["properties"]);
ASSERT_TRUE(!non_owner.IsOwner());
// Move constructor from owner -> owner
JSONDocument moved_from_owner(std::move(copy_constructor));
ASSERT_TRUE(moved_from_owner.IsOwner());
// Move constructor from non-owner -> non-owner
JSONDocument moved_from_non_owner(std::move(non_owner));
ASSERT_TRUE(!moved_from_non_owner.IsOwner());
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "SKIPPED as JSONDocument is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

478
utilities/geodb/geodb_impl.cc
View File

@ -1,478 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#include "utilities/geodb/geodb_impl.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <limits>
#include <map>
#include <string>
#include <vector>
#include "util/coding.h"
#include "util/filename.h"
#include "util/string_util.h"
//
// There are two types of keys. The first type of key-values
// maps a geo location to the set of object ids and their values.
// Table 1
// key : p + : + $quadkey + : + $id +
// : + $latitude + : + $longitude
// value : value of the object
// This table can be used to find all objects that reside near
// a specified geolocation.
//
// Table 2
// key : 'k' + : + $id
// value: $quadkey
namespace rocksdb {
const double GeoDBImpl::PI = 3.141592653589793;
const double GeoDBImpl::EarthRadius = 6378137;
const double GeoDBImpl::MinLatitude = -85.05112878;
const double GeoDBImpl::MaxLatitude = 85.05112878;
const double GeoDBImpl::MinLongitude = -180;
const double GeoDBImpl::MaxLongitude = 180;
GeoDBImpl::GeoDBImpl(DB* db, const GeoDBOptions& options) :
GeoDB(db, options), db_(db), options_(options) {
}
GeoDBImpl::~GeoDBImpl() {
}
Status GeoDBImpl::Insert(const GeoObject& obj) {
WriteBatch batch;
// It is possible that this id is already associated with
// with a different position. We first have to remove that
// association before we can insert the new one.
// remove existing object, if it exists
GeoObject old;
Status status = GetById(obj.id, &old);
if (status.ok()) {
assert(obj.id.compare(old.id) == 0);
std::string quadkey = PositionToQuad(old.position, Detail);
std::string key1 = MakeKey1(old.position, old.id, quadkey);
std::string key2 = MakeKey2(old.id);
batch.Delete(Slice(key1));
batch.Delete(Slice(key2));
} else if (status.IsNotFound()) {
// What if another thread is trying to insert the same ID concurrently?
} else {
return status;
}
// insert new object
std::string quadkey = PositionToQuad(obj.position, Detail);
std::string key1 = MakeKey1(obj.position, obj.id, quadkey);
std::string key2 = MakeKey2(obj.id);
batch.Put(Slice(key1), Slice(obj.value));
batch.Put(Slice(key2), Slice(quadkey));
return db_->Write(woptions_, &batch);
}
Status GeoDBImpl::GetByPosition(const GeoPosition& pos,
const Slice& id,
std::string* value) {
std::string quadkey = PositionToQuad(pos, Detail);
std::string key1 = MakeKey1(pos, id, quadkey);
return db_->Get(roptions_, Slice(key1), value);
}
Status GeoDBImpl::GetById(const Slice& id, GeoObject* object) {
Status status;
std::string quadkey;
// create an iterator so that we can get a consistent picture
// of the database.
Iterator* iter = db_->NewIterator(roptions_);
// create key for table2
std::string kt = MakeKey2(id);
Slice key2(kt);
iter->Seek(key2);
if (iter->Valid() && iter->status().ok()) {
if (iter->key().compare(key2) == 0) {
quadkey = iter->value().ToString();
}
}
if (quadkey.size() == 0) {
delete iter;
return Status::NotFound(key2);
}
//
// Seek to the quadkey + id prefix
//
std::string prefix = MakeKey1Prefix(quadkey, id);
iter->Seek(Slice(prefix));
assert(iter->Valid());
if (!iter->Valid() || !iter->status().ok()) {
delete iter;
return Status::NotFound();
}
// split the key into p + quadkey + id + lat + lon
Slice key = iter->key();
std::vector<std::string> parts = StringSplit(key.ToString(), ':');
assert(parts.size() == 5);
assert(parts[0] == "p");
assert(parts[1] == quadkey);
assert(parts[2] == id);
// fill up output parameters
object->position.latitude = atof(parts[3].c_str());
object->position.longitude = atof(parts[4].c_str());
object->id = id.ToString(); // this is redundant
object->value = iter->value().ToString();
delete iter;
return Status::OK();
}
Status GeoDBImpl::Remove(const Slice& id) {
// Read the object from the database
GeoObject obj;
Status status = GetById(id, &obj);
if (!status.ok()) {
return status;
}
// remove the object by atomically deleting it from both tables
std::string quadkey = PositionToQuad(obj.position, Detail);
std::string key1 = MakeKey1(obj.position, obj.id, quadkey);
std::string key2 = MakeKey2(obj.id);
WriteBatch batch;
batch.Delete(Slice(key1));
batch.Delete(Slice(key2));
return db_->Write(woptions_, &batch);
}
class GeoIteratorImpl : public GeoIterator {
private:
std::vector<GeoObject> values_;
std::vector<GeoObject>::iterator iter_;
public:
explicit GeoIteratorImpl(std::vector<GeoObject> values)
: values_(std::move(values)) {
iter_ = values_.begin();
}
virtual void Next() override;
virtual bool Valid() const override;
virtual const GeoObject& geo_object() override;
virtual Status status() const override;
};
class GeoErrorIterator : public GeoIterator {
private:
Status status_;
public:
explicit GeoErrorIterator(Status s) : status_(s) {}
virtual void Next() override {};
virtual bool Valid() const override { return false; }
virtual const GeoObject& geo_object() override {
GeoObject* g = new GeoObject();
return *g;
}
virtual Status status() const override { return status_; }
};
void GeoIteratorImpl::Next() {
assert(Valid());
iter_++;
}
bool GeoIteratorImpl::Valid() const {
return iter_ != values_.end();
}
const GeoObject& GeoIteratorImpl::geo_object() {
assert(Valid());
return *iter_;
}
Status GeoIteratorImpl::status() const {
return Status::OK();
}
GeoIterator* GeoDBImpl::SearchRadial(const GeoPosition& pos,
double radius,
int number_of_values) {
std::vector<GeoObject> values;
// Gather all bounding quadkeys
std::vector<std::string> qids;
Status s = searchQuadIds(pos, radius, &qids);
if (!s.ok()) {
return new GeoErrorIterator(s);
}
// create an iterator
Iterator* iter = db_->NewIterator(ReadOptions());
// Process each prospective quadkey
for (const std::string& qid : qids) {
// The user is interested in only these many objects.
if (number_of_values == 0) {
break;
}
// convert quadkey to db key prefix
std::string dbkey = MakeQuadKeyPrefix(qid);
for (iter->Seek(dbkey);
number_of_values > 0 && iter->Valid() && iter->status().ok();
iter->Next()) {
// split the key into p + quadkey + id + lat + lon
Slice key = iter->key();
std::vector<std::string> parts = StringSplit(key.ToString(), ':');
assert(parts.size() == 5);
assert(parts[0] == "p");
std::string* quadkey = &parts[1];
// If the key we are looking for is a prefix of the key
// we found from the database, then this is one of the keys
// we are looking for.
auto res = std::mismatch(qid.begin(), qid.end(), quadkey->begin());
if (res.first == qid.end()) {
GeoPosition obj_pos(atof(parts[3].c_str()), atof(parts[4].c_str()));
GeoObject obj(obj_pos, parts[2], iter->value().ToString());
values.push_back(obj);
number_of_values--;
} else {
break;
}
}
}
delete iter;
return new GeoIteratorImpl(std::move(values));
}
std::string GeoDBImpl::MakeKey1(const GeoPosition& pos, Slice id,
std::string quadkey) {
std::string lat = rocksdb::ToString(pos.latitude);
std::string lon = rocksdb::ToString(pos.longitude);
std::string key = "p:";
key.reserve(5 + quadkey.size() + id.size() + lat.size() + lon.size());
key.append(quadkey);
key.append(":");
key.append(id.ToString());
key.append(":");
key.append(lat);
key.append(":");
key.append(lon);
return key;
}
std::string GeoDBImpl::MakeKey2(Slice id) {
std::string key = "k:";
key.append(id.ToString());
return key;
}
std::string GeoDBImpl::MakeKey1Prefix(std::string quadkey,
Slice id) {
std::string key = "p:";
key.reserve(4 + quadkey.size() + id.size());
key.append(quadkey);
key.append(":");
key.append(id.ToString());
key.append(":");
return key;
}
std::string GeoDBImpl::MakeQuadKeyPrefix(std::string quadkey) {
std::string key = "p:";
key.append(quadkey);
return key;
}
// convert degrees to radians
double GeoDBImpl::radians(double x) {
return (x * PI) / 180;
}
// convert radians to degrees
double GeoDBImpl::degrees(double x) {
return (x * 180) / PI;
}
// convert a gps location to quad coordinate
std::string GeoDBImpl::PositionToQuad(const GeoPosition& pos,
int levelOfDetail) {
Pixel p = PositionToPixel(pos, levelOfDetail);
Tile tile = PixelToTile(p);
return TileToQuadKey(tile, levelOfDetail);
}
GeoPosition GeoDBImpl::displaceLatLon(double lat, double lon,
double deltay, double deltax) {
double dLat = deltay / EarthRadius;
double dLon = deltax / (EarthRadius * cos(radians(lat)));
return GeoPosition(lat + degrees(dLat),
lon + degrees(dLon));
}
//
// Return the distance between two positions on the earth
//
double GeoDBImpl::distance(double lat1, double lon1,
double lat2, double lon2) {
double lon = radians(lon2 - lon1);
double lat = radians(lat2 - lat1);
double a = (sin(lat / 2) * sin(lat / 2)) +
cos(radians(lat1)) * cos(radians(lat2)) *
(sin(lon / 2) * sin(lon / 2));
double angle = 2 * atan2(sqrt(a), sqrt(1 - a));
return angle * EarthRadius;
}
//
// Returns all the quadkeys inside the search range
//
Status GeoDBImpl::searchQuadIds(const GeoPosition& position,
double radius,
std::vector<std::string>* quadKeys) {
// get the outline of the search square
GeoPosition topLeftPos = boundingTopLeft(position, radius);
GeoPosition bottomRightPos = boundingBottomRight(position, radius);
Pixel topLeft = PositionToPixel(topLeftPos, Detail);
Pixel bottomRight = PositionToPixel(bottomRightPos, Detail);
// how many level of details to look for
int numberOfTilesAtMaxDepth = static_cast<int>(std::floor((bottomRight.x - topLeft.x) / 256));
int zoomLevelsToRise = static_cast<int>(std::floor(std::log(numberOfTilesAtMaxDepth) / std::log(2)));
zoomLevelsToRise++;
int levels = std::max(0, Detail - zoomLevelsToRise);
quadKeys->push_back(PositionToQuad(GeoPosition(topLeftPos.latitude,
topLeftPos.longitude),
levels));
quadKeys->push_back(PositionToQuad(GeoPosition(topLeftPos.latitude,
bottomRightPos.longitude),
levels));
quadKeys->push_back(PositionToQuad(GeoPosition(bottomRightPos.latitude,
topLeftPos.longitude),
levels));
quadKeys->push_back(PositionToQuad(GeoPosition(bottomRightPos.latitude,
bottomRightPos.longitude),
levels));
return Status::OK();
}
// Determines the ground resolution (in meters per pixel) at a specified
// latitude and level of detail.
// Latitude (in degrees) at which to measure the ground resolution.
// Level of detail, from 1 (lowest detail) to 23 (highest detail).
// Returns the ground resolution, in meters per pixel.
double GeoDBImpl::GroundResolution(double latitude, int levelOfDetail) {
latitude = clip(latitude, MinLatitude, MaxLatitude);
return cos(latitude * PI / 180) * 2 * PI * EarthRadius /
MapSize(levelOfDetail);
}
// Converts a point from latitude/longitude WGS-84 coordinates (in degrees)
// into pixel XY coordinates at a specified level of detail.
GeoDBImpl::Pixel GeoDBImpl::PositionToPixel(const GeoPosition& pos,
int levelOfDetail) {
double latitude = clip(pos.latitude, MinLatitude, MaxLatitude);
double x = (pos.longitude + 180) / 360;
double sinLatitude = sin(latitude * PI / 180);
double y = 0.5 - std::log((1 + sinLatitude) / (1 - sinLatitude)) / (4 * PI);
double mapSize = MapSize(levelOfDetail);
double X = std::floor(clip(x * mapSize + 0.5, 0, mapSize - 1));
double Y = std::floor(clip(y * mapSize + 0.5, 0, mapSize - 1));
return Pixel((unsigned int)X, (unsigned int)Y);
}
GeoPosition GeoDBImpl::PixelToPosition(const Pixel& pixel, int levelOfDetail) {
double mapSize = MapSize(levelOfDetail);
double x = (clip(pixel.x, 0, mapSize - 1) / mapSize) - 0.5;
double y = 0.5 - (clip(pixel.y, 0, mapSize - 1) / mapSize);
double latitude = 90 - 360 * atan(exp(-y * 2 * PI)) / PI;
double longitude = 360 * x;
return GeoPosition(latitude, longitude);
}
// Converts a Pixel to a Tile
GeoDBImpl::Tile GeoDBImpl::PixelToTile(const Pixel& pixel) {
unsigned int tileX = static_cast<unsigned int>(std::floor(pixel.x / 256));
unsigned int tileY = static_cast<unsigned int>(std::floor(pixel.y / 256));
return Tile(tileX, tileY);
}
GeoDBImpl::Pixel GeoDBImpl::TileToPixel(const Tile& tile) {
unsigned int pixelX = tile.x * 256;
unsigned int pixelY = tile.y * 256;
return Pixel(pixelX, pixelY);
}
// Convert a Tile to a quadkey
std::string GeoDBImpl::TileToQuadKey(const Tile& tile, int levelOfDetail) {
std::stringstream quadKey;
for (int i = levelOfDetail; i > 0; i--) {
char digit = '0';
int mask = 1 << (i - 1);
if ((tile.x & mask) != 0) {
digit++;
}
if ((tile.y & mask) != 0) {
digit++;
digit++;
}
quadKey << digit;
}
return quadKey.str();
}
//
// Convert a quadkey to a tile and its level of detail
//
void GeoDBImpl::QuadKeyToTile(std::string quadkey, Tile* tile,
int* levelOfDetail) {
tile->x = tile->y = 0;
*levelOfDetail = static_cast<int>(quadkey.size());
const char* key = reinterpret_cast<const char*>(quadkey.c_str());
for (int i = *levelOfDetail; i > 0; i--) {
int mask = 1 << (i - 1);
switch (key[*levelOfDetail - i]) {
case '0':
break;
case '1':
tile->x |= mask;
break;
case '2':
tile->y |= mask;
break;
case '3':
tile->x |= mask;
tile->y |= mask;
break;
default:
std::stringstream msg;
msg << quadkey;
msg << " Invalid QuadKey.";
throw std::runtime_error(msg.str());
}
}
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

185
utilities/geodb/geodb_impl.h
View File

@ -1,185 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#pragma once
#include <algorithm>
#include <cmath>
#include <string>
#include <sstream>
#include <stdexcept>
#include <vector>
#include "rocksdb/utilities/geo_db.h"
#include "rocksdb/utilities/stackable_db.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
namespace rocksdb {
// A specific implementation of GeoDB
class GeoDBImpl : public GeoDB {
public:
GeoDBImpl(DB* db, const GeoDBOptions& options);
~GeoDBImpl();
// Associate the GPS location with the identified by 'id'. The value
// is a blob that is associated with this object.
virtual Status Insert(const GeoObject& object) override;
// Retrieve the value of the object located at the specified GPS
// location and is identified by the 'id'.
virtual Status GetByPosition(const GeoPosition& pos, const Slice& id,
std::string* value) override;
// Retrieve the value of the object identified by the 'id'. This method
// could be potentially slower than GetByPosition
virtual Status GetById(const Slice& id, GeoObject* object) override;
// Delete the specified object
virtual Status Remove(const Slice& id) override;
// Returns a list of all items within a circular radius from the
// specified gps location
virtual GeoIterator* SearchRadial(const GeoPosition& pos, double radius,
int number_of_values) override;
private:
DB* db_;
const GeoDBOptions options_;
const WriteOptions woptions_;
const ReadOptions roptions_;
// MSVC requires the definition for this static const to be in .CC file
// The value of PI
static const double PI;
// convert degrees to radians
static double radians(double x);
// convert radians to degrees
static double degrees(double x);
// A pixel class that captures X and Y coordinates
class Pixel {
public:
unsigned int x;
unsigned int y;
Pixel(unsigned int a, unsigned int b) :
x(a), y(b) {
}
};
// A Tile in the geoid
class Tile {
public:
unsigned int x;
unsigned int y;
Tile(unsigned int a, unsigned int b) :
x(a), y(b) {
}
};
// convert a gps location to quad coordinate
static std::string PositionToQuad(const GeoPosition& pos, int levelOfDetail);
// arbitrary constant use for WGS84 via
// http://en.wikipedia.org/wiki/World_Geodetic_System
// http://mathforum.org/library/drmath/view/51832.html
// http://msdn.microsoft.com/en-us/library/bb259689.aspx
// http://www.tuicool.com/articles/NBrE73
//
const int Detail = 23;
// MSVC requires the definition for this static const to be in .CC file
static const double EarthRadius;
static const double MinLatitude;
static const double MaxLatitude;
static const double MinLongitude;
static const double MaxLongitude;
// clips a number to the specified minimum and maximum values.
static double clip(double n, double minValue, double maxValue) {
return fmin(fmax(n, minValue), maxValue);
}
// Determines the map width and height (in pixels) at a specified level
// of detail, from 1 (lowest detail) to 23 (highest detail).
// Returns the map width and height in pixels.
static unsigned int MapSize(int levelOfDetail) {
return (unsigned int)(256 << levelOfDetail);
}
// Determines the ground resolution (in meters per pixel) at a specified
// latitude and level of detail.
// Latitude (in degrees) at which to measure the ground resolution.
// Level of detail, from 1 (lowest detail) to 23 (highest detail).
// Returns the ground resolution, in meters per pixel.
static double GroundResolution(double latitude, int levelOfDetail);
// Converts a point from latitude/longitude WGS-84 coordinates (in degrees)
// into pixel XY coordinates at a specified level of detail.
static Pixel PositionToPixel(const GeoPosition& pos, int levelOfDetail);
static GeoPosition PixelToPosition(const Pixel& pixel, int levelOfDetail);
// Converts a Pixel to a Tile
static Tile PixelToTile(const Pixel& pixel);
static Pixel TileToPixel(const Tile& tile);
// Convert a Tile to a quadkey
static std::string TileToQuadKey(const Tile& tile, int levelOfDetail);
// Convert a quadkey to a tile and its level of detail
static void QuadKeyToTile(std::string quadkey, Tile* tile,
int *levelOfDetail);
// Return the distance between two positions on the earth
static double distance(double lat1, double lon1,
double lat2, double lon2);
static GeoPosition displaceLatLon(double lat, double lon,
double deltay, double deltax);
//
// Returns the top left position after applying the delta to
// the specified position
//
static GeoPosition boundingTopLeft(const GeoPosition& in, double radius) {
return displaceLatLon(in.latitude, in.longitude, -radius, -radius);
}
//
// Returns the bottom right position after applying the delta to
// the specified position
static GeoPosition boundingBottomRight(const GeoPosition& in,
double radius) {
return displaceLatLon(in.latitude, in.longitude, radius, radius);
}
//
// Get all quadkeys within a radius of a specified position
//
Status searchQuadIds(const GeoPosition& position,
double radius,
std::vector<std::string>* quadKeys);
//
// Create keys for accessing rocksdb table(s)
//
static std::string MakeKey1(const GeoPosition& pos,
Slice id,
std::string quadkey);
static std::string MakeKey2(Slice id);
static std::string MakeKey1Prefix(std::string quadkey,
Slice id);
static std::string MakeQuadKeyPrefix(std::string quadkey);
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

201
utilities/geodb/geodb_test.cc
View File

@ -1,201 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#include "utilities/geodb/geodb_impl.h"
#include <cctype>
#include "util/testharness.h"
namespace rocksdb {
class GeoDBTest : public testing::Test {
public:
static const std::string kDefaultDbName;
static Options options;
DB* db;
GeoDB* geodb;
GeoDBTest() {
GeoDBOptions geodb_options;
EXPECT_OK(DestroyDB(kDefaultDbName, options));
options.create_if_missing = true;
Status status = DB::Open(options, kDefaultDbName, &db);
geodb = new GeoDBImpl(db, geodb_options);
}
~GeoDBTest() {
delete geodb;
}
GeoDB* getdb() {
return geodb;
}
};
const std::string GeoDBTest::kDefaultDbName =
test::PerThreadDBPath("geodb_test");
Options GeoDBTest::options = Options();
// Insert, Get and Remove
TEST_F(GeoDBTest, SimpleTest) {
GeoPosition pos1(100, 101);
std::string id1("id1");
std::string value1("value1");
// insert first object into database
GeoObject obj1(pos1, id1, value1);
Status status = getdb()->Insert(obj1);
ASSERT_TRUE(status.ok());
// insert second object into database
GeoPosition pos2(200, 201);
std::string id2("id2");
std::string value2 = "value2";
GeoObject obj2(pos2, id2, value2);
status = getdb()->Insert(obj2);
ASSERT_TRUE(status.ok());
// retrieve first object using position
std::string value;
status = getdb()->GetByPosition(pos1, Slice(id1), &value);
ASSERT_TRUE(status.ok());
ASSERT_EQ(value, value1);
// retrieve first object using id
GeoObject obj;
status = getdb()->GetById(Slice(id1), &obj);
ASSERT_TRUE(status.ok());
ASSERT_EQ(obj.position.latitude, 100);
ASSERT_EQ(obj.position.longitude, 101);
ASSERT_EQ(obj.id.compare(id1), 0);
ASSERT_EQ(obj.value, value1);
// delete first object
status = getdb()->Remove(Slice(id1));
ASSERT_TRUE(status.ok());
status = getdb()->GetByPosition(pos1, Slice(id1), &value);
ASSERT_TRUE(status.IsNotFound());
status = getdb()->GetById(id1, &obj);
ASSERT_TRUE(status.IsNotFound());
// check that we can still find second object
status = getdb()->GetByPosition(pos2, id2, &value);
ASSERT_TRUE(status.ok());
ASSERT_EQ(value, value2);
status = getdb()->GetById(id2, &obj);
ASSERT_TRUE(status.ok());
}
// Search.
// Verify distances via http://www.stevemorse.org/nearest/distance.php
TEST_F(GeoDBTest, Search) {
GeoPosition pos1(45, 45);
std::string id1("mid1");
std::string value1 = "midvalue1";
// insert object at 45 degree latitude
GeoObject obj1(pos1, id1, value1);
Status status = getdb()->Insert(obj1);
ASSERT_TRUE(status.ok());
// search all objects centered at 46 degree latitude with
// a radius of 200 kilometers. We should find the one object that
// we inserted earlier.
GeoIterator* iter1 = getdb()->SearchRadial(GeoPosition(46, 46), 200000);
ASSERT_TRUE(status.ok());
ASSERT_EQ(iter1->geo_object().value, "midvalue1");
uint32_t size = 0;
while (iter1->Valid()) {
GeoObject obj;
status = getdb()->GetById(Slice(id1), &obj);
ASSERT_TRUE(status.ok());
ASSERT_EQ(iter1->geo_object().position.latitude, pos1.latitude);
ASSERT_EQ(iter1->geo_object().position.longitude, pos1.longitude);
ASSERT_EQ(iter1->geo_object().id.compare(id1), 0);
ASSERT_EQ(iter1->geo_object().value, value1);
size++;
iter1->Next();
ASSERT_TRUE(!iter1->Valid());
}
ASSERT_EQ(size, 1U);
delete iter1;
// search all objects centered at 46 degree latitude with
// a radius of 2 kilometers. There should be none.
GeoIterator* iter2 = getdb()->SearchRadial(GeoPosition(46, 46), 2);
ASSERT_TRUE(status.ok());
ASSERT_FALSE(iter2->Valid());
delete iter2;
}
TEST_F(GeoDBTest, DifferentPosInSameQuadkey) {
// insert obj1 into database
GeoPosition pos1(40.00001, 116.00001);
std::string id1("12");
std::string value1("value1");
GeoObject obj1(pos1, id1, value1);
Status status = getdb()->Insert(obj1);
ASSERT_TRUE(status.ok());
// insert obj2 into database
GeoPosition pos2(40.00002, 116.00002);
std::string id2("123");
std::string value2 = "value2";
GeoObject obj2(pos2, id2, value2);
status = getdb()->Insert(obj2);
ASSERT_TRUE(status.ok());
// get obj1's quadkey
ReadOptions opt;
PinnableSlice quadkey1;
status = getdb()->Get(opt, getdb()->DefaultColumnFamily(), "k:" + id1, &quadkey1);
ASSERT_TRUE(status.ok());
// get obj2's quadkey
PinnableSlice quadkey2;
status = getdb()->Get(opt, getdb()->DefaultColumnFamily(), "k:" + id2, &quadkey2);
ASSERT_TRUE(status.ok());
// obj1 and obj2 have the same quadkey
ASSERT_EQ(quadkey1, quadkey2);
// get obj1 by id, and check value
GeoObject obj;
status = getdb()->GetById(Slice(id1), &obj);
ASSERT_TRUE(status.ok());
ASSERT_EQ(obj.position.latitude, pos1.latitude);
ASSERT_EQ(obj.position.longitude, pos1.longitude);
ASSERT_EQ(obj.id.compare(id1), 0);
ASSERT_EQ(obj.value, value1);
// get obj2 by id, and check value
status = getdb()->GetById(Slice(id2), &obj);
ASSERT_TRUE(status.ok());
ASSERT_EQ(obj.position.latitude, pos2.latitude);
ASSERT_EQ(obj.position.longitude, pos2.longitude);
ASSERT_EQ(obj.id.compare(id2), 0);
ASSERT_EQ(obj.value, value2);
}
} // namespace rocksdb
int main(int argc, char* argv[]) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main() {
fprintf(stderr, "SKIPPED\n");
return 0;
}
#endif // !ROCKSDB_LITE

14
utilities/redis/README
View File

@ -1,14 +0,0 @@
This folder defines a REDIS-style interface for Rocksdb.
Right now it is written as a simple tag-on in the rocksdb::RedisLists class.
It implements Redis Lists, and supports only the "non-blocking operations".
Internally, the set of lists are stored in a rocksdb database, mapping keys to
values. Each "value" is the list itself, storing a sequence of "elements".
Each element is stored as a 32-bit-integer, followed by a sequence of bytes.
The 32-bit-integer represents the length of the element (that is, the number
of bytes that follow). And then that many bytes follow.
NOTE: This README file may be old. See the actual redis_lists.cc file for
definitive details on the implementation. There should be a header at the top
of that file, explaining a bit of the implementation details.

22
utilities/redis/redis_list_exception.h
View File

@ -1,22 +0,0 @@
/**
* A simple structure for exceptions in RedisLists.
*
* @author Deon Nicholas (dnicholas@fb.com)
* Copyright 2013 Facebook
*/
#pragma once
#ifndef ROCKSDB_LITE
#include <exception>
namespace rocksdb {
class RedisListException: public std::exception {
public:
const char* what() const throw() override {
return "Invalid operation or corrupt data in Redis List.";
}
};
} // namespace rocksdb
#endif

309
utilities/redis/redis_list_iterator.h
View File

@ -1,309 +0,0 @@
// Copyright 2013 Facebook
/**
* RedisListIterator:
* An abstraction over the "list" concept (e.g.: for redis lists).
* Provides functionality to read, traverse, edit, and write these lists.
*
* Upon construction, the RedisListIterator is given a block of list data.
* Internally, it stores a pointer to the data and a pointer to current item.
* It also stores a "result" list that will be mutated over time.
*
* Traversal and mutation are done by "forward iteration".
* The Push() and Skip() methods will advance the iterator to the next item.
* However, Push() will also "write the current item to the result".
* Skip() will simply move to next item, causing current item to be dropped.
*
* Upon completion, the result (accessible by WriteResult()) will be saved.
* All "skipped" items will be gone; all "pushed" items will remain.
*
* @throws Any of the operations may throw a RedisListException if an invalid
* operation is performed or if the data is found to be corrupt.
*
* @notes By default, if WriteResult() is called part-way through iteration,
* it will automatically advance the iterator to the end, and Keep()
* all items that haven't been traversed yet. This may be subject
* to review.
*
* @notes Can access the "current" item via GetCurrent(), and other
* list-specific information such as Length().
*
* @notes The internal representation is due to change at any time. Presently,
* the list is represented as follows:
* - 32-bit integer header: the number of items in the list
* - For each item:
* - 32-bit int (n): the number of bytes representing this item
* - n bytes of data: the actual data.
*
* @author Deon Nicholas (dnicholas@fb.com)
*/
#pragma once
#ifndef ROCKSDB_LITE
#include <string>
#include "redis_list_exception.h"
#include "rocksdb/slice.h"
#include "util/coding.h"
namespace rocksdb {
/// An abstraction over the "list" concept.
/// All operations may throw a RedisListException
class RedisListIterator {
public:
/// Construct a redis-list-iterator based on data.
/// If the data is non-empty, it must formatted according to @notes above.
///
/// If the data is valid, we can assume the following invariant(s):
/// a) length_, num_bytes_ are set correctly.
/// b) cur_byte_ always refers to the start of the current element,
/// just before the bytes that specify element length.
/// c) cur_elem_ is always the index of the current element.
/// d) cur_elem_length_ is always the number of bytes in current element,
/// excluding the 4-byte header itself.
/// e) result_ will always contain data_[0..cur_byte_) and a header
/// f) Whenever corrupt data is encountered or an invalid operation is
/// attempted, a RedisListException will immediately be thrown.
explicit RedisListIterator(const std::string& list_data)
: data_(list_data.data()),
num_bytes_(static_cast<uint32_t>(list_data.size())),
cur_byte_(0),
cur_elem_(0),
cur_elem_length_(0),
length_(0),
result_() {
// Initialize the result_ (reserve enough space for header)
InitializeResult();
// Parse the data only if it is not empty.
if (num_bytes_ == 0) {
return;
}
// If non-empty, but less than 4 bytes, data must be corrupt
if (num_bytes_ < sizeof(length_)) {
ThrowError("Corrupt header."); // Will break control flow
}
// Good. The first bytes specify the number of elements
length_ = DecodeFixed32(data_);
cur_byte_ = sizeof(length_);
// If we have at least one element, point to that element.
// Also, read the first integer of the element (specifying the size),
// if possible.
if (length_ > 0) {
if (cur_byte_ + sizeof(cur_elem_length_) <= num_bytes_) {
cur_elem_length_ = DecodeFixed32(data_+cur_byte_);
} else {
ThrowError("Corrupt data for first element.");
}
}
// At this point, we are fully set-up.
// The invariants described in the header should now be true.
}
/// Reserve some space for the result_.
/// Equivalent to result_.reserve(bytes).
void Reserve(int bytes) {
result_.reserve(bytes);
}
/// Go to next element in data file.
/// Also writes the current element to result_.
RedisListIterator& Push() {
WriteCurrentElement();
MoveNext();
return *this;
}
/// Go to next element in data file.
/// Drops/skips the current element. It will not be written to result_.
RedisListIterator& Skip() {
MoveNext();
--length_; // One less item
--cur_elem_; // We moved one forward, but index did not change
return *this;
}
/// Insert elem into the result_ (just BEFORE the current element / byte)
/// Note: if Done() (i.e.: iterator points to end), this will append elem.
void InsertElement(const Slice& elem) {
// Ensure we are in a valid state
CheckErrors();
const int kOrigSize = static_cast<int>(result_.size());
result_.resize(kOrigSize + SizeOf(elem));
EncodeFixed32(result_.data() + kOrigSize,
static_cast<uint32_t>(elem.size()));
memcpy(result_.data() + kOrigSize + sizeof(uint32_t), elem.data(),
elem.size());
++length_;
++cur_elem_;
}
/// Access the current element, and save the result into *curElem
void GetCurrent(Slice* curElem) {
// Ensure we are in a valid state
CheckErrors();
// Ensure that we are not past the last element.
if (Done()) {
ThrowError("Invalid dereferencing.");
}
// Dereference the element
*curElem = Slice(data_+cur_byte_+sizeof(cur_elem_length_),
cur_elem_length_);
}
// Number of elements
int Length() const {
return length_;
}
// Number of bytes in the final representation (i.e: WriteResult().size())
int Size() const {
// result_ holds the currently written data
// data_[cur_byte..num_bytes-1] is the remainder of the data
return static_cast<int>(result_.size() + (num_bytes_ - cur_byte_));
}
// Reached the end?
bool Done() const {
return cur_byte_ >= num_bytes_ || cur_elem_ >= length_;
}
/// Returns a string representing the final, edited, data.
/// Assumes that all bytes of data_ in the range [0,cur_byte_) have been read
/// and that result_ contains this data.
/// The rest of the data must still be written.
/// So, this method ADVANCES THE ITERATOR TO THE END before writing.
Slice WriteResult() {
CheckErrors();
// The header should currently be filled with dummy data (0's)
// Correctly update the header.
// Note, this is safe since result_ is a vector (guaranteed contiguous)
EncodeFixed32(&result_[0],length_);
// Append the remainder of the data to the result.
result_.insert(result_.end(),data_+cur_byte_, data_ +num_bytes_);
// Seek to end of file
cur_byte_ = num_bytes_;
cur_elem_ = length_;
cur_elem_length_ = 0;
// Return the result
return Slice(result_.data(),result_.size());
}
public: // Static public functions
/// An upper-bound on the amount of bytes needed to store this element.
/// This is used to hide representation information from the client.
/// E.G. This can be used to compute the bytes we want to Reserve().
static uint32_t SizeOf(const Slice& elem) {
// [Integer Length . Data]
return static_cast<uint32_t>(sizeof(uint32_t) + elem.size());
}
private: // Private functions
/// Initializes the result_ string.
/// It will fill the first few bytes with 0's so that there is
/// enough space for header information when we need to write later.
/// Currently, "header information" means: the length (number of elements)
/// Assumes that result_ is empty to begin with
void InitializeResult() {
assert(result_.empty()); // Should always be true.
result_.resize(sizeof(uint32_t),0); // Put a block of 0's as the header
}
/// Go to the next element (used in Push() and Skip())
void MoveNext() {
CheckErrors();
// Check to make sure we are not already in a finished state
if (Done()) {
ThrowError("Attempting to iterate past end of list.");
}
// Move forward one element.
cur_byte_ += sizeof(cur_elem_length_) + cur_elem_length_;
++cur_elem_;
// If we are at the end, finish
if (Done()) {
cur_elem_length_ = 0;
return;
}
// Otherwise, we should be able to read the new element's length
if (cur_byte_ + sizeof(cur_elem_length_) > num_bytes_) {
ThrowError("Corrupt element data.");
}
// Set the new element's length
cur_elem_length_ = DecodeFixed32(data_+cur_byte_);
return;
}
/// Append the current element (pointed to by cur_byte_) to result_
/// Assumes result_ has already been reserved appropriately.
void WriteCurrentElement() {
// First verify that the iterator is still valid.
CheckErrors();
if (Done()) {
ThrowError("Attempting to write invalid element.");
}
// Append the cur element.
result_.insert(result_.end(),
data_+cur_byte_,
data_+cur_byte_+ sizeof(uint32_t) + cur_elem_length_);
}
/// Will ThrowError() if necessary.
/// Checks for common/ubiquitous errors that can arise after most operations.
/// This method should be called before any reading operation.
/// If this function succeeds, then we are guaranteed to be in a valid state.
/// Other member functions should check for errors and ThrowError() also
/// if an error occurs that is specific to it even while in a valid state.
void CheckErrors() {
// Check if any crazy thing has happened recently
if ((cur_elem_ > length_) || // Bad index
(cur_byte_ > num_bytes_) || // No more bytes
(cur_byte_ + cur_elem_length_ > num_bytes_) || // Item too large
(cur_byte_ == num_bytes_ && cur_elem_ != length_) || // Too many items
(cur_elem_ == length_ && cur_byte_ != num_bytes_)) { // Too many bytes
ThrowError("Corrupt data.");
}
}
/// Will throw an exception based on the passed-in message.
/// This function is guaranteed to STOP THE CONTROL-FLOW.
/// (i.e.: you do not have to call "return" after calling ThrowError)
void ThrowError(const char* const /*msg*/ = nullptr) {
// TODO: For now we ignore the msg parameter. This can be expanded later.
throw RedisListException();
}
private:
const char* const data_; // A pointer to the data (the first byte)
const uint32_t num_bytes_; // The number of bytes in this list
uint32_t cur_byte_; // The current byte being read
uint32_t cur_elem_; // The current element being read
uint32_t cur_elem_length_; // The number of bytes in current element
uint32_t length_; // The number of elements in this list
std::vector<char> result_; // The output data
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

552
utilities/redis/redis_lists.cc
View File

@ -1,552 +0,0 @@
// Copyright 2013 Facebook
/**
* A (persistent) Redis API built using the rocksdb backend.
* Implements Redis Lists as described on: http://redis.io/commands#list
*
* @throws All functions may throw a RedisListException on error/corruption.
*
* @notes Internally, the set of lists is stored in a rocksdb database,
* mapping keys to values. Each "value" is the list itself, storing
* some kind of internal representation of the data. All the
* representation details are handled by the RedisListIterator class.
* The present file should be oblivious to the representation details,
* handling only the client (Redis) API, and the calls to rocksdb.
*
* @TODO Presently, all operations take at least O(NV) time where
* N is the number of elements in the list, and V is the average
* number of bytes per value in the list. So maybe, with merge operator
* we can improve this to an optimal O(V) amortized time, since we
* wouldn't have to read and re-write the entire list.
*
* @author Deon Nicholas (dnicholas@fb.com)
*/
#ifndef ROCKSDB_LITE
#include "redis_lists.h"
#include <iostream>
#include <memory>
#include <cmath>
#include "rocksdb/slice.h"
#include "util/coding.h"
namespace rocksdb
{
/// Constructors
RedisLists::RedisLists(const std::string& db_path,
Options options, bool destructive)
: put_option_(),
get_option_() {
// Store the name of the database
db_name_ = db_path;
// If destructive, destroy the DB before re-opening it.
if (destructive) {
DestroyDB(db_name_, Options());
}
// Now open and deal with the db
DB* db;
Status s = DB::Open(options, db_name_, &db);
if (!s.ok()) {
std::cerr << "ERROR " << s.ToString() << std::endl;
assert(false);
}
db_ = std::unique_ptr<DB>(db);
}
/// Accessors
// Number of elements in the list associated with key
// : throws RedisListException
int RedisLists::Length(const std::string& key) {
// Extract the string data representing the list.
std::string data;
db_->Get(get_option_, key, &data);
// Return the length
RedisListIterator it(data);
return it.Length();
}
// Get the element at the specified index in the (list: key)
// Returns <empty> ("") on out-of-bounds
// : throws RedisListException
bool RedisLists::Index(const std::string& key, int32_t index,
std::string* result) {
// Extract the string data representing the list.
std::string data;
db_->Get(get_option_, key, &data);
// Handle REDIS negative indices (from the end); fast iff Length() takes O(1)
if (index < 0) {
index = Length(key) - (-index); //replace (-i) with (N-i).
}
// Iterate through the list until the desired index is found.
int curIndex = 0;
RedisListIterator it(data);
while(curIndex < index && !it.Done()) {
++curIndex;
it.Skip();
}
// If we actually found the index
if (curIndex == index && !it.Done()) {
Slice elem;
it.GetCurrent(&elem);
if (result != nullptr) {
*result = elem.ToString();
}
return true;
} else {
return false;
}
}
// Return a truncated version of the list.
// First, negative values for first/last are interpreted as "end of list".
// So, if first == -1, then it is re-set to index: (Length(key) - 1)
// Then, return exactly those indices i such that first <= i <= last.
// : throws RedisListException
std::vector<std::string> RedisLists::Range(const std::string& key,
int32_t first, int32_t last) {
// Extract the string data representing the list.
std::string data;
db_->Get(get_option_, key, &data);
// Handle negative bounds (-1 means last element, etc.)
int listLen = Length(key);
if (first < 0) {
first = listLen - (-first); // Replace (-x) with (N-x)
}
if (last < 0) {
last = listLen - (-last);
}
// Verify bounds (and truncate the range so that it is valid)
first = std::max(first, 0);
last = std::min(last, listLen-1);
int len = std::max(last-first+1, 0);
// Initialize the resulting list
std::vector<std::string> result(len);
// Traverse the list and update the vector
int curIdx = 0;
Slice elem;
for (RedisListIterator it(data); !it.Done() && curIdx<=last; it.Skip()) {
if (first <= curIdx && curIdx <= last) {
it.GetCurrent(&elem);
result[curIdx-first].assign(elem.data(),elem.size());
}
++curIdx;
}
// Return the result. Might be empty
return result;
}
// Print the (list: key) out to stdout. For debugging mostly. Public for now.
void RedisLists::Print(const std::string& key) {
// Extract the string data representing the list.
std::string data;
db_->Get(get_option_, key, &data);
// Iterate through the list and print the items
Slice elem;
for (RedisListIterator it(data); !it.Done(); it.Skip()) {
it.GetCurrent(&elem);
std::cout << "ITEM " << elem.ToString() << std::endl;
}
//Now print the byte data
RedisListIterator it(data);
std::cout << "==Printing data==" << std::endl;
std::cout << data.size() << std::endl;
std::cout << it.Size() << " " << it.Length() << std::endl;
Slice result = it.WriteResult();
std::cout << result.data() << std::endl;
if (true) {
std::cout << "size: " << result.size() << std::endl;
const char* val = result.data();
for(int i=0; i<(int)result.size(); ++i) {
std::cout << (int)val[i] << " " << (val[i]>=32?val[i]:' ') << std::endl;
}
std::cout << std::endl;
}
}
/// Insert/Update Functions
/// Note: The "real" insert function is private. See below.
// InsertBefore and InsertAfter are simply wrappers around the Insert function.
int RedisLists::InsertBefore(const std::string& key, const std::string& pivot,
const std::string& value) {
return Insert(key, pivot, value, false);
}
int RedisLists::InsertAfter(const std::string& key, const std::string& pivot,
const std::string& value) {
return Insert(key, pivot, value, true);
}
// Prepend value onto beginning of (list: key)
// : throws RedisListException
int RedisLists::PushLeft(const std::string& key, const std::string& value) {
// Get the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Construct the result
RedisListIterator it(data);
it.Reserve(it.Size() + it.SizeOf(value));
it.InsertElement(value);
// Push the data back to the db and return the length
db_->Put(put_option_, key, it.WriteResult());
return it.Length();
}
// Append value onto end of (list: key)
// TODO: Make this O(1) time. Might require MergeOperator.
// : throws RedisListException
int RedisLists::PushRight(const std::string& key, const std::string& value) {
// Get the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Create an iterator to the data and seek to the end.
RedisListIterator it(data);
it.Reserve(it.Size() + it.SizeOf(value));
while (!it.Done()) {
it.Push(); // Write each element as we go
}
// Insert the new element at the current position (the end)
it.InsertElement(value);
// Push it back to the db, and return length
db_->Put(put_option_, key, it.WriteResult());
return it.Length();
}
// Set (list: key)[idx] = val. Return true on success, false on fail.
// : throws RedisListException
bool RedisLists::Set(const std::string& key, int32_t index,
const std::string& value) {
// Get the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Handle negative index for REDIS (meaning -index from end of list)
if (index < 0) {
index = Length(key) - (-index);
}
// Iterate through the list until we find the element we want
int curIndex = 0;
RedisListIterator it(data);
it.Reserve(it.Size() + it.SizeOf(value)); // Over-estimate is fine
while(curIndex < index && !it.Done()) {
it.Push();
++curIndex;
}
// If not found, return false (this occurs when index was invalid)
if (it.Done() || curIndex != index) {
return false;
}
// Write the new element value, and drop the previous element value
it.InsertElement(value);
it.Skip();
// Write the data to the database
// Check status, since it needs to return true/false guarantee
Status s = db_->Put(put_option_, key, it.WriteResult());
// Success
return s.ok();
}
/// Delete / Remove / Pop functions
// Trim (list: key) so that it will only contain the indices from start..stop
// Invalid indices will not generate an error, just empty,
// or the portion of the list that fits in this interval
// : throws RedisListException
bool RedisLists::Trim(const std::string& key, int32_t start, int32_t stop) {
// Get the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Handle negative indices in REDIS
int listLen = Length(key);
if (start < 0) {
start = listLen - (-start);
}
if (stop < 0) {
stop = listLen - (-stop);
}
// Truncate bounds to only fit in the list
start = std::max(start, 0);
stop = std::min(stop, listLen-1);
// Construct an iterator for the list. Drop all undesired elements.
int curIndex = 0;
RedisListIterator it(data);
it.Reserve(it.Size()); // Over-estimate
while(!it.Done()) {
// If not within the range, just skip the item (drop it).
// Otherwise, continue as usual.
if (start <= curIndex && curIndex <= stop) {
it.Push();
} else {
it.Skip();
}
// Increment the current index
++curIndex;
}
// Write the (possibly empty) result to the database
Status s = db_->Put(put_option_, key, it.WriteResult());
// Return true as long as the write succeeded
return s.ok();
}
// Return and remove the first element in the list (or "" if empty)
// : throws RedisListException
bool RedisLists::PopLeft(const std::string& key, std::string* result) {
// Get the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Point to first element in the list (if it exists), and get its value/size
RedisListIterator it(data);
if (it.Length() > 0) { // Proceed only if list is non-empty
Slice elem;
it.GetCurrent(&elem); // Store the value of the first element
it.Reserve(it.Size() - it.SizeOf(elem));
it.Skip(); // DROP the first item and move to next
// Update the db
db_->Put(put_option_, key, it.WriteResult());
// Return the value
if (result != nullptr) {
*result = elem.ToString();
}
return true;
} else {
return false;
}
}
// Remove and return the last element in the list (or "" if empty)
// TODO: Make this O(1). Might require MergeOperator.
// : throws RedisListException
bool RedisLists::PopRight(const std::string& key, std::string* result) {
// Extract the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Construct an iterator to the data and move to last element
RedisListIterator it(data);
it.Reserve(it.Size());
int len = it.Length();
int curIndex = 0;
while(curIndex < (len-1) && !it.Done()) {
it.Push();
++curIndex;
}
// Extract and drop/skip the last element
if (curIndex == len-1) {
assert(!it.Done()); // Sanity check. Should not have ended here.
// Extract and pop the element
Slice elem;
it.GetCurrent(&elem); // Save value of element.
it.Skip(); // Skip the element
// Write the result to the database
db_->Put(put_option_, key, it.WriteResult());
// Return the value
if (result != nullptr) {
*result = elem.ToString();
}
return true;
} else {
// Must have been an empty list
assert(it.Done() && len==0 && curIndex == 0);
return false;
}
}
// Remove the (first or last) "num" occurrences of value in (list: key)
// : throws RedisListException
int RedisLists::Remove(const std::string& key, int32_t num,
const std::string& value) {
// Negative num ==> RemoveLast; Positive num ==> Remove First
if (num < 0) {
return RemoveLast(key, -num, value);
} else if (num > 0) {
return RemoveFirst(key, num, value);
} else {
return RemoveFirst(key, Length(key), value);
}
}
// Remove the first "num" occurrences of value in (list: key).
// : throws RedisListException
int RedisLists::RemoveFirst(const std::string& key, int32_t num,
const std::string& value) {
// Ensure that the number is positive
assert(num >= 0);
// Extract the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Traverse the list, appending all but the desired occurrences of value
int numSkipped = 0; // Keep track of the number of times value is seen
Slice elem;
RedisListIterator it(data);
it.Reserve(it.Size());
while (!it.Done()) {
it.GetCurrent(&elem);
if (elem == value && numSkipped < num) {
// Drop this item if desired
it.Skip();
++numSkipped;
} else {
// Otherwise keep the item and proceed as normal
it.Push();
}
}
// Put the result back to the database
db_->Put(put_option_, key, it.WriteResult());
// Return the number of elements removed
return numSkipped;
}
// Remove the last "num" occurrences of value in (list: key).
// TODO: I traverse the list 2x. Make faster. Might require MergeOperator.
// : throws RedisListException
int RedisLists::RemoveLast(const std::string& key, int32_t num,
const std::string& value) {
// Ensure that the number is positive
assert(num >= 0);
// Extract the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Temporary variable to hold the "current element" in the blocks below
Slice elem;
// Count the total number of occurrences of value
int totalOccs = 0;
for (RedisListIterator it(data); !it.Done(); it.Skip()) {
it.GetCurrent(&elem);
if (elem == value) {
++totalOccs;
}
}
// Construct an iterator to the data. Reserve enough space for the result.
RedisListIterator it(data);
int bytesRemoved = std::min(num,totalOccs)*it.SizeOf(value);
it.Reserve(it.Size() - bytesRemoved);
// Traverse the list, appending all but the desired occurrences of value.
// Note: "Drop the last k occurrences" is equivalent to
// "keep only the first n-k occurrences", where n is total occurrences.
int numKept = 0; // Keep track of the number of times value is kept
while(!it.Done()) {
it.GetCurrent(&elem);
// If we are within the deletion range and equal to value, drop it.
// Otherwise, append/keep/push it.
if (elem == value) {
if (numKept < totalOccs - num) {
it.Push();
++numKept;
} else {
it.Skip();
}
} else {
// Always append the others
it.Push();
}
}
// Put the result back to the database
db_->Put(put_option_, key, it.WriteResult());
// Return the number of elements removed
return totalOccs - numKept;
}
/// Private functions
// Insert element value into (list: key), right before/after
// the first occurrence of pivot
// : throws RedisListException
int RedisLists::Insert(const std::string& key, const std::string& pivot,
const std::string& value, bool insert_after) {
// Get the original list data
std::string data;
db_->Get(get_option_, key, &data);
// Construct an iterator to the data and reserve enough space for result.
RedisListIterator it(data);
it.Reserve(it.Size() + it.SizeOf(value));
// Iterate through the list until we find the element we want
Slice elem;
bool found = false;
while(!it.Done() && !found) {
it.GetCurrent(&elem);
// When we find the element, insert the element and mark found
if (elem == pivot) { // Found it!
found = true;
if (insert_after == true) { // Skip one more, if inserting after it
it.Push();
}
it.InsertElement(value);
} else {
it.Push();
}
}
// Put the data (string) into the database
if (found) {
db_->Put(put_option_, key, it.WriteResult());
}
// Returns the new (possibly unchanged) length of the list
return it.Length();
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

108
utilities/redis/redis_lists.h
View File

@ -1,108 +0,0 @@
/**
* A (persistent) Redis API built using the rocksdb backend.
* Implements Redis Lists as described on: http://redis.io/commands#list
*
* @throws All functions may throw a RedisListException
*
* @author Deon Nicholas (dnicholas@fb.com)
* Copyright 2013 Facebook
*/
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
#include "rocksdb/db.h"
#include "redis_list_iterator.h"
#include "redis_list_exception.h"
namespace rocksdb {
/// The Redis functionality (see http://redis.io/commands#list)
/// All functions may THROW a RedisListException
class RedisLists {
public: // Constructors / Destructors
/// Construct a new RedisLists database, with name/path of db.
/// Will clear the database on open iff destructive is true (default false).
/// Otherwise, it will restore saved changes.
/// May throw RedisListException
RedisLists(const std::string& db_path,
Options options, bool destructive = false);
public: // Accessors
/// The number of items in (list: key)
int Length(const std::string& key);
/// Search the list for the (index)'th item (0-based) in (list:key)
/// A negative index indicates: "from end-of-list"
/// If index is within range: return true, and return the value in *result.
/// If (index < -length OR index>=length), then index is out of range:
/// return false (and *result is left unchanged)
/// May throw RedisListException
bool Index(const std::string& key, int32_t index,
std::string* result);
/// Return (list: key)[first..last] (inclusive)
/// May throw RedisListException
std::vector<std::string> Range(const std::string& key,
int32_t first, int32_t last);
/// Prints the entire (list: key), for debugging.
void Print(const std::string& key);
public: // Insert/Update
/// Insert value before/after pivot in (list: key). Return the length.
/// May throw RedisListException
int InsertBefore(const std::string& key, const std::string& pivot,
const std::string& value);
int InsertAfter(const std::string& key, const std::string& pivot,
const std::string& value);
/// Push / Insert value at beginning/end of the list. Return the length.
/// May throw RedisListException
int PushLeft(const std::string& key, const std::string& value);
int PushRight(const std::string& key, const std::string& value);
/// Set (list: key)[idx] = val. Return true on success, false on fail
/// May throw RedisListException
bool Set(const std::string& key, int32_t index, const std::string& value);
public: // Delete / Remove / Pop / Trim
/// Trim (list: key) so that it will only contain the indices from start..stop
/// Returns true on success
/// May throw RedisListException
bool Trim(const std::string& key, int32_t start, int32_t stop);
/// If list is empty, return false and leave *result unchanged.
/// Else, remove the first/last elem, store it in *result, and return true
bool PopLeft(const std::string& key, std::string* result); // First
bool PopRight(const std::string& key, std::string* result); // Last
/// Remove the first (or last) num occurrences of value from the list (key)
/// Return the number of elements removed.
/// May throw RedisListException
int Remove(const std::string& key, int32_t num,
const std::string& value);
int RemoveFirst(const std::string& key, int32_t num,
const std::string& value);
int RemoveLast(const std::string& key, int32_t num,
const std::string& value);
private: // Private Functions
/// Calls InsertBefore or InsertAfter
int Insert(const std::string& key, const std::string& pivot,
const std::string& value, bool insert_after);
private:
std::string db_name_; // The actual database name/path
WriteOptions put_option_;
ReadOptions get_option_;
/// The backend rocksdb database.
/// Map : key --> list
/// where a list is a sequence of elements
/// and an element is a 4-byte integer (n), followed by n bytes of data
std::unique_ptr<DB> db_;
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

894
utilities/redis/redis_lists_test.cc
View File

@ -1,894 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
/**
* A test harness for the Redis API built on rocksdb.
*
* USAGE: Build with: "make redis_test" (in rocksdb directory).
* Run unit tests with: "./redis_test"
* Manual/Interactive user testing: "./redis_test -m"
* Manual user testing + restart database: "./redis_test -m -d"
*
* TODO: Add LARGE random test cases to verify efficiency and scalability
*
* @author Deon Nicholas (dnicholas@fb.com)
*/
#ifndef ROCKSDB_LITE
#include <iostream>
#include <cctype>
#include "redis_lists.h"
#include "util/testharness.h"
#include "util/random.h"
using namespace rocksdb;
namespace rocksdb {
class RedisListsTest : public testing::Test {
public:
static const std::string kDefaultDbName;
static Options options;
RedisListsTest() {
options.create_if_missing = true;
}
};
const std::string RedisListsTest::kDefaultDbName =
test::PerThreadDBPath("redis_lists_test");
Options RedisListsTest::options = Options();
// operator== and operator<< are defined below for vectors (lists)
// Needed for ASSERT_EQ
namespace {
void AssertListEq(const std::vector<std::string>& result,
const std::vector<std::string>& expected_result) {
ASSERT_EQ(result.size(), expected_result.size());
for (size_t i = 0; i < result.size(); ++i) {
ASSERT_EQ(result[i], expected_result[i]);
}
}
} // namespace
// PushRight, Length, Index, Range
TEST_F(RedisListsTest, SimpleTest) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Simple PushRight (should return the new length each time)
ASSERT_EQ(redis.PushRight("k1", "v1"), 1);
ASSERT_EQ(redis.PushRight("k1", "v2"), 2);
ASSERT_EQ(redis.PushRight("k1", "v3"), 3);
// Check Length and Index() functions
ASSERT_EQ(redis.Length("k1"), 3); // Check length
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "v1"); // Check valid indices
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "v2");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "v3");
// Check range function and vectors
std::vector<std::string> result = redis.Range("k1", 0, 2); // Get the list
std::vector<std::string> expected_result(3);
expected_result[0] = "v1";
expected_result[1] = "v2";
expected_result[2] = "v3";
AssertListEq(result, expected_result);
}
// PushLeft, Length, Index, Range
TEST_F(RedisListsTest, SimpleTest2) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Simple PushRight
ASSERT_EQ(redis.PushLeft("k1", "v3"), 1);
ASSERT_EQ(redis.PushLeft("k1", "v2"), 2);
ASSERT_EQ(redis.PushLeft("k1", "v1"), 3);
// Check Length and Index() functions
ASSERT_EQ(redis.Length("k1"), 3); // Check length
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "v1"); // Check valid indices
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "v2");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "v3");
// Check range function and vectors
std::vector<std::string> result = redis.Range("k1", 0, 2); // Get the list
std::vector<std::string> expected_result(3);
expected_result[0] = "v1";
expected_result[1] = "v2";
expected_result[2] = "v3";
AssertListEq(result, expected_result);
}
// Exhaustive test of the Index() function
TEST_F(RedisListsTest, IndexTest) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Empty Index check (return empty and should not crash or edit tempv)
tempv = "yo";
ASSERT_TRUE(!redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "yo");
ASSERT_TRUE(!redis.Index("fda", 3, &tempv));
ASSERT_EQ(tempv, "yo");
ASSERT_TRUE(!redis.Index("random", -12391, &tempv));
ASSERT_EQ(tempv, "yo");
// Simple Pushes (will yield: [v6, v4, v4, v1, v2, v3]
redis.PushRight("k1", "v1");
redis.PushRight("k1", "v2");
redis.PushRight("k1", "v3");
redis.PushLeft("k1", "v4");
redis.PushLeft("k1", "v4");
redis.PushLeft("k1", "v6");
// Simple, non-negative indices
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "v6");
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "v4");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "v4");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "v1");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "v2");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "v3");
// Negative indices
ASSERT_TRUE(redis.Index("k1", -6, &tempv));
ASSERT_EQ(tempv, "v6");
ASSERT_TRUE(redis.Index("k1", -5, &tempv));
ASSERT_EQ(tempv, "v4");
ASSERT_TRUE(redis.Index("k1", -4, &tempv));
ASSERT_EQ(tempv, "v4");
ASSERT_TRUE(redis.Index("k1", -3, &tempv));
ASSERT_EQ(tempv, "v1");
ASSERT_TRUE(redis.Index("k1", -2, &tempv));
ASSERT_EQ(tempv, "v2");
ASSERT_TRUE(redis.Index("k1", -1, &tempv));
ASSERT_EQ(tempv, "v3");
// Out of bounds (return empty, no crash)
ASSERT_TRUE(!redis.Index("k1", 6, &tempv));
ASSERT_TRUE(!redis.Index("k1", 123219, &tempv));
ASSERT_TRUE(!redis.Index("k1", -7, &tempv));
ASSERT_TRUE(!redis.Index("k1", -129, &tempv));
}
// Exhaustive test of the Range() function
TEST_F(RedisListsTest, RangeTest) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Simple Pushes (will yield: [v6, v4, v4, v1, v2, v3])
redis.PushRight("k1", "v1");
redis.PushRight("k1", "v2");
redis.PushRight("k1", "v3");
redis.PushLeft("k1", "v4");
redis.PushLeft("k1", "v4");
redis.PushLeft("k1", "v6");
// Sanity check (check the length; make sure it's 6)
ASSERT_EQ(redis.Length("k1"), 6);
// Simple range
std::vector<std::string> res = redis.Range("k1", 1, 4);
ASSERT_EQ((int)res.size(), 4);
ASSERT_EQ(res[0], "v4");
ASSERT_EQ(res[1], "v4");
ASSERT_EQ(res[2], "v1");
ASSERT_EQ(res[3], "v2");
// Negative indices (i.e.: measured from the end)
res = redis.Range("k1", 2, -1);
ASSERT_EQ((int)res.size(), 4);
ASSERT_EQ(res[0], "v4");
ASSERT_EQ(res[1], "v1");
ASSERT_EQ(res[2], "v2");
ASSERT_EQ(res[3], "v3");
res = redis.Range("k1", -6, -4);
ASSERT_EQ((int)res.size(), 3);
ASSERT_EQ(res[0], "v6");
ASSERT_EQ(res[1], "v4");
ASSERT_EQ(res[2], "v4");
res = redis.Range("k1", -1, 5);
ASSERT_EQ((int)res.size(), 1);
ASSERT_EQ(res[0], "v3");
// Partial / Broken indices
res = redis.Range("k1", -3, 1000000);
ASSERT_EQ((int)res.size(), 3);
ASSERT_EQ(res[0], "v1");
ASSERT_EQ(res[1], "v2");
ASSERT_EQ(res[2], "v3");
res = redis.Range("k1", -1000000, 1);
ASSERT_EQ((int)res.size(), 2);
ASSERT_EQ(res[0], "v6");
ASSERT_EQ(res[1], "v4");
// Invalid indices
res = redis.Range("k1", 7, 9);
ASSERT_EQ((int)res.size(), 0);
res = redis.Range("k1", -8, -7);
ASSERT_EQ((int)res.size(), 0);
res = redis.Range("k1", 3, 2);
ASSERT_EQ((int)res.size(), 0);
res = redis.Range("k1", 5, -2);
ASSERT_EQ((int)res.size(), 0);
// Range matches Index
res = redis.Range("k1", -6, -4);
ASSERT_TRUE(redis.Index("k1", -6, &tempv));
ASSERT_EQ(tempv, res[0]);
ASSERT_TRUE(redis.Index("k1", -5, &tempv));
ASSERT_EQ(tempv, res[1]);
ASSERT_TRUE(redis.Index("k1", -4, &tempv));
ASSERT_EQ(tempv, res[2]);
// Last check
res = redis.Range("k1", 0, -6);
ASSERT_EQ((int)res.size(), 1);
ASSERT_EQ(res[0], "v6");
}
// Exhaustive test for InsertBefore(), and InsertAfter()
TEST_F(RedisListsTest, InsertTest) {
RedisLists redis(kDefaultDbName, options, true);
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Insert on empty list (return 0, and do not crash)
ASSERT_EQ(redis.InsertBefore("k1", "non-exist", "a"), 0);
ASSERT_EQ(redis.InsertAfter("k1", "other-non-exist", "c"), 0);
ASSERT_EQ(redis.Length("k1"), 0);
// Push some preliminary stuff [g, f, e, d, c, b, a]
redis.PushLeft("k1", "a");
redis.PushLeft("k1", "b");
redis.PushLeft("k1", "c");
redis.PushLeft("k1", "d");
redis.PushLeft("k1", "e");
redis.PushLeft("k1", "f");
redis.PushLeft("k1", "g");
ASSERT_EQ(redis.Length("k1"), 7);
// Test InsertBefore
int newLength = redis.InsertBefore("k1", "e", "hello");
ASSERT_EQ(newLength, 8);
ASSERT_EQ(redis.Length("k1"), newLength);
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "f");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "e");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "hello");
// Test InsertAfter
newLength = redis.InsertAfter("k1", "c", "bye");
ASSERT_EQ(newLength, 9);
ASSERT_EQ(redis.Length("k1"), newLength);
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "bye");
// Test bad value on InsertBefore
newLength = redis.InsertBefore("k1", "yo", "x");
ASSERT_EQ(newLength, 9);
ASSERT_EQ(redis.Length("k1"), newLength);
// Test bad value on InsertAfter
newLength = redis.InsertAfter("k1", "xxxx", "y");
ASSERT_EQ(newLength, 9);
ASSERT_EQ(redis.Length("k1"), newLength);
// Test InsertBefore beginning
newLength = redis.InsertBefore("k1", "g", "begggggggggggggggg");
ASSERT_EQ(newLength, 10);
ASSERT_EQ(redis.Length("k1"), newLength);
// Test InsertAfter end
newLength = redis.InsertAfter("k1", "a", "enddd");
ASSERT_EQ(newLength, 11);
ASSERT_EQ(redis.Length("k1"), newLength);
// Make sure nothing weird happened.
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "begggggggggggggggg");
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "g");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "f");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "hello");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "e");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "d");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "c");
ASSERT_TRUE(redis.Index("k1", 7, &tempv));
ASSERT_EQ(tempv, "bye");
ASSERT_TRUE(redis.Index("k1", 8, &tempv));
ASSERT_EQ(tempv, "b");
ASSERT_TRUE(redis.Index("k1", 9, &tempv));
ASSERT_EQ(tempv, "a");
ASSERT_TRUE(redis.Index("k1", 10, &tempv));
ASSERT_EQ(tempv, "enddd");
}
// Exhaustive test of Set function
TEST_F(RedisListsTest, SetTest) {
RedisLists redis(kDefaultDbName, options, true);
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Set on empty list (return false, and do not crash)
ASSERT_EQ(redis.Set("k1", 7, "a"), false);
ASSERT_EQ(redis.Set("k1", 0, "a"), false);
ASSERT_EQ(redis.Set("k1", -49, "cx"), false);
ASSERT_EQ(redis.Length("k1"), 0);
// Push some preliminary stuff [g, f, e, d, c, b, a]
redis.PushLeft("k1", "a");
redis.PushLeft("k1", "b");
redis.PushLeft("k1", "c");
redis.PushLeft("k1", "d");
redis.PushLeft("k1", "e");
redis.PushLeft("k1", "f");
redis.PushLeft("k1", "g");
ASSERT_EQ(redis.Length("k1"), 7);
// Test Regular Set
ASSERT_TRUE(redis.Set("k1", 0, "0"));
ASSERT_TRUE(redis.Set("k1", 3, "3"));
ASSERT_TRUE(redis.Set("k1", 6, "6"));
ASSERT_TRUE(redis.Set("k1", 2, "2"));
ASSERT_TRUE(redis.Set("k1", 5, "5"));
ASSERT_TRUE(redis.Set("k1", 1, "1"));
ASSERT_TRUE(redis.Set("k1", 4, "4"));
ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "0");
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "1");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "2");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "3");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "4");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "5");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "6");
// Set with negative indices
ASSERT_TRUE(redis.Set("k1", -7, "a"));
ASSERT_TRUE(redis.Set("k1", -4, "d"));
ASSERT_TRUE(redis.Set("k1", -1, "g"));
ASSERT_TRUE(redis.Set("k1", -5, "c"));
ASSERT_TRUE(redis.Set("k1", -2, "f"));
ASSERT_TRUE(redis.Set("k1", -6, "b"));
ASSERT_TRUE(redis.Set("k1", -3, "e"));
ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "a");
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "b");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "c");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "d");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "e");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "f");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "g");
// Bad indices (just out-of-bounds / off-by-one check)
ASSERT_EQ(redis.Set("k1", -8, "off-by-one in negative index"), false);
ASSERT_EQ(redis.Set("k1", 7, "off-by-one-error in positive index"), false);
ASSERT_EQ(redis.Set("k1", 43892, "big random index should fail"), false);
ASSERT_EQ(redis.Set("k1", -21391, "large negative index should fail"), false);
// One last check (to make sure nothing weird happened)
ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "a");
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "b");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "c");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "d");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "e");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "f");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "g");
}
// Testing Insert, Push, and Set, in a mixed environment
TEST_F(RedisListsTest, InsertPushSetTest) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// A series of pushes and insertions
// Will result in [newbegin, z, a, aftera, x, newend]
// Also, check the return value sometimes (should return length)
int lengthCheck;
lengthCheck = redis.PushLeft("k1", "a");
ASSERT_EQ(lengthCheck, 1);
redis.PushLeft("k1", "z");
redis.PushRight("k1", "x");
lengthCheck = redis.InsertAfter("k1", "a", "aftera");
ASSERT_EQ(lengthCheck , 4);
redis.InsertBefore("k1", "z", "newbegin"); // InsertBefore beginning of list
redis.InsertAfter("k1", "x", "newend"); // InsertAfter end of list
// Check
std::vector<std::string> res = redis.Range("k1", 0, -1); // Get the list
ASSERT_EQ((int)res.size(), 6);
ASSERT_EQ(res[0], "newbegin");
ASSERT_EQ(res[5], "newend");
ASSERT_EQ(res[3], "aftera");
// Testing duplicate values/pivots (multiple occurrences of 'a')
ASSERT_TRUE(redis.Set("k1", 0, "a")); // [a, z, a, aftera, x, newend]
redis.InsertAfter("k1", "a", "happy"); // [a, happy, z, a, aftera, ...]
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "happy");
redis.InsertBefore("k1", "a", "sad"); // [sad, a, happy, z, a, aftera, ...]
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "sad");
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "happy");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "aftera");
redis.InsertAfter("k1", "a", "zz"); // [sad, a, zz, happy, z, a, aftera, ...]
ASSERT_TRUE(redis.Index("k1", 2, &tempv));
ASSERT_EQ(tempv, "zz");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "aftera");
ASSERT_TRUE(redis.Set("k1", 1, "nota")); // [sad, nota, zz, happy, z, a, ...]
redis.InsertBefore("k1", "a", "ba"); // [sad, nota, zz, happy, z, ba, a, ...]
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "z");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "ba");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "a");
// We currently have: [sad, nota, zz, happy, z, ba, a, aftera, x, newend]
// redis.Print("k1"); // manually check
// Test Inserting before/after non-existent values
lengthCheck = redis.Length("k1"); // Ensure that the length doesn't change
ASSERT_EQ(lengthCheck, 10);
ASSERT_EQ(redis.InsertBefore("k1", "non-exist", "randval"), lengthCheck);
ASSERT_EQ(redis.InsertAfter("k1", "nothing", "a"), lengthCheck);
ASSERT_EQ(redis.InsertAfter("randKey", "randVal", "ranValue"), 0); // Empty
ASSERT_EQ(redis.Length("k1"), lengthCheck); // The length should not change
// Simply Test the Set() function
redis.Set("k1", 5, "ba2");
redis.InsertBefore("k1", "ba2", "beforeba2");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "z");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "beforeba2");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "ba2");
ASSERT_TRUE(redis.Index("k1", 7, &tempv));
ASSERT_EQ(tempv, "a");
// We have: [sad, nota, zz, happy, z, beforeba2, ba2, a, aftera, x, newend]
// Set() with negative indices
redis.Set("k1", -1, "endprank");
ASSERT_TRUE(!redis.Index("k1", 11, &tempv));
ASSERT_TRUE(redis.Index("k1", 10, &tempv));
ASSERT_EQ(tempv, "endprank"); // Ensure Set worked correctly
redis.Set("k1", -11, "t");
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "t");
// Test out of bounds Set
ASSERT_EQ(redis.Set("k1", -12, "ssd"), false);
ASSERT_EQ(redis.Set("k1", 11, "sasd"), false);
ASSERT_EQ(redis.Set("k1", 1200, "big"), false);
}
// Testing Trim, Pop
TEST_F(RedisListsTest, TrimPopTest) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// A series of pushes and insertions
// Will result in [newbegin, z, a, aftera, x, newend]
redis.PushLeft("k1", "a");
redis.PushLeft("k1", "z");
redis.PushRight("k1", "x");
redis.InsertBefore("k1", "z", "newbegin"); // InsertBefore start of list
redis.InsertAfter("k1", "x", "newend"); // InsertAfter end of list
redis.InsertAfter("k1", "a", "aftera");
// Simple PopLeft/Right test
ASSERT_TRUE(redis.PopLeft("k1", &tempv));
ASSERT_EQ(tempv, "newbegin");
ASSERT_EQ(redis.Length("k1"), 5);
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "z");
ASSERT_TRUE(redis.PopRight("k1", &tempv));
ASSERT_EQ(tempv, "newend");
ASSERT_EQ(redis.Length("k1"), 4);
ASSERT_TRUE(redis.Index("k1", -1, &tempv));
ASSERT_EQ(tempv, "x");
// Now have: [z, a, aftera, x]
// Test Trim
ASSERT_TRUE(redis.Trim("k1", 0, -1)); // [z, a, aftera, x] (do nothing)
ASSERT_EQ(redis.Length("k1"), 4);
ASSERT_TRUE(redis.Trim("k1", 0, 2)); // [z, a, aftera]
ASSERT_EQ(redis.Length("k1"), 3);
ASSERT_TRUE(redis.Index("k1", -1, &tempv));
ASSERT_EQ(tempv, "aftera");
ASSERT_TRUE(redis.Trim("k1", 1, 1)); // [a]
ASSERT_EQ(redis.Length("k1"), 1);
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "a");
// Test out of bounds (empty) trim
ASSERT_TRUE(redis.Trim("k1", 1, 0));
ASSERT_EQ(redis.Length("k1"), 0);
// Popping with empty list (return empty without error)
ASSERT_TRUE(!redis.PopLeft("k1", &tempv));
ASSERT_TRUE(!redis.PopRight("k1", &tempv));
ASSERT_TRUE(redis.Trim("k1", 0, 5));
// Exhaustive Trim test (negative and invalid indices)
// Will start in [newbegin, z, a, aftera, x, newend]
redis.PushLeft("k1", "a");
redis.PushLeft("k1", "z");
redis.PushRight("k1", "x");
redis.InsertBefore("k1", "z", "newbegin"); // InsertBefore start of list
redis.InsertAfter("k1", "x", "newend"); // InsertAfter end of list
redis.InsertAfter("k1", "a", "aftera");
ASSERT_TRUE(redis.Trim("k1", -6, -1)); // Should do nothing
ASSERT_EQ(redis.Length("k1"), 6);
ASSERT_TRUE(redis.Trim("k1", 1, -2));
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "z");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "x");
ASSERT_EQ(redis.Length("k1"), 4);
ASSERT_TRUE(redis.Trim("k1", -3, -2));
ASSERT_EQ(redis.Length("k1"), 2);
}
// Testing Remove, RemoveFirst, RemoveLast
TEST_F(RedisListsTest, RemoveTest) {
RedisLists redis(kDefaultDbName, options, true); // Destructive
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// A series of pushes and insertions
// Will result in [newbegin, z, a, aftera, x, newend, a, a]
redis.PushLeft("k1", "a");
redis.PushLeft("k1", "z");
redis.PushRight("k1", "x");
redis.InsertBefore("k1", "z", "newbegin"); // InsertBefore start of list
redis.InsertAfter("k1", "x", "newend"); // InsertAfter end of list
redis.InsertAfter("k1", "a", "aftera");
redis.PushRight("k1", "a");
redis.PushRight("k1", "a");
// Verify
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "newbegin");
ASSERT_TRUE(redis.Index("k1", -1, &tempv));
ASSERT_EQ(tempv, "a");
// Check RemoveFirst (Remove the first two 'a')
// Results in [newbegin, z, aftera, x, newend, a]
int numRemoved = redis.Remove("k1", 2, "a");
ASSERT_EQ(numRemoved, 2);
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "newbegin");
ASSERT_TRUE(redis.Index("k1", 1, &tempv));
ASSERT_EQ(tempv, "z");
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "newend");
ASSERT_TRUE(redis.Index("k1", 5, &tempv));
ASSERT_EQ(tempv, "a");
ASSERT_EQ(redis.Length("k1"), 6);
// Repopulate some stuff
// Results in: [x, x, x, x, x, newbegin, z, x, aftera, x, newend, a, x]
redis.PushLeft("k1", "x");
redis.PushLeft("k1", "x");
redis.PushLeft("k1", "x");
redis.PushLeft("k1", "x");
redis.PushLeft("k1", "x");
redis.PushRight("k1", "x");
redis.InsertAfter("k1", "z", "x");
// Test removal from end
numRemoved = redis.Remove("k1", -2, "x");
ASSERT_EQ(numRemoved, 2);
ASSERT_TRUE(redis.Index("k1", 8, &tempv));
ASSERT_EQ(tempv, "aftera");
ASSERT_TRUE(redis.Index("k1", 9, &tempv));
ASSERT_EQ(tempv, "newend");
ASSERT_TRUE(redis.Index("k1", 10, &tempv));
ASSERT_EQ(tempv, "a");
ASSERT_TRUE(!redis.Index("k1", 11, &tempv));
numRemoved = redis.Remove("k1", -2, "x");
ASSERT_EQ(numRemoved, 2);
ASSERT_TRUE(redis.Index("k1", 4, &tempv));
ASSERT_EQ(tempv, "newbegin");
ASSERT_TRUE(redis.Index("k1", 6, &tempv));
ASSERT_EQ(tempv, "aftera");
// We now have: [x, x, x, x, newbegin, z, aftera, newend, a]
ASSERT_EQ(redis.Length("k1"), 9);
ASSERT_TRUE(redis.Index("k1", -1, &tempv));
ASSERT_EQ(tempv, "a");
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "x");
// Test over-shooting (removing more than there exists)
numRemoved = redis.Remove("k1", -9000, "x");
ASSERT_EQ(numRemoved , 4); // Only really removed 4
ASSERT_EQ(redis.Length("k1"), 5);
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "newbegin");
numRemoved = redis.Remove("k1", 1, "x");
ASSERT_EQ(numRemoved, 0);
// Try removing ALL!
numRemoved = redis.Remove("k1", 0, "newbegin"); // REMOVE 0 will remove all!
ASSERT_EQ(numRemoved, 1);
// Removal from an empty-list
ASSERT_TRUE(redis.Trim("k1", 1, 0));
numRemoved = redis.Remove("k1", 1, "z");
ASSERT_EQ(numRemoved, 0);
}
// Test Multiple keys and Persistence
TEST_F(RedisListsTest, PersistenceMultiKeyTest) {
std::string tempv; // Used below for all Index(), PopRight(), PopLeft()
// Block one: populate a single key in the database
{
RedisLists redis(kDefaultDbName, options, true); // Destructive
// A series of pushes and insertions
// Will result in [newbegin, z, a, aftera, x, newend, a, a]
redis.PushLeft("k1", "a");
redis.PushLeft("k1", "z");
redis.PushRight("k1", "x");
redis.InsertBefore("k1", "z", "newbegin"); // InsertBefore start of list
redis.InsertAfter("k1", "x", "newend"); // InsertAfter end of list
redis.InsertAfter("k1", "a", "aftera");
redis.PushRight("k1", "a");
redis.PushRight("k1", "a");
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "aftera");
}
// Block two: make sure changes were saved and add some other key
{
RedisLists redis(kDefaultDbName, options, false); // Persistent, non-destructive
// Check
ASSERT_EQ(redis.Length("k1"), 8);
ASSERT_TRUE(redis.Index("k1", 3, &tempv));
ASSERT_EQ(tempv, "aftera");
redis.PushRight("k2", "randomkey");
redis.PushLeft("k2", "sas");
redis.PopLeft("k1", &tempv);
}
// Block three: Verify the changes from block 2
{
RedisLists redis(kDefaultDbName, options, false); // Persistent, non-destructive
// Check
ASSERT_EQ(redis.Length("k1"), 7);
ASSERT_EQ(redis.Length("k2"), 2);
ASSERT_TRUE(redis.Index("k1", 0, &tempv));
ASSERT_EQ(tempv, "z");
ASSERT_TRUE(redis.Index("k2", -2, &tempv));
ASSERT_EQ(tempv, "sas");
}
}
/// THE manual REDIS TEST begins here
/// THIS WILL ONLY OCCUR IF YOU RUN: ./redis_test -m
namespace {
void MakeUpper(std::string* const s) {
int len = static_cast<int>(s->length());
for (int i = 0; i < len; ++i) {
(*s)[i] = static_cast<char>(toupper((*s)[i])); // C-version defined in <ctype.h>
}
}
/// Allows the user to enter in REDIS commands into the command-line.
/// This is useful for manual / interacticve testing / debugging.
/// Use destructive=true to clean the database before use.
/// Use destructive=false to remember the previous state (i.e.: persistent)
/// Should be called from main function.
int manual_redis_test(bool destructive){
RedisLists redis(RedisListsTest::kDefaultDbName,
RedisListsTest::options,
destructive);
// TODO: Right now, please use spaces to separate each word.
// In actual redis, you can use quotes to specify compound values
// Example: RPUSH mylist "this is a compound value"
std::string command;
while(true) {
std::cin >> command;
MakeUpper(&command);
if (command == "LINSERT") {
std::string k, t, p, v;
std::cin >> k >> t >> p >> v;
MakeUpper(&t);
if (t=="BEFORE") {
std::cout << redis.InsertBefore(k, p, v) << std::endl;
} else if (t=="AFTER") {
std::cout << redis.InsertAfter(k, p, v) << std::endl;
}
} else if (command == "LPUSH") {
std::string k, v;
std::cin >> k >> v;
redis.PushLeft(k, v);
} else if (command == "RPUSH") {
std::string k, v;
std::cin >> k >> v;
redis.PushRight(k, v);
} else if (command == "LPOP") {
std::string k;
std::cin >> k;
std::string res;
redis.PopLeft(k, &res);
std::cout << res << std::endl;
} else if (command == "RPOP") {
std::string k;
std::cin >> k;
std::string res;
redis.PopRight(k, &res);
std::cout << res << std::endl;
} else if (command == "LREM") {
std::string k;
int amt;
std::string v;
std::cin >> k >> amt >> v;
std::cout << redis.Remove(k, amt, v) << std::endl;
} else if (command == "LLEN") {
std::string k;
std::cin >> k;
std::cout << redis.Length(k) << std::endl;
} else if (command == "LRANGE") {
std::string k;
int i, j;
std::cin >> k >> i >> j;
std::vector<std::string> res = redis.Range(k, i, j);
for (auto it = res.begin(); it != res.end(); ++it) {
std::cout << " " << (*it);
}
std::cout << std::endl;
} else if (command == "LTRIM") {
std::string k;
int i, j;
std::cin >> k >> i >> j;
redis.Trim(k, i, j);
} else if (command == "LSET") {
std::string k;
int idx;
std::string v;
std::cin >> k >> idx >> v;
redis.Set(k, idx, v);
} else if (command == "LINDEX") {
std::string k;
int idx;
std::cin >> k >> idx;
std::string res;
redis.Index(k, idx, &res);
std::cout << res << std::endl;
} else if (command == "PRINT") { // Added by Deon
std::string k;
std::cin >> k;
redis.Print(k);
} else if (command == "QUIT") {
return 0;
} else {
std::cout << "unknown command: " << command << std::endl;
}
}
}
} // namespace
} // namespace rocksdb
// USAGE: "./redis_test" for default (unit tests)
// "./redis_test -m" for manual testing (redis command api)
// "./redis_test -m -d" for destructive manual test (erase db before use)
namespace {
// Check for "want" argument in the argument list
bool found_arg(int argc, char* argv[], const char* want){
for(int i=1; i<argc; ++i){
if (strcmp(argv[i], want) == 0) {
return true;
}
}
return false;
}
} // namespace
// Will run unit tests.
// However, if -m is specified, it will do user manual/interactive testing
// -m -d is manual and destructive (will clear the database before use)
int main(int argc, char* argv[]) {
::testing::InitGoogleTest(&argc, argv);
if (found_arg(argc, argv, "-m")) {
bool destructive = found_arg(argc, argv, "-d");
return rocksdb::manual_redis_test(destructive);
} else {
return RUN_ALL_TESTS();
}
}
#else
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "SKIPPED as redis is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

919
utilities/spatialdb/spatial_db.cc
View File

@ -1,919 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "rocksdb/utilities/spatial_db.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <algorithm>
#include <condition_variable>
#include <inttypes.h>
#include <string>
#include <vector>
#include <mutex>
#include <thread>
#include <set>
#include <unordered_set>
#include "rocksdb/cache.h"
#include "rocksdb/options.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/statistics.h"
#include "rocksdb/table.h"
#include "rocksdb/db.h"
#include "rocksdb/utilities/stackable_db.h"
#include "util/coding.h"
#include "utilities/spatialdb/utils.h"
#include "port/port.h"
namespace rocksdb {
namespace spatial {
// Column families are used to store element's data and spatial indexes. We use
// [default] column family to store the element data. This is the format of
// [default] column family:
// * id (fixed 64 big endian) -> blob (length prefixed slice) feature_set
// (serialized)
// We have one additional column family for each spatial index. The name of the
// column family is [spatial$<spatial_index_name>]. The format is:
// * quad_key (fixed 64 bit big endian) id (fixed 64 bit big endian) -> ""
// We store information about indexes in [metadata] column family. Format is:
// * spatial$<spatial_index_name> -> bbox (4 double encodings) tile_bits
// (varint32)
namespace {
const std::string kMetadataColumnFamilyName("metadata");
inline std::string GetSpatialIndexColumnFamilyName(
const std::string& spatial_index_name) {
return "spatial$" + spatial_index_name;
}
inline bool GetSpatialIndexName(const std::string& column_family_name,
Slice* dst) {
*dst = Slice(column_family_name);
if (dst->starts_with("spatial$")) {
dst->remove_prefix(8); // strlen("spatial$")
return true;
}
return false;
}
} // namespace
void Variant::Init(const Variant& v, Data& d) {
switch (v.type_) {
case kNull:
break;
case kBool:
d.b = v.data_.b;
break;
case kInt:
d.i = v.data_.i;
break;
case kDouble:
d.d = v.data_.d;
break;
case kString:
new (d.s) std::string(*GetStringPtr(v.data_));
break;
default:
assert(false);
}
}
Variant& Variant::operator=(const Variant& v) {
// Construct first a temp so exception from a string ctor
// does not change this object
Data tmp;
Init(v, tmp);
Type thisType = type_;
// Boils down to copying bits so safe
std::swap(tmp, data_);
type_ = v.type_;
Destroy(thisType, tmp);
return *this;
}
Variant& Variant::operator=(Variant&& rhs) {
Destroy(type_, data_);
if (rhs.type_ == kString) {
new (data_.s) std::string(std::move(*GetStringPtr(rhs.data_)));
} else {
data_ = rhs.data_;
}
type_ = rhs.type_;
rhs.type_ = kNull;
return *this;
}
bool Variant::operator==(const Variant& rhs) const {
if (type_ != rhs.type_) {
return false;
}
switch (type_) {
case kNull:
return true;
case kBool:
return data_.b == rhs.data_.b;
case kInt:
return data_.i == rhs.data_.i;
case kDouble:
return data_.d == rhs.data_.d;
case kString:
return *GetStringPtr(data_) == *GetStringPtr(rhs.data_);
default:
assert(false);
}
// it will never reach here, but otherwise the compiler complains
return false;
}
FeatureSet* FeatureSet::Set(const std::string& key, const Variant& value) {
map_.insert({key, value});
return this;
}
bool FeatureSet::Contains(const std::string& key) const {
return map_.find(key) != map_.end();
}
const Variant& FeatureSet::Get(const std::string& key) const {
auto itr = map_.find(key);
assert(itr != map_.end());
return itr->second;
}
FeatureSet::iterator FeatureSet::Find(const std::string& key) const {
return iterator(map_.find(key));
}
void FeatureSet::Clear() { map_.clear(); }
void FeatureSet::Serialize(std::string* output) const {
for (const auto& iter : map_) {
PutLengthPrefixedSlice(output, iter.first);
output->push_back(static_cast<char>(iter.second.type()));
switch (iter.second.type()) {
case Variant::kNull:
break;
case Variant::kBool:
output->push_back(static_cast<char>(iter.second.get_bool()));
break;
case Variant::kInt:
PutVarint64(output, iter.second.get_int());
break;
case Variant::kDouble: {
PutDouble(output, iter.second.get_double());
break;
}
case Variant::kString:
PutLengthPrefixedSlice(output, iter.second.get_string());
break;
default:
assert(false);
}
}
}
bool FeatureSet::Deserialize(const Slice& input) {
assert(map_.empty());
Slice s(input);
while (s.size()) {
Slice key;
if (!GetLengthPrefixedSlice(&s, &key) || s.size() == 0) {
return false;
}
char type = s[0];
s.remove_prefix(1);
switch (type) {
case Variant::kNull: {
map_.insert({key.ToString(), Variant()});
break;
}
case Variant::kBool: {
if (s.size() == 0) {
return false;
}
map_.insert({key.ToString(), Variant(static_cast<bool>(s[0]))});
s.remove_prefix(1);
break;
}
case Variant::kInt: {
uint64_t v;
if (!GetVarint64(&s, &v)) {
return false;
}
map_.insert({key.ToString(), Variant(v)});
break;
}
case Variant::kDouble: {
double d;
if (!GetDouble(&s, &d)) {
return false;
}
map_.insert({key.ToString(), Variant(d)});
break;
}
case Variant::kString: {
Slice str;
if (!GetLengthPrefixedSlice(&s, &str)) {
return false;
}
map_.insert({key.ToString(), str.ToString()});
break;
}
default:
return false;
}
}
return true;
}
std::string FeatureSet::DebugString() const {
std::string out = "{";
bool comma = false;
for (const auto& iter : map_) {
if (comma) {
out.append(", ");
} else {
comma = true;
}
out.append("\"" + iter.first + "\": ");
switch (iter.second.type()) {
case Variant::kNull:
out.append("null");
break;
case Variant::kBool:
if (iter.second.get_bool()) {
out.append("true");
} else {
out.append("false");
}
break;
case Variant::kInt: {
char buf[32];
snprintf(buf, sizeof(buf), "%" PRIu64, iter.second.get_int());
out.append(buf);
break;
}
case Variant::kDouble: {
char buf[32];
snprintf(buf, sizeof(buf), "%lf", iter.second.get_double());
out.append(buf);
break;
}
case Variant::kString:
out.append("\"" + iter.second.get_string() + "\"");
break;
default:
assert(false);
}
}
return out + "}";
}
class ValueGetter {
public:
ValueGetter() {}
virtual ~ValueGetter() {}
virtual bool Get(uint64_t id) = 0;
virtual const Slice value() const = 0;
virtual Status status() const = 0;
};
class ValueGetterFromDB : public ValueGetter {
public:
ValueGetterFromDB(DB* db, ColumnFamilyHandle* cf) : db_(db), cf_(cf) {}
virtual bool Get(uint64_t id) override {
std::string encoded_id;
PutFixed64BigEndian(&encoded_id, id);
status_ = db_->Get(ReadOptions(), cf_, encoded_id, &value_);
if (status_.IsNotFound()) {
status_ = Status::Corruption("Index inconsistency");
return false;
}
return true;
}
virtual const Slice value() const override { return value_; }
virtual Status status() const override { return status_; }
private:
std::string value_;
DB* db_;
ColumnFamilyHandle* cf_;
Status status_;
};
class ValueGetterFromIterator : public ValueGetter {
public:
explicit ValueGetterFromIterator(Iterator* iterator) : iterator_(iterator) {}
virtual bool Get(uint64_t id) override {
std::string encoded_id;
PutFixed64BigEndian(&encoded_id, id);
iterator_->Seek(encoded_id);
if (!iterator_->Valid() || iterator_->key() != Slice(encoded_id)) {
status_ = Status::Corruption("Index inconsistency");
return false;
}
return true;
}
virtual const Slice value() const override { return iterator_->value(); }
virtual Status status() const override { return status_; }
private:
std::unique_ptr<Iterator> iterator_;
Status status_;
};
class SpatialIndexCursor : public Cursor {
public:
// tile_box is inclusive
SpatialIndexCursor(Iterator* spatial_iterator, ValueGetter* value_getter,
const BoundingBox<uint64_t>& tile_bbox, uint32_t tile_bits)
: value_getter_(value_getter), valid_(true) {
// calculate quad keys we'll need to query
std::vector<uint64_t> quad_keys;
quad_keys.reserve(static_cast<size_t>((tile_bbox.max_x - tile_bbox.min_x + 1) *
(tile_bbox.max_y - tile_bbox.min_y + 1)));
for (uint64_t x = tile_bbox.min_x; x <= tile_bbox.max_x; ++x) {
for (uint64_t y = tile_bbox.min_y; y <= tile_bbox.max_y; ++y) {
quad_keys.push_back(GetQuadKeyFromTile(x, y, tile_bits));
}
}
std::sort(quad_keys.begin(), quad_keys.end());
// load primary key ids for all quad keys
for (auto quad_key : quad_keys) {
std::string encoded_quad_key;
PutFixed64BigEndian(&encoded_quad_key, quad_key);
Slice slice_quad_key(encoded_quad_key);
// If CheckQuadKey is true, there is no need to reseek, since
// spatial_iterator is already pointing at the correct quad key. This is
// an optimization.
if (!CheckQuadKey(spatial_iterator, slice_quad_key)) {
spatial_iterator->Seek(slice_quad_key);
}
while (CheckQuadKey(spatial_iterator, slice_quad_key)) {
// extract ID from spatial_iterator
uint64_t id;
bool ok = GetFixed64BigEndian(
Slice(spatial_iterator->key().data() + sizeof(uint64_t),
sizeof(uint64_t)),
&id);
if (!ok) {
valid_ = false;
status_ = Status::Corruption("Spatial index corruption");
break;
}
primary_key_ids_.insert(id);
spatial_iterator->Next();
}
}
if (!spatial_iterator->status().ok()) {
status_ = spatial_iterator->status();
valid_ = false;
}
delete spatial_iterator;
valid_ = valid_ && !primary_key_ids_.empty();
if (valid_) {
primary_keys_iterator_ = primary_key_ids_.begin();
ExtractData();
}
}
virtual bool Valid() const override { return valid_; }
virtual void Next() override {
assert(valid_);
++primary_keys_iterator_;
if (primary_keys_iterator_ == primary_key_ids_.end()) {
valid_ = false;
return;
}
ExtractData();
}
virtual const Slice blob() override { return current_blob_; }
virtual const FeatureSet& feature_set() override {
return current_feature_set_;
}
virtual Status status() const override {
if (!status_.ok()) {
return status_;
}
return value_getter_->status();
}
private:
// * returns true if spatial iterator is on the current quad key and all is
// well
// * returns false if spatial iterator is not on current, or iterator is
// invalid or corruption
bool CheckQuadKey(Iterator* spatial_iterator, const Slice& quad_key) {
if (!spatial_iterator->Valid()) {
return false;
}
if (spatial_iterator->key().size() != 2 * sizeof(uint64_t)) {
status_ = Status::Corruption("Invalid spatial index key");
valid_ = false;
return false;
}
Slice spatial_iterator_quad_key(spatial_iterator->key().data(),
sizeof(uint64_t));
if (spatial_iterator_quad_key != quad_key) {
// caller needs to reseek
return false;
}
// if we come to here, we have found the quad key
return true;
}
void ExtractData() {
assert(valid_);
valid_ = value_getter_->Get(*primary_keys_iterator_);
if (valid_) {
Slice data = value_getter_->value();
current_feature_set_.Clear();
if (!GetLengthPrefixedSlice(&data, &current_blob_) ||
!current_feature_set_.Deserialize(data)) {
status_ = Status::Corruption("Primary key column family corruption");
valid_ = false;
}
}
}
std::unique_ptr<ValueGetter> value_getter_;
bool valid_;
Status status_;
FeatureSet current_feature_set_;
Slice current_blob_;
// This is loaded from spatial iterator.
std::unordered_set<uint64_t> primary_key_ids_;
std::unordered_set<uint64_t>::iterator primary_keys_iterator_;
};
class ErrorCursor : public Cursor {
public:
explicit ErrorCursor(Status s) : s_(s) { assert(!s.ok()); }
virtual Status status() const override { return s_; }
virtual bool Valid() const override { return false; }
virtual void Next() override { assert(false); }
virtual const Slice blob() override {
assert(false);
return Slice();
}
virtual const FeatureSet& feature_set() override {
assert(false);
// compiler complains otherwise
return trash_;
}
private:
Status s_;
FeatureSet trash_;
};
class SpatialDBImpl : public SpatialDB {
public:
// * db -- base DB that needs to be forwarded to StackableDB
// * data_column_family -- column family used to store the data
// * spatial_indexes -- a list of spatial indexes together with column
// families that correspond to those spatial indexes
// * next_id -- next ID in auto-incrementing ID. This is usually
// `max_id_currenty_in_db + 1`
SpatialDBImpl(
DB* db, ColumnFamilyHandle* data_column_family,
const std::vector<std::pair<SpatialIndexOptions, ColumnFamilyHandle*>>&
spatial_indexes,
uint64_t next_id, bool read_only)
: SpatialDB(db),
data_column_family_(data_column_family),
next_id_(next_id),
read_only_(read_only) {
for (const auto& index : spatial_indexes) {
name_to_index_.insert(
{index.first.name, IndexColumnFamily(index.first, index.second)});
}
}
~SpatialDBImpl() {
for (auto& iter : name_to_index_) {
delete iter.second.column_family;
}
delete data_column_family_;
}
virtual Status Insert(
const WriteOptions& write_options, const BoundingBox<double>& bbox,
const Slice& blob, const FeatureSet& feature_set,
const std::vector<std::string>& spatial_indexes) override {
WriteBatch batch;
if (spatial_indexes.size() == 0) {
return Status::InvalidArgument("Spatial indexes can't be empty");
}
const size_t kWriteOutEveryBytes = 1024 * 1024; // 1MB
uint64_t id = next_id_.fetch_add(1);
for (const auto& si : spatial_indexes) {
auto itr = name_to_index_.find(si);
if (itr == name_to_index_.end()) {
return Status::InvalidArgument("Can't find index " + si);
}
const auto& spatial_index = itr->second.index;
if (!spatial_index.bbox.Intersects(bbox)) {
continue;
}
BoundingBox<uint64_t> tile_bbox = GetTileBoundingBox(spatial_index, bbox);
for (uint64_t x = tile_bbox.min_x; x <= tile_bbox.max_x; ++x) {
for (uint64_t y = tile_bbox.min_y; y <= tile_bbox.max_y; ++y) {
// see above for format
std::string key;
PutFixed64BigEndian(
&key, GetQuadKeyFromTile(x, y, spatial_index.tile_bits));
PutFixed64BigEndian(&key, id);
batch.Put(itr->second.column_family, key, Slice());
if (batch.GetDataSize() >= kWriteOutEveryBytes) {
Status s = Write(write_options, &batch);
batch.Clear();
if (!s.ok()) {
return s;
}
}
}
}
}
// see above for format
std::string data_key;
PutFixed64BigEndian(&data_key, id);
std::string data_value;
PutLengthPrefixedSlice(&data_value, blob);
feature_set.Serialize(&data_value);
batch.Put(data_column_family_, data_key, data_value);
return Write(write_options, &batch);
}
virtual Status Compact(int num_threads) override {
std::vector<ColumnFamilyHandle*> column_families;
column_families.push_back(data_column_family_);
for (auto& iter : name_to_index_) {
column_families.push_back(iter.second.column_family);
}
std::mutex state_mutex;
std::condition_variable cv;
Status s;
int threads_running = 0;
std::vector<port::Thread> threads;
for (auto cfh : column_families) {
threads.emplace_back([&, cfh] {
{
std::unique_lock<std::mutex> lk(state_mutex);
cv.wait(lk, [&] { return threads_running < num_threads; });
threads_running++;
}
Status t = Flush(FlushOptions(), cfh);
if (t.ok()) {
t = CompactRange(CompactRangeOptions(), cfh, nullptr, nullptr);
}
{
std::unique_lock<std::mutex> lk(state_mutex);
threads_running--;
if (s.ok() && !t.ok()) {
s = t;
}
cv.notify_one();
}
});
}
for (auto& t : threads) {
t.join();
}
return s;
}
virtual Cursor* Query(const ReadOptions& read_options,
const BoundingBox<double>& bbox,
const std::string& spatial_index) override {
auto itr = name_to_index_.find(spatial_index);
if (itr == name_to_index_.end()) {
return new ErrorCursor(Status::InvalidArgument(
"Spatial index " + spatial_index + " not found"));
}
const auto& si = itr->second.index;
Iterator* spatial_iterator;
ValueGetter* value_getter;
if (read_only_) {
spatial_iterator = NewIterator(read_options, itr->second.column_family);
value_getter = new ValueGetterFromDB(this, data_column_family_);
} else {
std::vector<Iterator*> iterators;
Status s = NewIterators(read_options,
{data_column_family_, itr->second.column_family},
&iterators);
if (!s.ok()) {
return new ErrorCursor(s);
}
spatial_iterator = iterators[1];
value_getter = new ValueGetterFromIterator(iterators[0]);
}
return new SpatialIndexCursor(spatial_iterator, value_getter,
GetTileBoundingBox(si, bbox), si.tile_bits);
}
private:
ColumnFamilyHandle* data_column_family_;
struct IndexColumnFamily {
SpatialIndexOptions index;
ColumnFamilyHandle* column_family;
IndexColumnFamily(const SpatialIndexOptions& _index,
ColumnFamilyHandle* _cf)
: index(_index), column_family(_cf) {}
};
// constant after construction!
std::unordered_map<std::string, IndexColumnFamily> name_to_index_;
std::atomic<uint64_t> next_id_;
bool read_only_;
};
namespace {
DBOptions GetDBOptionsFromSpatialDBOptions(const SpatialDBOptions& options) {
DBOptions db_options;
db_options.max_open_files = 50000;
db_options.max_background_compactions = 3 * options.num_threads / 4;
db_options.max_background_flushes =
options.num_threads - db_options.max_background_compactions;
db_options.env->SetBackgroundThreads(db_options.max_background_compactions,
Env::LOW);
db_options.env->SetBackgroundThreads(db_options.max_background_flushes,
Env::HIGH);
db_options.statistics = CreateDBStatistics();
if (options.bulk_load) {
db_options.stats_dump_period_sec = 600;
} else {
db_options.stats_dump_period_sec = 1800; // 30min
}
return db_options;
}
ColumnFamilyOptions GetColumnFamilyOptions(const SpatialDBOptions& /*options*/,
std::shared_ptr<Cache> block_cache) {
ColumnFamilyOptions column_family_options;
column_family_options.write_buffer_size = 128 * 1024 * 1024; // 128MB
column_family_options.max_write_buffer_number = 4;
column_family_options.max_bytes_for_level_base = 256 * 1024 * 1024; // 256MB
column_family_options.target_file_size_base = 64 * 1024 * 1024; // 64MB
column_family_options.level0_file_num_compaction_trigger = 2;
column_family_options.level0_slowdown_writes_trigger = 16;
column_family_options.level0_stop_writes_trigger = 32;
// only compress levels >= 2
column_family_options.compression_per_level.resize(
column_family_options.num_levels);
for (int i = 0; i < column_family_options.num_levels; ++i) {
if (i < 2) {
column_family_options.compression_per_level[i] = kNoCompression;
} else {
column_family_options.compression_per_level[i] = kLZ4Compression;
}
}
BlockBasedTableOptions table_options;
table_options.block_cache = block_cache;
column_family_options.table_factory.reset(
NewBlockBasedTableFactory(table_options));
return column_family_options;
}
ColumnFamilyOptions OptimizeOptionsForDataColumnFamily(
ColumnFamilyOptions options, std::shared_ptr<Cache> block_cache) {
options.prefix_extractor.reset(NewNoopTransform());
BlockBasedTableOptions block_based_options;
block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
block_based_options.block_cache = block_cache;
options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
return options;
}
} // namespace
class MetadataStorage {
public:
MetadataStorage(DB* db, ColumnFamilyHandle* cf) : db_(db), cf_(cf) {}
~MetadataStorage() {}
// format: <min_x double> <min_y double> <max_x double> <max_y double>
// <tile_bits varint32>
Status AddIndex(const SpatialIndexOptions& index) {
std::string encoded_index;
PutDouble(&encoded_index, index.bbox.min_x);
PutDouble(&encoded_index, index.bbox.min_y);
PutDouble(&encoded_index, index.bbox.max_x);
PutDouble(&encoded_index, index.bbox.max_y);
PutVarint32(&encoded_index, index.tile_bits);
return db_->Put(WriteOptions(), cf_,
GetSpatialIndexColumnFamilyName(index.name), encoded_index);
}
Status GetIndex(const std::string& name, SpatialIndexOptions* dst) {
std::string value;
Status s = db_->Get(ReadOptions(), cf_,
GetSpatialIndexColumnFamilyName(name), &value);
if (!s.ok()) {
return s;
}
dst->name = name;
Slice encoded_index(value);
bool ok = GetDouble(&encoded_index, &(dst->bbox.min_x));
ok = ok && GetDouble(&encoded_index, &(dst->bbox.min_y));
ok = ok && GetDouble(&encoded_index, &(dst->bbox.max_x));
ok = ok && GetDouble(&encoded_index, &(dst->bbox.max_y));
ok = ok && GetVarint32(&encoded_index, &(dst->tile_bits));
return ok ? Status::OK() : Status::Corruption("Index encoding corrupted");
}
private:
DB* db_;
ColumnFamilyHandle* cf_;
};
Status SpatialDB::Create(
const SpatialDBOptions& options, const std::string& name,
const std::vector<SpatialIndexOptions>& spatial_indexes) {
DBOptions db_options = GetDBOptionsFromSpatialDBOptions(options);
db_options.create_if_missing = true;
db_options.create_missing_column_families = true;
db_options.error_if_exists = true;
auto block_cache = NewLRUCache(static_cast<size_t>(options.cache_size));
ColumnFamilyOptions column_family_options =
GetColumnFamilyOptions(options, block_cache);
std::vector<ColumnFamilyDescriptor> column_families;
column_families.push_back(ColumnFamilyDescriptor(
kDefaultColumnFamilyName,
OptimizeOptionsForDataColumnFamily(column_family_options, block_cache)));
column_families.push_back(
ColumnFamilyDescriptor(kMetadataColumnFamilyName, column_family_options));
for (const auto& index : spatial_indexes) {
column_families.emplace_back(GetSpatialIndexColumnFamilyName(index.name),
column_family_options);
}
std::vector<ColumnFamilyHandle*> handles;
DB* base_db;
Status s = DB::Open(db_options, name, column_families, &handles, &base_db);
if (!s.ok()) {
return s;
}
MetadataStorage metadata(base_db, handles[1]);
for (const auto& index : spatial_indexes) {
s = metadata.AddIndex(index);
if (!s.ok()) {
break;
}
}
for (auto h : handles) {
delete h;
}
delete base_db;
return s;
}
Status SpatialDB::Open(const SpatialDBOptions& options, const std::string& name,
SpatialDB** db, bool read_only) {
DBOptions db_options = GetDBOptionsFromSpatialDBOptions(options);
auto block_cache = NewLRUCache(static_cast<size_t>(options.cache_size));
ColumnFamilyOptions column_family_options =
GetColumnFamilyOptions(options, block_cache);
Status s;
std::vector<std::string> existing_column_families;
std::vector<std::string> spatial_indexes;
s = DB::ListColumnFamilies(db_options, name, &existing_column_families);
if (!s.ok()) {
return s;
}
for (const auto& cf_name : existing_column_families) {
Slice spatial_index;
if (GetSpatialIndexName(cf_name, &spatial_index)) {
spatial_indexes.emplace_back(spatial_index.data(), spatial_index.size());
}
}
std::vector<ColumnFamilyDescriptor> column_families;
column_families.push_back(ColumnFamilyDescriptor(
kDefaultColumnFamilyName,
OptimizeOptionsForDataColumnFamily(column_family_options, block_cache)));
column_families.push_back(
ColumnFamilyDescriptor(kMetadataColumnFamilyName, column_family_options));
for (const auto& index : spatial_indexes) {
column_families.emplace_back(GetSpatialIndexColumnFamilyName(index),
column_family_options);
}
std::vector<ColumnFamilyHandle*> handles;
DB* base_db;
if (read_only) {
s = DB::OpenForReadOnly(db_options, name, column_families, &handles,
&base_db);
} else {
s = DB::Open(db_options, name, column_families, &handles, &base_db);
}
if (!s.ok()) {
return s;
}
MetadataStorage metadata(base_db, handles[1]);
std::vector<std::pair<SpatialIndexOptions, ColumnFamilyHandle*>> index_cf;
assert(handles.size() == spatial_indexes.size() + 2);
for (size_t i = 0; i < spatial_indexes.size(); ++i) {
SpatialIndexOptions index_options;
s = metadata.GetIndex(spatial_indexes[i], &index_options);
if (!s.ok()) {
break;
}
index_cf.emplace_back(index_options, handles[i + 2]);
}
uint64_t next_id = 1;
if (s.ok()) {
// find next_id
Iterator* iter = base_db->NewIterator(ReadOptions(), handles[0]);
iter->SeekToLast();
if (iter->Valid()) {
uint64_t last_id = 0;
if (!GetFixed64BigEndian(iter->key(), &last_id)) {
s = Status::Corruption("Invalid key in data column family");
} else {
next_id = last_id + 1;
}
}
delete iter;
}
if (!s.ok()) {
for (auto h : handles) {
delete h;
}
delete base_db;
return s;
}
// I don't need metadata column family any more, so delete it
delete handles[1];
*db = new SpatialDBImpl(base_db, handles[0], index_cf, next_id, read_only);
return Status::OK();
}
} // namespace spatial
} // namespace rocksdb
#endif // ROCKSDB_LITE

307
utilities/spatialdb/spatial_db_test.cc
View File

@ -1,307 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include <vector>
#include <string>
#include <set>
#include "rocksdb/utilities/spatial_db.h"
#include "util/compression.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "util/random.h"
namespace rocksdb {
namespace spatial {
class SpatialDBTest : public testing::Test {
public:
SpatialDBTest() {
dbname_ = test::PerThreadDBPath("spatial_db_test");
DestroyDB(dbname_, Options());
}
void AssertCursorResults(BoundingBox<double> bbox, const std::string& index,
const std::vector<std::string>& blobs) {
Cursor* c = db_->Query(ReadOptions(), bbox, index);
ASSERT_OK(c->status());
std::multiset<std::string> b;
for (auto x : blobs) {
b.insert(x);
}
while (c->Valid()) {
auto itr = b.find(c->blob().ToString());
ASSERT_TRUE(itr != b.end());
b.erase(itr);
c->Next();
}
ASSERT_EQ(b.size(), 0U);
ASSERT_OK(c->status());
delete c;
}
std::string dbname_;
SpatialDB* db_;
};
TEST_F(SpatialDBTest, FeatureSetSerializeTest) {
if (!LZ4_Supported()) {
return;
}
FeatureSet fs;
fs.Set("a", std::string("b"));
fs.Set("x", static_cast<uint64_t>(3));
fs.Set("y", false);
fs.Set("n", Variant()); // null
fs.Set("m", 3.25);
ASSERT_TRUE(fs.Find("w") == fs.end());
ASSERT_TRUE(fs.Find("x") != fs.end());
ASSERT_TRUE((*fs.Find("x")).second == Variant(static_cast<uint64_t>(3)));
ASSERT_TRUE((*fs.Find("y")).second != Variant(true));
std::set<std::string> keys({"a", "x", "y", "n", "m"});
for (const auto& x : fs) {
ASSERT_TRUE(keys.find(x.first) != keys.end());
keys.erase(x.first);
}
ASSERT_EQ(keys.size(), 0U);
std::string serialized;
fs.Serialize(&serialized);
FeatureSet deserialized;
ASSERT_TRUE(deserialized.Deserialize(serialized));
ASSERT_TRUE(deserialized.Contains("a"));
ASSERT_EQ(deserialized.Get("a").type(), Variant::kString);
ASSERT_EQ(deserialized.Get("a").get_string(), "b");
ASSERT_TRUE(deserialized.Contains("x"));
ASSERT_EQ(deserialized.Get("x").type(), Variant::kInt);
ASSERT_EQ(deserialized.Get("x").get_int(), static_cast<uint64_t>(3));
ASSERT_TRUE(deserialized.Contains("y"));
ASSERT_EQ(deserialized.Get("y").type(), Variant::kBool);
ASSERT_EQ(deserialized.Get("y").get_bool(), false);
ASSERT_TRUE(deserialized.Contains("n"));
ASSERT_EQ(deserialized.Get("n").type(), Variant::kNull);
ASSERT_TRUE(deserialized.Contains("m"));
ASSERT_EQ(deserialized.Get("m").type(), Variant::kDouble);
ASSERT_EQ(deserialized.Get("m").get_double(), 3.25);
// corrupted serialization
serialized = serialized.substr(0, serialized.size() - 1);
deserialized.Clear();
ASSERT_TRUE(!deserialized.Deserialize(serialized));
}
TEST_F(SpatialDBTest, TestNextID) {
if (!LZ4_Supported()) {
return;
}
ASSERT_OK(SpatialDB::Create(
SpatialDBOptions(), dbname_,
{SpatialIndexOptions("simple", BoundingBox<double>(0, 0, 100, 100), 2)}));
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(5, 5, 10, 10),
"one", FeatureSet(), {"simple"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(10, 10, 15, 15),
"two", FeatureSet(), {"simple"}));
delete db_;
db_ = nullptr;
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
assert(db_ != nullptr);
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(55, 55, 65, 65),
"three", FeatureSet(), {"simple"}));
delete db_;
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
AssertCursorResults(BoundingBox<double>(0, 0, 100, 100), "simple",
{"one", "two", "three"});
delete db_;
}
TEST_F(SpatialDBTest, FeatureSetTest) {
if (!LZ4_Supported()) {
return;
}
ASSERT_OK(SpatialDB::Create(
SpatialDBOptions(), dbname_,
{SpatialIndexOptions("simple", BoundingBox<double>(0, 0, 100, 100), 2)}));
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
FeatureSet fs;
fs.Set("a", std::string("b"));
fs.Set("c", std::string("d"));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(5, 5, 10, 10),
"one", fs, {"simple"}));
Cursor* c =
db_->Query(ReadOptions(), BoundingBox<double>(5, 5, 10, 10), "simple");
ASSERT_TRUE(c->Valid());
ASSERT_EQ(c->blob().compare("one"), 0);
FeatureSet returned = c->feature_set();
ASSERT_TRUE(returned.Contains("a"));
ASSERT_TRUE(!returned.Contains("b"));
ASSERT_TRUE(returned.Contains("c"));
ASSERT_EQ(returned.Get("a").type(), Variant::kString);
ASSERT_EQ(returned.Get("a").get_string(), "b");
ASSERT_EQ(returned.Get("c").type(), Variant::kString);
ASSERT_EQ(returned.Get("c").get_string(), "d");
c->Next();
ASSERT_TRUE(!c->Valid());
delete c;
delete db_;
}
TEST_F(SpatialDBTest, SimpleTest) {
if (!LZ4_Supported()) {
return;
}
// iter 0 -- not read only
// iter 1 -- read only
for (int iter = 0; iter < 2; ++iter) {
DestroyDB(dbname_, Options());
ASSERT_OK(SpatialDB::Create(
SpatialDBOptions(), dbname_,
{SpatialIndexOptions("index", BoundingBox<double>(0, 0, 128, 128),
3)}));
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
assert(db_ != nullptr);
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(33, 17, 63, 79),
"one", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(65, 65, 111, 111),
"two", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 49, 127, 63),
"three", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(20, 100, 21, 101),
"four", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(81, 33, 127, 63),
"five", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 65, 47, 95),
"six", FeatureSet(), {"index"}));
if (iter == 1) {
delete db_;
db_ = nullptr;
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_, true));
}
AssertCursorResults(BoundingBox<double>(33, 17, 47, 31), "index", {"one"});
AssertCursorResults(BoundingBox<double>(17, 33, 79, 63), "index",
{"one", "three"});
AssertCursorResults(BoundingBox<double>(17, 81, 63, 111), "index",
{"four", "six"});
AssertCursorResults(BoundingBox<double>(85, 86, 85, 86), "index", {"two"});
AssertCursorResults(BoundingBox<double>(33, 1, 127, 111), "index",
{"one", "two", "three", "five", "six"});
// even though the bounding box doesn't intersect, we got "four" back
// because
// it's in the same tile
AssertCursorResults(BoundingBox<double>(18, 98, 19, 99), "index", {"four"});
AssertCursorResults(BoundingBox<double>(130, 130, 131, 131), "index", {});
AssertCursorResults(BoundingBox<double>(81, 17, 127, 31), "index", {});
AssertCursorResults(BoundingBox<double>(90, 50, 91, 51), "index",
{"three", "five"});
delete db_;
db_ = nullptr;
}
}
namespace {
std::string RandomStr(Random* rnd) {
std::string r;
for (int k = 0; k < 10; ++k) {
r.push_back(static_cast<char>(rnd->Uniform(26)) + 'a');
}
return r;
}
BoundingBox<int> RandomBoundingBox(int limit, Random* rnd, int max_size) {
BoundingBox<int> r;
r.min_x = rnd->Uniform(limit - 1);
r.min_y = rnd->Uniform(limit - 1);
r.max_x = r.min_x + rnd->Uniform(std::min(limit - 1 - r.min_x, max_size)) + 1;
r.max_y = r.min_y + rnd->Uniform(std::min(limit - 1 - r.min_y, max_size)) + 1;
return r;
}
BoundingBox<double> ScaleBB(BoundingBox<int> b, double step) {
return BoundingBox<double>(b.min_x * step + 1, b.min_y * step + 1,
(b.max_x + 1) * step - 1,
(b.max_y + 1) * step - 1);
}
} // namespace
TEST_F(SpatialDBTest, RandomizedTest) {
if (!LZ4_Supported()) {
return;
}
Random rnd(301);
std::vector<std::pair<std::string, BoundingBox<int>>> elements;
BoundingBox<double> spatial_index_bounds(0, 0, (1LL << 32), (1LL << 32));
ASSERT_OK(SpatialDB::Create(
SpatialDBOptions(), dbname_,
{SpatialIndexOptions("index", spatial_index_bounds, 7)}));
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
double step = (1LL << 32) / (1 << 7);
for (int i = 0; i < 1000; ++i) {
std::string blob = RandomStr(&rnd);
BoundingBox<int> bbox = RandomBoundingBox(128, &rnd, 10);
ASSERT_OK(db_->Insert(WriteOptions(), ScaleBB(bbox, step), blob,
FeatureSet(), {"index"}));
elements.push_back(make_pair(blob, bbox));
}
// parallel
db_->Compact(2);
// serial
db_->Compact(1);
for (int i = 0; i < 1000; ++i) {
BoundingBox<int> int_bbox = RandomBoundingBox(128, &rnd, 10);
BoundingBox<double> double_bbox = ScaleBB(int_bbox, step);
std::vector<std::string> blobs;
for (auto e : elements) {
if (e.second.Intersects(int_bbox)) {
blobs.push_back(e.first);
}
}
AssertCursorResults(double_bbox, "index", blobs);
}
delete db_;
}
} // namespace spatial
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "SKIPPED as SpatialDB is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

95
utilities/spatialdb/utils.h
View File

@ -1,95 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <string>
#include <algorithm>
#include "rocksdb/utilities/spatial_db.h"
namespace rocksdb {
namespace spatial {
// indexing idea from http://msdn.microsoft.com/en-us/library/bb259689.aspx
inline uint64_t GetTileFromCoord(double x, double start, double end,
uint32_t tile_bits) {
if (x < start) {
return 0;
}
uint64_t tiles = 1ull << tile_bits;
uint64_t r = static_cast<uint64_t>(((x - start) / (end - start)) * tiles);
return std::min(r, tiles - 1);
}
inline uint64_t GetQuadKeyFromTile(uint64_t tile_x, uint64_t tile_y,
uint32_t tile_bits) {
uint64_t quad_key = 0;
for (uint32_t i = 0; i < tile_bits; ++i) {
uint64_t mask = (1ull << i);
quad_key |= (tile_x & mask) << i;
quad_key |= (tile_y & mask) << (i + 1);
}
return quad_key;
}
inline BoundingBox<uint64_t> GetTileBoundingBox(
const SpatialIndexOptions& spatial_index, BoundingBox<double> bbox) {
return BoundingBox<uint64_t>(
GetTileFromCoord(bbox.min_x, spatial_index.bbox.min_x,
spatial_index.bbox.max_x, spatial_index.tile_bits),
GetTileFromCoord(bbox.min_y, spatial_index.bbox.min_y,
spatial_index.bbox.max_y, spatial_index.tile_bits),
GetTileFromCoord(bbox.max_x, spatial_index.bbox.min_x,
spatial_index.bbox.max_x, spatial_index.tile_bits),
GetTileFromCoord(bbox.max_y, spatial_index.bbox.min_y,
spatial_index.bbox.max_y, spatial_index.tile_bits));
}
// big endian can be compared using memcpy
inline void PutFixed64BigEndian(std::string* dst, uint64_t value) {
char buf[sizeof(value)];
buf[0] = (value >> 56) & 0xff;
buf[1] = (value >> 48) & 0xff;
buf[2] = (value >> 40) & 0xff;
buf[3] = (value >> 32) & 0xff;
buf[4] = (value >> 24) & 0xff;
buf[5] = (value >> 16) & 0xff;
buf[6] = (value >> 8) & 0xff;
buf[7] = value & 0xff;
dst->append(buf, sizeof(buf));
}
// big endian can be compared using memcpy
inline bool GetFixed64BigEndian(const Slice& input, uint64_t* value) {
if (input.size() < sizeof(uint64_t)) {
return false;
}
auto ptr = input.data();
*value = (static_cast<uint64_t>(static_cast<unsigned char>(ptr[0])) << 56) |
(static_cast<uint64_t>(static_cast<unsigned char>(ptr[1])) << 48) |
(static_cast<uint64_t>(static_cast<unsigned char>(ptr[2])) << 40) |
(static_cast<uint64_t>(static_cast<unsigned char>(ptr[3])) << 32) |
(static_cast<uint64_t>(static_cast<unsigned char>(ptr[4])) << 24) |
(static_cast<uint64_t>(static_cast<unsigned char>(ptr[5])) << 16) |
(static_cast<uint64_t>(static_cast<unsigned char>(ptr[6])) << 8) |
static_cast<uint64_t>(static_cast<unsigned char>(ptr[7]));
return true;
}
inline void PutDouble(std::string* dst, double d) {
dst->append(reinterpret_cast<char*>(&d), sizeof(double));
}
inline bool GetDouble(Slice* input, double* d) {
if (input->size() < sizeof(double)) {
return false;
}
memcpy(d, input->data(), sizeof(double));
input->remove_prefix(sizeof(double));
return true;
}
} // namespace spatial
} // namespace rocksdb