From 03bbf718cb5f715eefa45f7366b59e49a7f45c49 Mon Sep 17 00:00:00 2001 From: stash93 Date: Fri, 27 Feb 2015 00:04:14 +0300 Subject: [PATCH] Return fbson Summary: mac compile is fixed in fbson, so it can be returned back from 7ce1b2c Test Plan: make all check make valgrind_check Reviewers: golovachalexander, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D33855 --- include/rocksdb/utilities/json_document.h | 149 +-- .../lint_engine/FacebookFbcodeLintEngine.php | 3 +- third-party/fbson/COMMIT.md | 1 + third-party/fbson/FbsonDocument.h | 886 ++++++++++++++++ third-party/fbson/FbsonJsonParser.h | 746 ++++++++++++++ third-party/fbson/FbsonStream.h | 175 ++++ third-party/fbson/FbsonUtil.h | 168 +++ third-party/fbson/FbsonWriter.h | 435 ++++++++ third-party/rapidjson/document.h | 821 --------------- third-party/rapidjson/filestream.h | 46 - third-party/rapidjson/internal/pow10.h | 54 - third-party/rapidjson/internal/stack.h | 82 -- third-party/rapidjson/internal/strfunc.h | 24 - third-party/rapidjson/license.txt | 19 - third-party/rapidjson/prettywriter.h | 156 --- third-party/rapidjson/rapidjson.h | 525 ---------- third-party/rapidjson/reader.h | 683 ------------ third-party/rapidjson/stringbuffer.h | 49 - third-party/rapidjson/writer.h | 241 ----- utilities/document/document_db.cc | 159 +-- utilities/document/document_db_test.cc | 24 +- utilities/document/json_document.cc | 970 +++++++++--------- utilities/document/json_document_builder.cc | 115 +++ utilities/document/json_document_test.cc | 216 +++- 24 files changed, 3397 insertions(+), 3350 deletions(-) create mode 100644 third-party/fbson/COMMIT.md create mode 100644 third-party/fbson/FbsonDocument.h create mode 100644 third-party/fbson/FbsonJsonParser.h create mode 100644 third-party/fbson/FbsonStream.h create mode 100644 third-party/fbson/FbsonUtil.h create mode 100644 third-party/fbson/FbsonWriter.h delete mode 100644 third-party/rapidjson/document.h delete mode 100644 third-party/rapidjson/filestream.h delete mode 100644 third-party/rapidjson/internal/pow10.h delete mode 100644 third-party/rapidjson/internal/stack.h delete mode 100644 third-party/rapidjson/internal/strfunc.h delete mode 100644 third-party/rapidjson/license.txt delete mode 100644 third-party/rapidjson/prettywriter.h delete mode 100644 third-party/rapidjson/rapidjson.h delete mode 100644 third-party/rapidjson/reader.h delete mode 100644 third-party/rapidjson/stringbuffer.h delete mode 100644 third-party/rapidjson/writer.h create mode 100644 utilities/document/json_document_builder.cc diff --git a/include/rocksdb/utilities/json_document.h b/include/rocksdb/utilities/json_document.h index ceb058cf95..a5e3ab2562 100644 --- a/include/rocksdb/utilities/json_document.h +++ b/include/rocksdb/utilities/json_document.h @@ -5,15 +5,27 @@ #pragma once #ifndef ROCKSDB_LITE -#include +#include #include +#include +#include #include +#include #include #include "rocksdb/slice.h" // We use JSONDocument for DocumentDB API -// Implementation inspired by folly::dynamic and rapidjson +// Implementation inspired by folly::dynamic, rapidjson and fbson + +namespace fbson { + class FbsonValue; + class ObjectVal; + template + class FbsonWriterT; + class FbsonOutStream; + typedef FbsonWriterT FbsonWriter; +} // namespace fbson namespace rocksdb { @@ -33,52 +45,38 @@ class JSONDocument { kString, }; - JSONDocument(); // null + /* implicit */ JSONDocument(); // null /* implicit */ JSONDocument(bool b); /* implicit */ JSONDocument(double d); + /* implicit */ JSONDocument(int8_t i); + /* implicit */ JSONDocument(int16_t i); + /* implicit */ JSONDocument(int32_t i); /* implicit */ JSONDocument(int64_t i); /* implicit */ JSONDocument(const std::string& s); /* implicit */ JSONDocument(const char* s); // constructs JSONDocument of specific type with default value - explicit JSONDocument(Type type); + explicit JSONDocument(Type _type); - // copy constructor JSONDocument(const JSONDocument& json_document); - ~JSONDocument(); + JSONDocument(JSONDocument&& json_document); Type type() const; // REQUIRES: IsObject() bool Contains(const std::string& key) const; - // Returns nullptr if !Contains() - // don't delete the returned pointer // REQUIRES: IsObject() - const JSONDocument* Get(const std::string& key) const; - // REQUIRES: IsObject() - JSONDocument& operator[](const std::string& key); - // REQUIRES: IsObject() - const JSONDocument& operator[](const std::string& key) const; - // returns `this`, so you can chain operations. - // Copies value - // REQUIRES: IsObject() - JSONDocument* Set(const std::string& key, const JSONDocument& value); + // Returns non-owner object + JSONDocument operator[](const std::string& key) const; // REQUIRES: IsArray() == true || IsObject() == true size_t Count() const; // REQUIRES: IsArray() - const JSONDocument* GetFromArray(size_t i) const; - // REQUIRES: IsArray() - JSONDocument& operator[](size_t i); - // REQUIRES: IsArray() - const JSONDocument& operator[](size_t i) const; - // returns `this`, so you can chain operations. - // Copies the value - // REQUIRES: IsArray() && i < Count() - JSONDocument* SetInArray(size_t i, const JSONDocument& value); - // REQUIRES: IsArray() - JSONDocument* PushBack(const JSONDocument& value); + // Returns non-owner object + JSONDocument operator[](size_t i) const; + + JSONDocument& operator=(JSONDocument jsonDocument); bool IsNull() const; bool IsArray() const; @@ -95,10 +93,16 @@ class JSONDocument { // REQUIRES: IsInt64() == true int64_t GetInt64() const; // REQUIRES: IsString() == true - const std::string& GetString() const; + std::string GetString() const; bool operator==(const JSONDocument& rhs) const; + bool operator!=(const JSONDocument& rhs) const; + + JSONDocument Copy() const; + + bool IsOwner() const; + std::string DebugString() const; private: @@ -114,53 +118,42 @@ class JSONDocument { static JSONDocument* Deserialize(const Slice& src); private: - void SerializeInternal(std::string* dst, bool type_prefix) const; - // returns false if Slice doesn't represent valid serialized JSONDocument. - // Otherwise, true - bool DeserializeInternal(Slice* input); + friend class JSONDocumentBuilder; - typedef std::vector Array; - typedef std::unordered_map Object; + JSONDocument(fbson::FbsonValue* val, bool makeCopy); + + void InitFromValue(const fbson::FbsonValue* val); // iteration on objects class const_item_iterator { - public: - typedef Object::const_iterator It; - typedef Object::value_type value_type; - /* implicit */ const_item_iterator(It it) : it_(it) {} - It& operator++() { return ++it_; } - bool operator!=(const const_item_iterator& other) { - return it_ != other.it_; - } - value_type operator*() { return *it_; } - private: - It it_; + class Impl; + public: + typedef std::pair value_type; + explicit const_item_iterator(Impl* impl); + const_item_iterator(const_item_iterator&&); + const_item_iterator& operator++(); + bool operator!=(const const_item_iterator& other); + value_type operator*(); + ~const_item_iterator(); + private: + friend class ItemsIteratorGenerator; + std::unique_ptr it_; }; + class ItemsIteratorGenerator { public: - /* implicit */ ItemsIteratorGenerator(const Object& object) - : object_(object) {} - const_item_iterator begin() { return object_.begin(); } - const_item_iterator end() { return object_.end(); } + explicit ItemsIteratorGenerator(const fbson::ObjectVal& object); + const_item_iterator begin() const; + + const_item_iterator end() const; private: - const Object& object_; + const fbson::ObjectVal& object_; }; - union Data { - Data() : n(nullptr) {} - ~Data() {} - - void* n; - Array a; - bool b; - double d; - int64_t i; - std::string s; - Object o; - } data_; - const Type type_; + std::unique_ptr data_; + mutable fbson::FbsonValue* value_; // Our serialization format's first byte specifies the encoding version. That // way, we can easily change our format while providing backwards @@ -169,6 +162,34 @@ class JSONDocument { static const char kSerializationFormatVersion; }; +class JSONDocumentBuilder { + public: + JSONDocumentBuilder(); + + explicit JSONDocumentBuilder(fbson::FbsonOutStream* out); + + void Reset(); + + bool WriteStartArray(); + + bool WriteEndArray(); + + bool WriteStartObject(); + + bool WriteEndObject(); + + bool WriteKeyValue(const std::string& key, const JSONDocument& value); + + bool WriteJSONDocument(const JSONDocument& value); + + JSONDocument GetJSONDocument(); + + ~JSONDocumentBuilder(); + + private: + std::unique_ptr writer_; +}; + } // namespace rocksdb #endif // ROCKSDB_LITE diff --git a/linters/lint_engine/FacebookFbcodeLintEngine.php b/linters/lint_engine/FacebookFbcodeLintEngine.php index 131b34efb9..6b10b8c060 100644 --- a/linters/lint_engine/FacebookFbcodeLintEngine.php +++ b/linters/lint_engine/FacebookFbcodeLintEngine.php @@ -87,7 +87,8 @@ class FacebookFbcodeLintEngine extends ArcanistLintEngine { $spelling_linter->addPath($path); $spelling_linter->addData($path, $this->loadData($path)); } - if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) { + if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path) + && !preg_match('/third-party/', $path)) { foreach ($cpp_linters as &$linter) { $linter->addPath($path); $linter->addData($path, $this->loadData($path)); diff --git a/third-party/fbson/COMMIT.md b/third-party/fbson/COMMIT.md new file mode 100644 index 0000000000..4485e836d7 --- /dev/null +++ b/third-party/fbson/COMMIT.md @@ -0,0 +1 @@ +fbson commit: c8f16edf02243dce5a9dc48495f8f5a99f1c7895 diff --git a/third-party/fbson/FbsonDocument.h b/third-party/fbson/FbsonDocument.h new file mode 100644 index 0000000000..604a2712f0 --- /dev/null +++ b/third-party/fbson/FbsonDocument.h @@ -0,0 +1,886 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +/* + * This header defines FbsonDocument, FbsonKeyValue, and various value classes + * which are derived from FbsonValue, and a forward iterator for container + * values - essentially everything that is related to FBSON binary data + * structures. + * + * Implementation notes: + * + * None of the classes in this header file can be instantiated directly (i.e. + * you cannot create a FbsonKeyValue or FbsonValue object - all constructors + * are declared non-public). We use the classes as wrappers on the packed FBSON + * bytes (serialized), and cast the classes (types) to the underlying packed + * byte array. + * + * For the same reason, we cannot define any FBSON value class to be virtual, + * since we never call constructors, and will not instantiate vtbl and vptrs. + * + * Therefore, the classes are defined as packed structures (i.e. no data + * alignment and padding), and the private member variables of the classes are + * defined precisely in the same order as the FBSON spec. This ensures we + * access the packed FBSON bytes correctly. + * + * The packed structures are highly optimized for in-place operations with low + * overhead. The reads (and in-place writes) are performed directly on packed + * bytes. There is no memory allocation at all at runtime. + * + * For updates/writes of values that will expand the original FBSON size, the + * write will fail, and the caller needs to handle buffer increase. + * + * ** Iterator ** + * Both ObjectVal class and ArrayVal class have iterator type that you can use + * to declare an iterator on a container object to go through the key-value + * pairs or value list. The iterator has both non-const and const types. + * + * Note: iterators are forward direction only. + * + * ** Query ** + * Querying into containers is through the member functions find (for key/value + * pairs) and get (for array elements), and is in streaming style. We don't + * need to read/scan the whole FBSON packed bytes in order to return results. + * Once the key/index is found, we will stop search. You can use text to query + * both objects and array (for array, text will be converted to integer index), + * and use index to retrieve from array. Array index is 0-based. + * + * ** External dictionary ** + * During query processing, you can also pass a call-back function, so the + * search will first try to check if the key string exists in the dictionary. + * If so, search will be based on the id instead of the key string. + * + * @author Tian Xia + */ + +#ifndef FBSON_FBSONDOCUMENT_H +#define FBSON_FBSONDOCUMENT_H + +#include +#include + +namespace fbson { + +#pragma pack(push, 1) + +#define FBSON_VER 1 + +// forward declaration +class FbsonValue; +class ObjectVal; + +/* + * FbsonDocument is the main object that accesses and queries FBSON packed + * bytes. NOTE: FbsonDocument only allows object container as the top level + * FBSON value. However, you can use the static method "createValue" to get any + * FbsonValue object from the packed bytes. + * + * FbsonDocument object also dereferences to an object container value + * (ObjectVal) once FBSON is loaded. + * + * ** Load ** + * FbsonDocument is usable after loading packed bytes (memory location) into + * the object. We only need the header and first few bytes of the payload after + * header to verify the FBSON. + * + * Note: creating an FbsonDocument (through createDocument) does not allocate + * any memory. The document object is an efficient wrapper on the packed bytes + * which is accessed directly. + * + * ** Query ** + * Query is through dereferencing into ObjectVal. + */ +class FbsonDocument { + public: + // create an FbsonDocument object from FBSON packed bytes + static FbsonDocument* createDocument(const char* pb, uint32_t size); + + // create an FbsonValue from FBSON packed bytes + static FbsonValue* createValue(const char* pb, uint32_t size); + + uint8_t version() { return header_.ver_; } + + FbsonValue* getValue() { return ((FbsonValue*)payload_); } + + ObjectVal* operator->() { return ((ObjectVal*)payload_); } + + const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); } + + private: + /* + * FbsonHeader class defines FBSON header (internal to FbsonDocument). + * + * Currently it only contains version information (1-byte). We may expand the + * header to include checksum of the FBSON binary for more security. + */ + struct FbsonHeader { + uint8_t ver_; + } header_; + + char payload_[0]; + + FbsonDocument(); +}; + +/* + * FbsonFwdIteratorT implements FBSON's iterator template. + * + * Note: it is an FORWARD iterator only due to the design of FBSON format. + */ +template +class FbsonFwdIteratorT { + typedef Iter_Type iterator; + typedef typename std::iterator_traits::pointer pointer; + typedef typename std::iterator_traits::reference reference; + + public: + explicit FbsonFwdIteratorT(const iterator& i) : current_(i) {} + + // allow non-const to const iterator conversion (same container type) + template + FbsonFwdIteratorT(const FbsonFwdIteratorT& rhs) + : current_(rhs.base()) {} + + bool operator==(const FbsonFwdIteratorT& rhs) const { + return (current_ == rhs.current_); + } + + bool operator!=(const FbsonFwdIteratorT& rhs) const { + return !operator==(rhs); + } + + bool operator<(const FbsonFwdIteratorT& rhs) const { + return (current_ < rhs.current_); + } + + bool operator>(const FbsonFwdIteratorT& rhs) const { return !operator<(rhs); } + + FbsonFwdIteratorT& operator++() { + current_ = (iterator)(((char*)current_) + current_->numPackedBytes()); + return *this; + } + + FbsonFwdIteratorT operator++(int) { + auto tmp = *this; + current_ = (iterator)(((char*)current_) + current_->numPackedBytes()); + return tmp; + } + + explicit operator pointer() { return current_; } + + reference operator*() const { return *current_; } + + pointer operator->() const { return current_; } + + iterator base() const { return current_; } + + private: + iterator current_; +}; + +typedef int (*hDictInsert)(const char* key, unsigned len); +typedef int (*hDictFind)(const char* key, unsigned len); + +/* + * FbsonType defines 10 primitive types and 2 container types, as described + * below. + * + * primitive_value ::= + * 0x00 //null value (0 byte) + * | 0x01 //boolean true (0 byte) + * | 0x02 //boolean false (0 byte) + * | 0x03 int8 //char/int8 (1 byte) + * | 0x04 int16 //int16 (2 bytes) + * | 0x05 int32 //int32 (4 bytes) + * | 0x06 int64 //int64 (8 bytes) + * | 0x07 double //floating point (8 bytes) + * | 0x08 string //variable length string + * | 0x09 binary //variable length binary + * + * container ::= + * 0x0A int32 key_value_list //object, int32 is the total bytes of the object + * | 0x0B int32 value_list //array, int32 is the total bytes of the array + */ +enum class FbsonType : char { + T_Null = 0x00, + T_True = 0x01, + T_False = 0x02, + T_Int8 = 0x03, + T_Int16 = 0x04, + T_Int32 = 0x05, + T_Int64 = 0x06, + T_Double = 0x07, + T_String = 0x08, + T_Binary = 0x09, + T_Object = 0x0A, + T_Array = 0x0B, + NUM_TYPES, +}; + +typedef std::underlying_type::type FbsonTypeUnder; + +/* + * FbsonKeyValue class defines FBSON key type, as described below. + * + * key ::= + * 0x00 int8 //1-byte dictionary id + * | int8 (byte*) //int8 (>0) is the size of the key string + * + * value ::= primitive_value | container + * + * FbsonKeyValue can be either an id mapping to the key string in an external + * dictionary, or it is the original key string. Whether to read an id or a + * string is decided by the first byte (size_). + * + * Note: a key object must be followed by a value object. Therefore, a key + * object implicitly refers to a key-value pair, and you can get the value + * object right after the key object. The function numPackedBytes hence + * indicates the total size of the key-value pair, so that we will be able go + * to next pair from the key. + * + * ** Dictionary size ** + * By default, the dictionary size is 255 (1-byte). Users can define + * "USE_LARGE_DICT" to increase the dictionary size to 655535 (2-byte). + */ +class FbsonKeyValue { + public: +#ifdef USE_LARGE_DICT + static const int sMaxKeyId = 65535; + typedef uint16_t keyid_type; +#else + static const int sMaxKeyId = 255; + typedef uint8_t keyid_type; +#endif // #ifdef USE_LARGE_DICT + + static const uint8_t sMaxKeyLen = 64; + + // size of the key. 0 indicates it is stored as id + uint8_t klen() const { return size_; } + + // get the key string. Note the string may not be null terminated. + const char* getKeyStr() const { return key_.str_; } + + keyid_type getKeyId() const { return key_.id_; } + + unsigned int keyPackedBytes() const { + return size_ ? (sizeof(size_) + size_) + : (sizeof(size_) + sizeof(keyid_type)); + } + + FbsonValue* value() const { + return (FbsonValue*)(((char*)this) + keyPackedBytes()); + } + + // size of the total packed bytes (key+value) + unsigned int numPackedBytes() const; + + private: + uint8_t size_; + + union key_ { + keyid_type id_; + char str_[1]; + } key_; + + FbsonKeyValue(); +}; + +/* + * FbsonValue is the base class of all FBSON types. It contains only one member + * variable - type info, which can be retrieved by member functions is[Type]() + * or type(). + */ +class FbsonValue { + public: + static const uint32_t sMaxValueLen = 1 << 24; // 16M + + bool isNull() const { return (type_ == FbsonType::T_Null); } + bool isTrue() const { return (type_ == FbsonType::T_True); } + bool isFalse() const { return (type_ == FbsonType::T_False); } + bool isInt8() const { return (type_ == FbsonType::T_Int8); } + bool isInt16() const { return (type_ == FbsonType::T_Int16); } + bool isInt32() const { return (type_ == FbsonType::T_Int32); } + bool isInt64() const { return (type_ == FbsonType::T_Int64); } + bool isDouble() const { return (type_ == FbsonType::T_Double); } + bool isString() const { return (type_ == FbsonType::T_String); } + bool isBinary() const { return (type_ == FbsonType::T_Binary); } + bool isObject() const { return (type_ == FbsonType::T_Object); } + bool isArray() const { return (type_ == FbsonType::T_Array); } + + FbsonType type() const { return type_; } + + // size of the total packed bytes + unsigned int numPackedBytes() const; + + // size of the value in bytes + unsigned int size() const; + + // get the raw byte array of the value + const char* getValuePtr() const; + + // find the FBSON value by a key path string (null terminated) + FbsonValue* findPath(const char* key_path, + const char* delim = ".", + hDictFind handler = nullptr) { + return findPath(key_path, (unsigned int)strlen(key_path), delim, handler); + } + + // find the FBSON value by a key path string (with length) + FbsonValue* findPath(const char* key_path, + unsigned int len, + const char* delim, + hDictFind handler); + + protected: + FbsonType type_; // type info + + FbsonValue(); +}; + +/* + * NumerValT is the template class (derived from FbsonValue) of all number + * types (integers and double). + */ +template +class NumberValT : public FbsonValue { + public: + T val() const { return num_; } + + unsigned int numPackedBytes() const { return sizeof(FbsonValue) + sizeof(T); } + + // catch all unknow specialization of the template class + bool setVal(T value) { return false; } + + private: + T num_; + + NumberValT(); +}; + +typedef NumberValT Int8Val; + +// override setVal for Int8Val +template <> +inline bool Int8Val::setVal(int8_t value) { + if (!isInt8()) { + return false; + } + + num_ = value; + return true; +} + +typedef NumberValT Int16Val; + +// override setVal for Int16Val +template <> +inline bool Int16Val::setVal(int16_t value) { + if (!isInt16()) { + return false; + } + + num_ = value; + return true; +} + +typedef NumberValT Int32Val; + +// override setVal for Int32Val +template <> +inline bool Int32Val::setVal(int32_t value) { + if (!isInt32()) { + return false; + } + + num_ = value; + return true; +} + +typedef NumberValT Int64Val; + +// override setVal for Int64Val +template <> +inline bool Int64Val::setVal(int64_t value) { + if (!isInt64()) { + return false; + } + + num_ = value; + return true; +} + +typedef NumberValT DoubleVal; + +// override setVal for DoubleVal +template <> +inline bool DoubleVal::setVal(double value) { + if (!isDouble()) { + return false; + } + + num_ = value; + return true; +} + +/* + * BlobVal is the base class (derived from FbsonValue) for string and binary + * types. The size_ indicates the total bytes of the payload_. + */ +class BlobVal : public FbsonValue { + public: + // size of the blob payload only + unsigned int getBlobLen() const { return size_; } + + // return the blob as byte array + const char* getBlob() const { return payload_; } + + // size of the total packed bytes + unsigned int numPackedBytes() const { + return sizeof(FbsonValue) + sizeof(size_) + size_; + } + + protected: + uint32_t size_; + char payload_[0]; + + // set new blob bytes + bool internalSetVal(const char* blob, uint32_t blobSize) { + // if we cannot fit the new blob, fail the operation + if (blobSize > size_) { + return false; + } + + memcpy(payload_, blob, blobSize); + + // Set the reset of the bytes to 0. Note we cannot change the size_ of the + // current payload, as all values are packed. + memset(payload_ + blobSize, 0, size_ - blobSize); + + return true; + } + + BlobVal(); +}; + +/* + * Binary type + */ +class BinaryVal : public BlobVal { + public: + bool setVal(const char* blob, uint32_t blobSize) { + if (!isBinary()) { + return false; + } + + return internalSetVal(blob, blobSize); + } + + private: + BinaryVal(); +}; + +/* + * String type + * Note: FBSON string may not be a c-string (NULL-terminated) + */ +class StringVal : public BlobVal { + public: + bool setVal(const char* str, uint32_t blobSize) { + if (!isString()) { + return false; + } + + return internalSetVal(str, blobSize); + } + + private: + StringVal(); +}; + +/* + * ContainerVal is the base class (derived from FbsonValue) for object and + * array types. The size_ indicates the total bytes of the payload_. + */ +class ContainerVal : public FbsonValue { + public: + // size of the container payload only + unsigned int getContainerSize() const { return size_; } + + // return the container payload as byte array + const char* getPayload() const { return payload_; } + + // size of the total packed bytes + unsigned int numPackedBytes() const { + return sizeof(FbsonValue) + sizeof(size_) + size_; + } + + protected: + uint32_t size_; + char payload_[0]; + + ContainerVal(); +}; + +/* + * Object type + */ +class ObjectVal : public ContainerVal { + public: + // find the FBSON value by a key string (null terminated) + FbsonValue* find(const char* key, hDictFind handler = nullptr) const { + if (!key) + return nullptr; + + return find(key, (unsigned int)strlen(key), handler); + } + + // find the FBSON value by a key string (with length) + FbsonValue* find(const char* key, + unsigned int klen, + hDictFind handler = nullptr) const { + if (!key || !klen) + return nullptr; + + int key_id = -1; + if (handler && (key_id = handler(key, klen)) >= 0) { + return find(key_id); + } + + return internalFind(key, klen); + } + + // find the FBSON value by a key dictionary ID + FbsonValue* find(int key_id) const { + if (key_id < 0 || key_id > FbsonKeyValue::sMaxKeyId) + return nullptr; + + const char* pch = payload_; + const char* fence = payload_ + size_; + + while (pch < fence) { + FbsonKeyValue* pkey = (FbsonKeyValue*)(pch); + if (!pkey->klen() && key_id == pkey->getKeyId()) { + return pkey->value(); + } + pch += pkey->numPackedBytes(); + } + + assert(pch == fence); + + return nullptr; + } + + typedef FbsonKeyValue value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef FbsonFwdIteratorT iterator; + typedef FbsonFwdIteratorT const_iterator; + + iterator begin() { return iterator((pointer)payload_); } + + const_iterator begin() const { return const_iterator((pointer)payload_); } + + iterator end() { return iterator((pointer)(payload_ + size_)); } + + const_iterator end() const { + return const_iterator((pointer)(payload_ + size_)); + } + + private: + FbsonValue* internalFind(const char* key, unsigned int klen) const { + const char* pch = payload_; + const char* fence = payload_ + size_; + + while (pch < fence) { + FbsonKeyValue* pkey = (FbsonKeyValue*)(pch); + if (klen == pkey->klen() && strncmp(key, pkey->getKeyStr(), klen) == 0) { + return pkey->value(); + } + pch += pkey->numPackedBytes(); + } + + assert(pch == fence); + + return nullptr; + } + + private: + ObjectVal(); +}; + +/* + * Array type + */ +class ArrayVal : public ContainerVal { + public: + // get the FBSON value at index + FbsonValue* get(int idx) const { + if (idx < 0) + return nullptr; + + const char* pch = payload_; + const char* fence = payload_ + size_; + + while (pch < fence && idx-- > 0) + pch += ((FbsonValue*)pch)->numPackedBytes(); + + if (idx == -1) + return (FbsonValue*)pch; + else { + assert(pch == fence); + return nullptr; + } + } + + // Get number of elements in array + unsigned int numElem() const { + const char* pch = payload_; + const char* fence = payload_ + size_; + + unsigned int num = 0; + while (pch < fence) { + ++num; + pch += ((FbsonValue*)pch)->numPackedBytes(); + } + + assert(pch == fence); + + return num; + } + + typedef FbsonValue value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef FbsonFwdIteratorT iterator; + typedef FbsonFwdIteratorT const_iterator; + + iterator begin() { return iterator((pointer)payload_); } + + const_iterator begin() const { return const_iterator((pointer)payload_); } + + iterator end() { return iterator((pointer)(payload_ + size_)); } + + const_iterator end() const { + return const_iterator((pointer)(payload_ + size_)); + } + + private: + ArrayVal(); +}; + +inline FbsonDocument* FbsonDocument::createDocument(const char* pb, + uint32_t size) { + if (!pb || size < sizeof(FbsonHeader) + sizeof(FbsonValue)) { + return nullptr; + } + + FbsonDocument* doc = (FbsonDocument*)pb; + if (doc->header_.ver_ != FBSON_VER) { + return nullptr; + } + + FbsonValue* val = (FbsonValue*)doc->payload_; + if (!val->isObject() || size != sizeof(FbsonHeader) + val->numPackedBytes()) { + return nullptr; + } + + return doc; +} + +inline FbsonValue* FbsonDocument::createValue(const char* pb, uint32_t size) { + if (!pb || size < sizeof(FbsonHeader) + sizeof(FbsonValue)) { + return nullptr; + } + + FbsonDocument* doc = (FbsonDocument*)pb; + if (doc->header_.ver_ != FBSON_VER) { + return nullptr; + } + + FbsonValue* val = (FbsonValue*)doc->payload_; + if (size != sizeof(FbsonHeader) + val->numPackedBytes()) { + return nullptr; + } + + return val; +} + +inline unsigned int FbsonKeyValue::numPackedBytes() const { + unsigned int ks = keyPackedBytes(); + FbsonValue* val = (FbsonValue*)(((char*)this) + ks); + return ks + val->numPackedBytes(); +} + +// Poor man's "virtual" function FbsonValue::numPackedBytes +inline unsigned int FbsonValue::numPackedBytes() const { + switch (type_) { + case FbsonType::T_Null: + case FbsonType::T_True: + case FbsonType::T_False: { + return sizeof(type_); + } + + case FbsonType::T_Int8: { + return sizeof(type_) + sizeof(int8_t); + } + case FbsonType::T_Int16: { + return sizeof(type_) + sizeof(int16_t); + } + case FbsonType::T_Int32: { + return sizeof(type_) + sizeof(int32_t); + } + case FbsonType::T_Int64: { + return sizeof(type_) + sizeof(int64_t); + } + case FbsonType::T_Double: { + return sizeof(type_) + sizeof(double); + } + case FbsonType::T_String: + case FbsonType::T_Binary: { + return ((BlobVal*)(this))->numPackedBytes(); + } + + case FbsonType::T_Object: + case FbsonType::T_Array: { + return ((ContainerVal*)(this))->numPackedBytes(); + } + default: + return 0; + } +} + +inline unsigned int FbsonValue::size() const { + switch (type_) { + case FbsonType::T_Int8: { + return sizeof(int8_t); + } + case FbsonType::T_Int16: { + return sizeof(int16_t); + } + case FbsonType::T_Int32: { + return sizeof(int32_t); + } + case FbsonType::T_Int64: { + return sizeof(int64_t); + } + case FbsonType::T_Double: { + return sizeof(double); + } + case FbsonType::T_String: + case FbsonType::T_Binary: { + return ((BlobVal*)(this))->getBlobLen(); + } + + case FbsonType::T_Object: + case FbsonType::T_Array: { + return ((ContainerVal*)(this))->getContainerSize(); + } + case FbsonType::T_Null: + case FbsonType::T_True: + case FbsonType::T_False: + default: + return 0; + } +} + +inline const char* FbsonValue::getValuePtr() const { + switch (type_) { + case FbsonType::T_Int8: + case FbsonType::T_Int16: + case FbsonType::T_Int32: + case FbsonType::T_Int64: + case FbsonType::T_Double: + return ((char*)this) + sizeof(FbsonType); + + case FbsonType::T_String: + case FbsonType::T_Binary: + return ((BlobVal*)(this))->getBlob(); + + case FbsonType::T_Object: + case FbsonType::T_Array: + return ((ContainerVal*)(this))->getPayload(); + + case FbsonType::T_Null: + case FbsonType::T_True: + case FbsonType::T_False: + default: + return nullptr; + } +} + +inline FbsonValue* FbsonValue::findPath(const char* key_path, + unsigned int kp_len, + const char* delim = ".", + hDictFind handler = nullptr) { + if (!key_path || !kp_len) + return nullptr; + + if (!delim) + delim = "."; // default delimiter + + FbsonValue* pval = this; + const char* fence = key_path + kp_len; + char idx_buf[21]; // buffer to parse array index (integer value) + + while (pval && key_path < fence) { + const char* key = key_path; + unsigned int klen = 0; + // find the current key + for (; key_path != fence && *key_path != *delim; ++key_path, ++klen) + ; + + if (!klen) + return nullptr; + + switch (pval->type_) { + case FbsonType::T_Object: { + pval = ((ObjectVal*)pval)->find(key, klen, handler); + break; + } + + case FbsonType::T_Array: { + // parse string into an integer (array index) + if (klen >= sizeof(idx_buf)) + return nullptr; + + memcpy(idx_buf, key, klen); + idx_buf[klen] = 0; + + char* end = nullptr; + int index = (int)strtol(idx_buf, &end, 10); + if (end && !*end) + pval = ((fbson::ArrayVal*)pval)->get(index); + else + // incorrect index string + return nullptr; + break; + } + + default: + return nullptr; + } + + // skip the delimiter + if (key_path < fence) { + ++key_path; + if (key_path == fence) + // we have a trailing delimiter at the end + return nullptr; + } + } + + return pval; +} + +#pragma pack(pop) + +} // namespace fbson + +#endif // FBSON_FBSONDOCUMENT_H diff --git a/third-party/fbson/FbsonJsonParser.h b/third-party/fbson/FbsonJsonParser.h new file mode 100644 index 0000000000..3525b68b5b --- /dev/null +++ b/third-party/fbson/FbsonJsonParser.h @@ -0,0 +1,746 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +/* + * This file defines FbsonJsonParserT (template) and FbsonJsonParser. + * + * FbsonJsonParserT is a template class which implements a JSON parser. + * FbsonJsonParserT parses JSON text, and serialize it to FBSON binary format + * by using FbsonWriterT object. By default, FbsonJsonParserT creates a new + * FbsonWriterT object with an output stream object. However, you can also + * pass in your FbsonWriterT or any stream object that implements some basic + * interface of std::ostream (see FbsonStream.h). + * + * FbsonJsonParser specializes FbsonJsonParserT with FbsonOutStream type (see + * FbsonStream.h). So unless you want to provide own a different output stream + * type, use FbsonJsonParser object. + * + * ** Parsing JSON ** + * FbsonJsonParserT parses JSON string, and directly serializes into FBSON + * packed bytes. There are three ways to parse a JSON string: (1) using + * c-string, (2) using string with len, (3) using std::istream object. You can + * use custome streambuf to redirect output. FbsonOutBuffer is a streambuf used + * internally if the input is raw character buffer. + * + * You can reuse an FbsonJsonParserT object to parse/serialize multiple JSON + * strings, and the previous FBSON will be overwritten. + * + * If parsing fails (returned false), the error code will be set to one of + * FbsonErrType, and can be retrieved by calling getErrorCode(). + * + * ** External dictionary ** + * During parsing a JSON string, you can pass a call-back function to map a key + * string to an id, and store the dictionary id in FBSON to save space. The + * purpose of using an external dictionary is more towards a collection of + * documents (which has common keys) rather than a single document, so that + * space saving will be siginificant. + * + * ** Endianness ** + * Note: FBSON serialization doesn't assume endianness of the server. However + * you will need to ensure that the endianness at the reader side is the same + * as that at the writer side (if they are on different machines). Otherwise, + * proper conversion is needed when a number value is returned to the + * caller/writer. + * + * @author Tian Xia + */ + +#ifndef FBSON_FBSONPARSER_H +#define FBSON_FBSONPARSER_H + +#include +#include +#include "FbsonDocument.h" +#include "FbsonWriter.h" + +namespace fbson { + +const char* const kJsonDelim = " ,]}\t\r\n"; +const char* const kWhiteSpace = " \t\n\r"; + +/* + * Error codes + */ +enum class FbsonErrType { + E_NONE = 0, + E_INVALID_VER, + E_EMPTY_STR, + E_OUTPUT_FAIL, + E_INVALID_DOCU, + E_INVALID_VALUE, + E_INVALID_KEY, + E_INVALID_STR, + E_INVALID_OBJ, + E_INVALID_ARR, + E_INVALID_HEX, + E_INVALID_OCTAL, + E_INVALID_DECIMAL, + E_INVALID_EXPONENT, + E_HEX_OVERFLOW, + E_OCTAL_OVERFLOW, + E_DECIMAL_OVERFLOW, + E_DOUBLE_OVERFLOW, + E_EXPONENT_OVERFLOW, +}; + +/* + * Template FbsonJsonParserT + */ +template +class FbsonJsonParserT { + public: + FbsonJsonParserT() : err_(FbsonErrType::E_NONE) {} + + explicit FbsonJsonParserT(OS_TYPE& os) + : writer_(os), err_(FbsonErrType::E_NONE) {} + + // parse a UTF-8 JSON string + bool parse(const std::string& str, hDictInsert handler = nullptr) { + return parse(str.c_str(), (unsigned int)str.size(), handler); + } + + // parse a UTF-8 JSON c-style string (NULL terminated) + bool parse(const char* c_str, hDictInsert handler = nullptr) { + return parse(c_str, (unsigned int)strlen(c_str), handler); + } + + // parse a UTF-8 JSON string with length + bool parse(const char* pch, unsigned int len, hDictInsert handler = nullptr) { + if (!pch || len == 0) { + err_ = FbsonErrType::E_EMPTY_STR; + return false; + } + + FbsonInBuffer sb(pch, len); + std::istream in(&sb); + return parse(in, handler); + } + + // parse UTF-8 JSON text from an input stream + bool parse(std::istream& in, hDictInsert handler = nullptr) { + bool res = false; + + // reset output stream + writer_.reset(); + + trim(in); + + if (in.peek() == '{') { + in.ignore(); + res = parseObject(in, handler); + } else if (in.peek() == '[') { + in.ignore(); + res = parseArray(in, handler); + } else { + err_ = FbsonErrType::E_INVALID_DOCU; + } + + trim(in); + if (res && !in.eof()) { + err_ = FbsonErrType::E_INVALID_DOCU; + return false; + } + + return res; + } + + FbsonWriterT& getWriter() { return writer_; } + + FbsonErrType getErrorCode() { return err_; } + + // clear error code + void clearErr() { err_ = FbsonErrType::E_NONE; } + + private: + // parse a JSON object (comma-separated list of key-value pairs) + bool parseObject(std::istream& in, hDictInsert handler) { + if (!writer_.writeStartObject()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + trim(in); + + if (in.peek() == '}') { + in.ignore(); + // empty object + if (!writer_.writeEndObject()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + return true; + } + + while (in.good()) { + if (in.get() != '"') { + err_ = FbsonErrType::E_INVALID_KEY; + return false; + } + + if (!parseKVPair(in, handler)) { + return false; + } + + trim(in); + + char ch = in.get(); + if (ch == '}') { + // end of the object + if (!writer_.writeEndObject()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + return true; + } else if (ch != ',') { + err_ = FbsonErrType::E_INVALID_OBJ; + return false; + } + + trim(in); + } + + err_ = FbsonErrType::E_INVALID_OBJ; + return false; + } + + // parse a JSON array (comma-separated list of values) + bool parseArray(std::istream& in, hDictInsert handler) { + if (!writer_.writeStartArray()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + trim(in); + + if (in.peek() == ']') { + in.ignore(); + // empty array + if (!writer_.writeEndArray()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + return true; + } + + while (in.good()) { + if (!parseValue(in, handler)) { + return false; + } + + trim(in); + + char ch = in.get(); + if (ch == ']') { + // end of the array + if (!writer_.writeEndArray()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + return true; + } else if (ch != ',') { + err_ = FbsonErrType::E_INVALID_ARR; + return false; + } + + trim(in); + } + + err_ = FbsonErrType::E_INVALID_ARR; + return false; + } + + // parse a key-value pair, separated by ":" + bool parseKVPair(std::istream& in, hDictInsert handler) { + if (parseKey(in, handler) && parseValue(in, handler)) { + return true; + } + + return false; + } + + // parse a key (must be string) + bool parseKey(std::istream& in, hDictInsert handler) { + char key[FbsonKeyValue::sMaxKeyLen]; + int i = 0; + while (in.good() && in.peek() != '"' && i < FbsonKeyValue::sMaxKeyLen) { + key[i++] = in.get(); + } + + if (!in.good() || in.peek() != '"' || i == 0) { + err_ = FbsonErrType::E_INVALID_KEY; + return false; + } + + in.ignore(); // discard '"' + + int key_id = -1; + if (handler) { + key_id = handler(key, i); + } + + if (key_id < 0) { + writer_.writeKey(key, i); + } else { + writer_.writeKey(key_id); + } + + trim(in); + + if (in.get() != ':') { + err_ = FbsonErrType::E_INVALID_OBJ; + return false; + } + + return true; + } + + // parse a value + bool parseValue(std::istream& in, hDictInsert handler) { + bool res = false; + + trim(in); + + switch (in.peek()) { + case 'N': + case 'n': { + in.ignore(); + res = parseNull(in); + break; + } + case 'T': + case 't': { + in.ignore(); + res = parseTrue(in); + break; + } + case 'F': + case 'f': { + in.ignore(); + res = parseFalse(in); + break; + } + case '"': { + in.ignore(); + res = parseString(in); + break; + } + case '{': { + in.ignore(); + res = parseObject(in, handler); + break; + } + case '[': { + in.ignore(); + res = parseArray(in, handler); + break; + } + default: { + res = parseNumber(in); + break; + } + } + + return res; + } + + // parse NULL value + bool parseNull(std::istream& in) { + if (tolower(in.get()) == 'u' && tolower(in.get()) == 'l' && + tolower(in.get()) == 'l') { + writer_.writeNull(); + return true; + } + + err_ = FbsonErrType::E_INVALID_VALUE; + return false; + } + + // parse TRUE value + bool parseTrue(std::istream& in) { + if (tolower(in.get()) == 'r' && tolower(in.get()) == 'u' && + tolower(in.get()) == 'e') { + writer_.writeBool(true); + return true; + } + + err_ = FbsonErrType::E_INVALID_VALUE; + return false; + } + + // parse FALSE value + bool parseFalse(std::istream& in) { + if (tolower(in.get()) == 'a' && tolower(in.get()) == 'l' && + tolower(in.get()) == 's' && tolower(in.get()) == 'e') { + writer_.writeBool(false); + return true; + } + + err_ = FbsonErrType::E_INVALID_VALUE; + return false; + } + + // parse a string + bool parseString(std::istream& in) { + if (!writer_.writeStartString()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + bool escaped = false; + char buffer[4096]; // write 4KB at a time + int nread = 0; + while (in.good()) { + char ch = in.get(); + if (ch != '"' || escaped) { + buffer[nread++] = ch; + if (nread == 4096) { + // flush buffer + if (!writer_.writeString(buffer, nread)) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + nread = 0; + } + // set/reset escape + if (ch == '\\' || escaped) { + escaped = !escaped; + } + } else { + // write all remaining bytes in the buffer + if (nread > 0) { + if (!writer_.writeString(buffer, nread)) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + } + // end writing string + if (!writer_.writeEndString()) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + return true; + } + } + + err_ = FbsonErrType::E_INVALID_STR; + return false; + } + + // parse a number + // Number format can be hex, octal, or decimal (including float). + // Only decimal can have (+/-) sign prefix. + bool parseNumber(std::istream& in) { + bool ret = false; + switch (in.peek()) { + case '0': { + in.ignore(); + + if (in.peek() == 'x' || in.peek() == 'X') { + in.ignore(); + ret = parseHex(in); + } else if (in.peek() == '.') { + in.ignore(); + ret = parseDouble(in, 0, 0, 1); + } else { + ret = parseOctal(in); + } + + break; + } + case '-': { + in.ignore(); + ret = parseDecimal(in, -1); + break; + } + case '+': + in.ignore(); + // fall through + default: + ret = parseDecimal(in, 1); + break; + } + + return ret; + } + + // parse a number in hex format + bool parseHex(std::istream& in) { + uint64_t val = 0; + int num_digits = 0; + char ch = tolower(in.peek()); + while (in.good() && !strchr(kJsonDelim, ch) && (++num_digits) <= 16) { + if (ch >= '0' && ch <= '9') { + val = (val << 4) + (ch - '0'); + } else if (ch >= 'a' && ch <= 'f') { + val = (val << 4) + (ch - 'a' + 10); + } else { // unrecognized hex digit + err_ = FbsonErrType::E_INVALID_HEX; + return false; + } + + in.ignore(); + ch = tolower(in.peek()); + } + + int size = 0; + if (num_digits <= 2) { + size = writer_.writeInt8((int8_t)val); + } else if (num_digits <= 4) { + size = writer_.writeInt16((int16_t)val); + } else if (num_digits <= 8) { + size = writer_.writeInt32((int32_t)val); + } else if (num_digits <= 16) { + size = writer_.writeInt64(val); + } else { + err_ = FbsonErrType::E_HEX_OVERFLOW; + return false; + } + + if (size == 0) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + return true; + } + + // parse a number in octal format + bool parseOctal(std::istream& in) { + int64_t val = 0; + char ch = in.peek(); + while (in.good() && !strchr(kJsonDelim, ch)) { + if (ch >= '0' && ch <= '7') { + val = val * 8 + (ch - '0'); + } else { + err_ = FbsonErrType::E_INVALID_OCTAL; + return false; + } + + // check if the number overflows + if (val < 0) { + err_ = FbsonErrType::E_OCTAL_OVERFLOW; + return false; + } + + in.ignore(); + ch = in.peek(); + } + + int size = 0; + if (val <= std::numeric_limits::max()) { + size = writer_.writeInt8((int8_t)val); + } else if (val <= std::numeric_limits::max()) { + size = writer_.writeInt16((int16_t)val); + } else if (val <= std::numeric_limits::max()) { + size = writer_.writeInt32((int32_t)val); + } else { // val <= INT64_MAX + size = writer_.writeInt64(val); + } + + if (size == 0) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + return true; + } + + // parse a number in decimal (including float) + bool parseDecimal(std::istream& in, int sign) { + int64_t val = 0; + int precision = 0; + + char ch = 0; + while (in.good() && (ch = in.peek()) == '0') + in.ignore(); + + while (in.good() && !strchr(kJsonDelim, ch)) { + if (ch >= '0' && ch <= '9') { + val = val * 10 + (ch - '0'); + ++precision; + } else if (ch == '.') { + // note we don't pop out '.' + return parseDouble(in, val, precision, sign); + } else { + err_ = FbsonErrType::E_INVALID_DECIMAL; + return false; + } + + in.ignore(); + + // if the number overflows int64_t, first parse it as double iff we see a + // decimal point later. Otherwise, will treat it as overflow + if (val < 0 && val > std::numeric_limits::min()) { + return parseDouble(in, (uint64_t)val, precision, sign); + } + + ch = in.peek(); + } + + if (sign < 0) { + val = -val; + } + + int size = 0; + if (val >= std::numeric_limits::min() && + val <= std::numeric_limits::max()) { + size = writer_.writeInt8((int8_t)val); + } else if (val >= std::numeric_limits::min() && + val <= std::numeric_limits::max()) { + size = writer_.writeInt16((int16_t)val); + } else if (val >= std::numeric_limits::min() && + val <= std::numeric_limits::max()) { + size = writer_.writeInt32((int32_t)val); + } else { // val <= INT64_MAX + size = writer_.writeInt64(val); + } + + if (size == 0) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + return true; + } + + // parse IEEE745 double precision: + // Significand precision length - 15 + // Maximum exponent value - 308 + // + // "If a decimal string with at most 15 significant digits is converted to + // IEEE 754 double precision representation and then converted back to a + // string with the same number of significant digits, then the final string + // should match the original" + bool parseDouble(std::istream& in, double val, int precision, int sign) { + int integ = precision; + int frac = 0; + bool is_frac = false; + + char ch = in.peek(); + if (ch == '.') { + is_frac = true; + in.ignore(); + ch = in.peek(); + } + + int exp = 0; + while (in.good() && !strchr(kJsonDelim, ch)) { + if (ch >= '0' && ch <= '9') { + if (precision < 15) { + val = val * 10 + (ch - '0'); + if (is_frac) { + ++frac; + } else { + ++integ; + } + ++precision; + } else if (!is_frac) { + ++exp; + } + } else if (ch == 'e' || ch == 'E') { + in.ignore(); + int exp2; + if (!parseExponent(in, exp2)) { + return false; + } + + exp += exp2; + // check if exponent overflows + if (exp > 308 || exp < -308) { + err_ = FbsonErrType::E_EXPONENT_OVERFLOW; + return false; + } + + is_frac = true; + break; + } + + in.ignore(); + ch = in.peek(); + } + + if (!is_frac) { + err_ = FbsonErrType::E_DECIMAL_OVERFLOW; + return false; + } + + val *= std::pow(10, exp - frac); + if (std::isnan(val) || std::isinf(val)) { + err_ = FbsonErrType::E_DOUBLE_OVERFLOW; + return false; + } + + if (sign < 0) { + val = -val; + } + + if (writer_.writeDouble(val) == 0) { + err_ = FbsonErrType::E_OUTPUT_FAIL; + return false; + } + + return true; + } + + // parse the exponent part of a double number + bool parseExponent(std::istream& in, int& exp) { + bool neg = false; + + char ch = in.peek(); + if (ch == '+') { + in.ignore(); + ch = in.peek(); + } else if (ch == '-') { + neg = true; + in.ignore(); + ch = in.peek(); + } + + exp = 0; + while (in.good() && !strchr(kJsonDelim, ch)) { + if (ch >= '0' && ch <= '9') { + exp = exp * 10 + (ch - '0'); + } else { + err_ = FbsonErrType::E_INVALID_EXPONENT; + return false; + } + + if (exp > 308) { + err_ = FbsonErrType::E_EXPONENT_OVERFLOW; + return false; + } + + in.ignore(); + ch = in.peek(); + } + + if (neg) { + exp = -exp; + } + + return true; + } + + void trim(std::istream& in) { + while (in.good() && strchr(kWhiteSpace, in.peek())) { + in.ignore(); + } + } + + private: + FbsonWriterT writer_; + FbsonErrType err_; +}; + +typedef FbsonJsonParserT FbsonJsonParser; + +} // namespace fbson + +#endif // FBSON_FBSONPARSER_H diff --git a/third-party/fbson/FbsonStream.h b/third-party/fbson/FbsonStream.h new file mode 100644 index 0000000000..82c8233efd --- /dev/null +++ b/third-party/fbson/FbsonStream.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +/* + * This header file defines FbsonInBuffer and FbsonOutStream classes. + * + * ** Input Buffer ** + * FbsonInBuffer is a customer input buffer to wrap raw character buffer. Its + * object instances are used to create std::istream objects interally. + * + * ** Output Stream ** + * FbsonOutStream is a custom output stream classes, to contain the FBSON + * serialized binary. The class is conveniently used to specialize templates of + * FbsonParser and FbsonWriter. + * + * @author Tian Xia + */ + +#ifndef FBSON_FBSONSTREAM_H +#define FBSON_FBSONSTREAM_H + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include +#include + +namespace fbson { + +// lengths includes sign +#define MAX_INT_DIGITS 11 +#define MAX_INT64_DIGITS 20 +#define MAX_DOUBLE_DIGITS 23 // 1(sign)+16(significant)+1(decimal)+5(exponent) + +/* + * FBSON's implementation of input buffer + */ +class FbsonInBuffer : public std::streambuf { + public: + FbsonInBuffer(const char* str, uint32_t len) { + // this is read buffer and the str will not be changed + // so we use const_cast (ugly!) to remove constness + char* pch(const_cast(str)); + setg(pch, pch, pch + len); + } +}; + +/* + * FBSON's implementation of output stream. + * + * This is a wrapper of a char buffer. By default, the buffer capacity is 1024 + * bytes. We will double the buffer if realloc is needed for writes. + */ +class FbsonOutStream : public std::ostream { + public: + explicit FbsonOutStream(uint32_t capacity = 1024) + : head_(nullptr), size_(0), capacity_(capacity), alloc_(true) { + if (capacity_ == 0) { + capacity_ = 1024; + } + + head_ = (char*)malloc(capacity_); + } + + FbsonOutStream(char* buffer, uint32_t capacity) + : head_(buffer), size_(0), capacity_(capacity), alloc_(false) { + assert(buffer && capacity_ > 0); + } + + ~FbsonOutStream() { + if (alloc_) { + free(head_); + } + } + + void put(char c) { write(&c, 1); } + + void write(const char* c_str) { write(c_str, (uint32_t)strlen(c_str)); } + + void write(const char* bytes, uint32_t len) { + if (len == 0) + return; + + if (size_ + len > capacity_) { + realloc(len); + } + + memcpy(head_ + size_, bytes, len); + size_ += len; + } + + // write the integer to string + void write(int i) { + // snprintf automatically adds a NULL, so we need one more char + if (size_ + MAX_INT_DIGITS + 1 > capacity_) { + realloc(MAX_INT_DIGITS + 1); + } + + int len = snprintf(head_ + size_, MAX_INT_DIGITS + 1, "%d", i); + assert(len > 0); + size_ += len; + } + + // write the 64bit integer to string + void write(int64_t l) { + // snprintf automatically adds a NULL, so we need one more char + if (size_ + MAX_INT64_DIGITS + 1 > capacity_) { + realloc(MAX_INT64_DIGITS + 1); + } + + int len = snprintf(head_ + size_, MAX_INT64_DIGITS + 1, "%" PRIi64, l); + assert(len > 0); + size_ += len; + } + + // write the double to string + void write(double d) { + // snprintf automatically adds a NULL, so we need one more char + if (size_ + MAX_DOUBLE_DIGITS + 1 > capacity_) { + realloc(MAX_DOUBLE_DIGITS + 1); + } + + int len = snprintf(head_ + size_, MAX_DOUBLE_DIGITS + 1, "%.15g", d); + assert(len > 0); + size_ += len; + } + + pos_type tellp() const { return size_; } + + void seekp(pos_type pos) { size_ = (uint32_t)pos; } + + const char* getBuffer() const { return head_; } + + pos_type getSize() const { return tellp(); } + + private: + void realloc(uint32_t len) { + assert(capacity_ > 0); + + capacity_ *= 2; + while (capacity_ < size_ + len) { + capacity_ *= 2; + } + + if (alloc_) { + char* new_buf = (char*)::realloc(head_, capacity_); + assert(new_buf); + head_ = new_buf; + } else { + char* new_buf = (char*)::malloc(capacity_); + assert(new_buf); + memcpy(new_buf, head_, size_); + head_ = new_buf; + alloc_ = true; + } + } + + private: + char* head_; + uint32_t size_; + uint32_t capacity_; + bool alloc_; +}; + +} // namespace fbson + +#endif // FBSON_FBSONSTREAM_H diff --git a/third-party/fbson/FbsonUtil.h b/third-party/fbson/FbsonUtil.h new file mode 100644 index 0000000000..ab965630d2 --- /dev/null +++ b/third-party/fbson/FbsonUtil.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +/* + * This header file defines miscellaneous utility classes. + * + * @author Tian Xia + */ + +#ifndef FBSON_FBSONUTIL_H +#define FBSON_FBSONUTIL_H + +#include +#include "FbsonDocument.h" + +namespace fbson { + +#define OUT_BUF_SIZE 1024 + +/* + * FbsonToJson converts an FbsonValue object to a JSON string. + */ +class FbsonToJson { + public: + FbsonToJson() : os_(buffer_, OUT_BUF_SIZE) {} + + // get json string + const char* json(const FbsonValue* pval) { + os_.clear(); + os_.seekp(0); + + if (pval) { + intern_json(pval); + } + + os_.put(0); + return os_.getBuffer(); + } + + private: + // recursively convert FbsonValue + void intern_json(const FbsonValue* val) { + switch (val->type()) { + case FbsonType::T_Null: { + os_.write("null", 4); + break; + } + case FbsonType::T_True: { + os_.write("true", 4); + break; + } + case FbsonType::T_False: { + os_.write("false", 5); + break; + } + case FbsonType::T_Int8: { + os_.write(((Int8Val*)val)->val()); + break; + } + case FbsonType::T_Int16: { + os_.write(((Int16Val*)val)->val()); + break; + } + case FbsonType::T_Int32: { + os_.write(((Int32Val*)val)->val()); + break; + } + case FbsonType::T_Int64: { + os_.write(((Int64Val*)val)->val()); + break; + } + case FbsonType::T_Double: { + os_.write(((DoubleVal*)val)->val()); + break; + } + case FbsonType::T_String: { + os_.put('"'); + os_.write(((StringVal*)val)->getBlob(), ((StringVal*)val)->getBlobLen()); + os_.put('"'); + break; + } + case FbsonType::T_Binary: { + os_.write("\"", 9); + os_.write(((BinaryVal*)val)->getBlob(), ((BinaryVal*)val)->getBlobLen()); + os_.write("\"", 9); + break; + } + case FbsonType::T_Object: { + object_to_json((ObjectVal*)val); + break; + } + case FbsonType::T_Array: { + array_to_json((ArrayVal*)val); + break; + } + default: + break; + } + } + + // convert object + void object_to_json(const ObjectVal* val) { + os_.put('{'); + + auto iter = val->begin(); + auto iter_fence = val->end(); + + while (iter < iter_fence) { + // write key + if (iter->klen()) { + os_.put('"'); + os_.write(iter->getKeyStr(), iter->klen()); + os_.put('"'); + } else { + os_.write(iter->getKeyId()); + } + os_.put(':'); + + // convert value + intern_json(iter->value()); + + ++iter; + if (iter != iter_fence) { + os_.put(','); + } + } + + assert(iter == iter_fence); + + os_.put('}'); + } + + // convert array to json + void array_to_json(const ArrayVal* val) { + os_.put('['); + + auto iter = val->begin(); + auto iter_fence = val->end(); + + while (iter != iter_fence) { + // convert value + intern_json((const FbsonValue*)iter); + ++iter; + if (iter != iter_fence) { + os_.put(','); + } + } + + assert(iter == iter_fence); + + os_.put(']'); + } + + private: + FbsonOutStream os_; + char buffer_[OUT_BUF_SIZE]; +}; + +} // namespace fbson + +#endif // FBSON_FBSONUTIL_H diff --git a/third-party/fbson/FbsonWriter.h b/third-party/fbson/FbsonWriter.h new file mode 100644 index 0000000000..21bd6f232b --- /dev/null +++ b/third-party/fbson/FbsonWriter.h @@ -0,0 +1,435 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +/* + * This file defines FbsonWriterT (template) and FbsonWriter. + * + * FbsonWriterT is a template class which implements an FBSON serializer. + * Users call various write functions of FbsonWriterT object to write values + * directly to FBSON packed bytes. All write functions of value or key return + * the number of bytes written to FBSON, or 0 if there is an error. To write an + * object, an array, or a string, you must call writeStart[..] before writing + * values or key, and call writeEnd[..] after finishing at the end. + * + * By default, an FbsonWriterT object creates an output stream buffer. + * Alternatively, you can also pass any output stream object to a writer, as + * long as the stream object implements some basic functions of std::ostream + * (such as FbsonOutStream, see FbsonStream.h). + * + * FbsonWriter specializes FbsonWriterT with FbsonOutStream type (see + * FbsonStream.h). So unless you want to provide own a different output stream + * type, use FbsonParser object. + * + * @author Tian Xia + */ + +#ifndef FBSON_FBSONWRITER_H +#define FBSON_FBSONWRITER_H + +#include +#include "FbsonDocument.h" +#include "FbsonStream.h" + +namespace fbson { + +template +class FbsonWriterT { + public: + FbsonWriterT() + : alloc_(true), hasHdr_(false), kvState_(WS_Value), str_pos_(0) { + os_ = new OS_TYPE(); + } + + explicit FbsonWriterT(OS_TYPE& os) + : os_(&os), + alloc_(false), + hasHdr_(false), + kvState_(WS_Value), + str_pos_(0) {} + + ~FbsonWriterT() { + if (alloc_) { + delete os_; + } + } + + void reset() { + os_->clear(); + os_->seekp(0); + hasHdr_ = false; + kvState_ = WS_Value; + for (; !stack_.empty(); stack_.pop()) + ; + } + + // write a key string (or key id if an external dict is provided) + uint32_t writeKey(const char* key, + uint8_t len, + hDictInsert handler = nullptr) { + if (len && !stack_.empty() && verifyKeyState()) { + int key_id = -1; + if (handler) { + key_id = handler(key, len); + } + + uint32_t size = sizeof(uint8_t); + if (key_id < 0) { + os_->put(len); + os_->write(key, len); + size += len; + } else if (key_id <= FbsonKeyValue::sMaxKeyId) { + FbsonKeyValue::keyid_type idx = key_id; + os_->put(0); + os_->write((char*)&idx, sizeof(FbsonKeyValue::keyid_type)); + size += sizeof(FbsonKeyValue::keyid_type); + } else { // key id overflow + assert(0); + return 0; + } + + kvState_ = WS_Key; + return size; + } + + return 0; + } + + // write a key id + uint32_t writeKey(FbsonKeyValue::keyid_type idx) { + if (!stack_.empty() && verifyKeyState()) { + os_->put(0); + os_->write((char*)&idx, sizeof(FbsonKeyValue::keyid_type)); + kvState_ = WS_Key; + return sizeof(uint8_t) + sizeof(FbsonKeyValue::keyid_type); + } + + return 0; + } + + uint32_t writeNull() { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Null); + kvState_ = WS_Value; + return sizeof(FbsonValue); + } + + return 0; + } + + uint32_t writeBool(bool b) { + if (!stack_.empty() && verifyValueState()) { + if (b) { + os_->put((FbsonTypeUnder)FbsonType::T_True); + } else { + os_->put((FbsonTypeUnder)FbsonType::T_False); + } + + kvState_ = WS_Value; + return sizeof(FbsonValue); + } + + return 0; + } + + uint32_t writeInt8(int8_t v) { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Int8); + os_->put(v); + kvState_ = WS_Value; + return sizeof(Int8Val); + } + + return 0; + } + + uint32_t writeInt16(int16_t v) { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Int16); + os_->write((char*)&v, sizeof(int16_t)); + kvState_ = WS_Value; + return sizeof(Int16Val); + } + + return 0; + } + + uint32_t writeInt32(int32_t v) { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Int32); + os_->write((char*)&v, sizeof(int32_t)); + kvState_ = WS_Value; + return sizeof(Int32Val); + } + + return 0; + } + + uint32_t writeInt64(int64_t v) { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Int64); + os_->write((char*)&v, sizeof(int64_t)); + kvState_ = WS_Value; + return sizeof(Int64Val); + } + + return 0; + } + + uint32_t writeDouble(double v) { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Double); + os_->write((char*)&v, sizeof(double)); + kvState_ = WS_Value; + return sizeof(DoubleVal); + } + + return 0; + } + + // must call writeStartString before writing a string val + bool writeStartString() { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_String); + str_pos_ = os_->tellp(); + + // fill the size bytes with 0 for now + uint32_t size = 0; + os_->write((char*)&size, sizeof(uint32_t)); + + kvState_ = WS_String; + return true; + } + + return false; + } + + // finish writing a string val + bool writeEndString() { + if (kvState_ == WS_String) { + std::streampos cur_pos = os_->tellp(); + int32_t size = (int32_t)(cur_pos - str_pos_ - sizeof(uint32_t)); + assert(size >= 0); + + os_->seekp(str_pos_); + os_->write((char*)&size, sizeof(uint32_t)); + os_->seekp(cur_pos); + + kvState_ = WS_Value; + return true; + } + + return false; + } + + uint32_t writeString(const char* str, uint32_t len) { + if (kvState_ == WS_String) { + os_->write(str, len); + return len; + } + + return 0; + } + + uint32_t writeString(char ch) { + if (kvState_ == WS_String) { + os_->put(ch); + return 1; + } + + return 0; + } + + // must call writeStartBinary before writing a binary val + bool writeStartBinary() { + if (!stack_.empty() && verifyValueState()) { + os_->put((FbsonTypeUnder)FbsonType::T_Binary); + str_pos_ = os_->tellp(); + + // fill the size bytes with 0 for now + uint32_t size = 0; + os_->write((char*)&size, sizeof(uint32_t)); + + kvState_ = WS_Binary; + return true; + } + + return false; + } + + // finish writing a binary val + bool writeEndBinary() { + if (kvState_ == WS_Binary) { + std::streampos cur_pos = os_->tellp(); + int32_t size = (int32_t)(cur_pos - str_pos_ - sizeof(uint32_t)); + assert(size >= 0); + + os_->seekp(str_pos_); + os_->write((char*)&size, sizeof(uint32_t)); + os_->seekp(cur_pos); + + kvState_ = WS_Value; + return true; + } + + return false; + } + + uint32_t writeBinary(const char* bin, uint32_t len) { + if (kvState_ == WS_Binary) { + os_->write(bin, len); + return len; + } + + return 0; + } + + // must call writeStartObject before writing an object val + bool writeStartObject() { + if (stack_.empty() || verifyValueState()) { + if (stack_.empty()) { + // if this is a new FBSON, write the header + if (!hasHdr_) { + writeHeader(); + } else + return false; + } + + os_->put((FbsonTypeUnder)FbsonType::T_Object); + // save the size position + stack_.push(WriteInfo({WS_Object, os_->tellp()})); + + // fill the size bytes with 0 for now + uint32_t size = 0; + os_->write((char*)&size, sizeof(uint32_t)); + + kvState_ = WS_Value; + return true; + } + + return false; + } + + // finish writing an object val + bool writeEndObject() { + if (!stack_.empty() && stack_.top().state == WS_Object && + kvState_ == WS_Value) { + WriteInfo& ci = stack_.top(); + std::streampos cur_pos = os_->tellp(); + int32_t size = (int32_t)(cur_pos - ci.sz_pos - sizeof(uint32_t)); + assert(size >= 0); + + os_->seekp(ci.sz_pos); + os_->write((char*)&size, sizeof(uint32_t)); + os_->seekp(cur_pos); + stack_.pop(); + + return true; + } + + return false; + } + + // must call writeStartArray before writing an array val + bool writeStartArray() { + if (stack_.empty() || verifyValueState()) { + if (stack_.empty()) { + // if this is a new FBSON, write the header + if (!hasHdr_) { + writeHeader(); + } else + return false; + } + + os_->put((FbsonTypeUnder)FbsonType::T_Array); + // save the size position + stack_.push(WriteInfo({WS_Array, os_->tellp()})); + + // fill the size bytes with 0 for now + uint32_t size = 0; + os_->write((char*)&size, sizeof(uint32_t)); + + kvState_ = WS_Value; + return true; + } + + return false; + } + + // finish writing an array val + bool writeEndArray() { + if (!stack_.empty() && stack_.top().state == WS_Array && + kvState_ == WS_Value) { + WriteInfo& ci = stack_.top(); + std::streampos cur_pos = os_->tellp(); + int32_t size = (int32_t)(cur_pos - ci.sz_pos - sizeof(uint32_t)); + assert(size >= 0); + + os_->seekp(ci.sz_pos); + os_->write((char*)&size, sizeof(uint32_t)); + os_->seekp(cur_pos); + stack_.pop(); + + return true; + } + + return false; + } + + OS_TYPE* getOutput() { return os_; } + + private: + // verify we are in the right state before writing a value + bool verifyValueState() { + assert(!stack_.empty()); + return (stack_.top().state == WS_Object && kvState_ == WS_Key) || + (stack_.top().state == WS_Array && kvState_ == WS_Value); + } + + // verify we are in the right state before writing a key + bool verifyKeyState() { + assert(!stack_.empty()); + return stack_.top().state == WS_Object && kvState_ == WS_Value; + } + + void writeHeader() { + os_->put(FBSON_VER); + hasHdr_ = true; + } + + private: + enum WriteState { + WS_NONE, + WS_Array, + WS_Object, + WS_Key, + WS_Value, + WS_String, + WS_Binary, + }; + + struct WriteInfo { + WriteState state; + std::streampos sz_pos; + }; + + private: + OS_TYPE* os_; + bool alloc_; + bool hasHdr_; + WriteState kvState_; // key or value state + std::streampos str_pos_; + std::stack stack_; +}; + +typedef FbsonWriterT FbsonWriter; + +} // namespace fbson + +#endif // FBSON_FBSONWRITER_H diff --git a/third-party/rapidjson/document.h b/third-party/rapidjson/document.h deleted file mode 100644 index 83d95a33d0..0000000000 --- a/third-party/rapidjson/document.h +++ /dev/null @@ -1,821 +0,0 @@ -#ifndef RAPIDJSON_DOCUMENT_H_ -#define RAPIDJSON_DOCUMENT_H_ - -#include "reader.h" -#include "internal/strfunc.h" -#include // placement new - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4127) // conditional expression is constant -#endif - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// GenericValue - -//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. -/*! - A JSON value can be one of 7 types. This class is a variant type supporting - these types. - - Use the Value if UTF8 and default allocator - - \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) - \tparam Allocator Allocator type for allocating memory of object, array and string. -*/ -#pragma pack (push, 4) -template > -class GenericValue { -public: - //! Name-value pair in an object. - struct Member { - GenericValue name; //!< name of member (must be a string) - GenericValue value; //!< value of member. - }; - - typedef Encoding EncodingType; //!< Encoding type from template parameter. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef Member* MemberIterator; //!< Member iterator for iterating in object. - typedef const Member* ConstMemberIterator; //!< Constant member iterator for iterating in object. - typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. - typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. - - //!@name Constructors and destructor. - //@{ - - //! Default constructor creates a null value. - GenericValue() : flags_(kNullFlag) {} - - //! Copy constructor is not permitted. -private: - GenericValue(const GenericValue& rhs); - -public: - - //! Constructor with JSON value type. - /*! This creates a Value of specified type with default content. - \param type Type of the value. - \note Default content for number is zero. - */ - GenericValue(Type type) { - static const unsigned defaultFlags[7] = { - kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kConstStringFlag, - kNumberFlag | kIntFlag | kUintFlag | kInt64Flag | kUint64Flag | kDoubleFlag - }; - RAPIDJSON_ASSERT(type <= kNumberType); - flags_ = defaultFlags[type]; - memset(&data_, 0, sizeof(data_)); - } - - //! Constructor for boolean value. - GenericValue(bool b) : flags_(b ? kTrueFlag : kFalseFlag) {} - - //! Constructor for int value. - GenericValue(int i) : flags_(kNumberIntFlag) { - data_.n.i64 = i; - if (i >= 0) - flags_ |= kUintFlag | kUint64Flag; - } - - //! Constructor for unsigned value. - GenericValue(unsigned u) : flags_(kNumberUintFlag) { - data_.n.u64 = u; - if (!(u & 0x80000000)) - flags_ |= kIntFlag | kInt64Flag; - } - - //! Constructor for int64_t value. - GenericValue(int64_t i64) : flags_(kNumberInt64Flag) { - data_.n.i64 = i64; - if (i64 >= 0) { - flags_ |= kNumberUint64Flag; - if (!(i64 & 0xFFFFFFFF00000000LL)) - flags_ |= kUintFlag; - if (!(i64 & 0xFFFFFFFF80000000LL)) - flags_ |= kIntFlag; - } - else if (i64 >= -2147483648LL) - flags_ |= kIntFlag; - } - - //! Constructor for uint64_t value. - GenericValue(uint64_t u64) : flags_(kNumberUint64Flag) { - data_.n.u64 = u64; - if (!(u64 & 0x8000000000000000ULL)) - flags_ |= kInt64Flag; - if (!(u64 & 0xFFFFFFFF00000000ULL)) - flags_ |= kUintFlag; - if (!(u64 & 0xFFFFFFFF80000000ULL)) - flags_ |= kIntFlag; - } - - //! Constructor for double value. - GenericValue(double d) : flags_(kNumberDoubleFlag) { data_.n.d = d; } - - //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) { - RAPIDJSON_ASSERT(s != NULL); - flags_ = kConstStringFlag; - data_.s.str = s; - data_.s.length = length; - } - - //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s) { SetStringRaw(s, internal::StrLen(s)); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) { SetStringRaw(s, length, allocator); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) { SetStringRaw(s, internal::StrLen(s), allocator); } - - //! Destructor. - /*! Need to destruct elements of array, members of object, or copy-string. - */ - ~GenericValue() { - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(flags_) { - case kArrayFlag: - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(data_.a.elements); - break; - - case kObjectFlag: - for (Member* m = data_.o.members; m != data_.o.members + data_.o.size; ++m) { - m->name.~GenericValue(); - m->value.~GenericValue(); - } - Allocator::Free(data_.o.members); - break; - - case kCopyStringFlag: - Allocator::Free(const_cast(data_.s.str)); - break; - } - } - } - - //@} - - //!@name Assignment operators - //@{ - - //! Assignment with move semantics. - /*! \param rhs Source of the assignment. It will become a null value after assignment. - */ - GenericValue& operator=(GenericValue& rhs) { - RAPIDJSON_ASSERT(this != &rhs); - this->~GenericValue(); - memcpy(this, &rhs, sizeof(GenericValue)); - rhs.flags_ = kNullFlag; - return *this; - } - - //! Assignment with primitive types. - /*! \tparam T Either Type, int, unsigned, int64_t, uint64_t, const Ch* - \param value The value to be assigned. - */ - template - GenericValue& operator=(T value) { - this->~GenericValue(); - new (this) GenericValue(value); - return *this; - } - //@} - - //!@name Type - //@{ - - Type GetType() const { return static_cast(flags_ & kTypeMask); } - bool IsNull() const { return flags_ == kNullFlag; } - bool IsFalse() const { return flags_ == kFalseFlag; } - bool IsTrue() const { return flags_ == kTrueFlag; } - bool IsBool() const { return (flags_ & kBoolFlag) != 0; } - bool IsObject() const { return flags_ == kObjectFlag; } - bool IsArray() const { return flags_ == kArrayFlag; } - bool IsNumber() const { return (flags_ & kNumberFlag) != 0; } - bool IsInt() const { return (flags_ & kIntFlag) != 0; } - bool IsUint() const { return (flags_ & kUintFlag) != 0; } - bool IsInt64() const { return (flags_ & kInt64Flag) != 0; } - bool IsUint64() const { return (flags_ & kUint64Flag) != 0; } - bool IsDouble() const { return (flags_ & kDoubleFlag) != 0; } - bool IsString() const { return (flags_ & kStringFlag) != 0; } - - //@} - - //!@name Null - //@{ - - GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } - - //@} - - //!@name Bool - //@{ - - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return flags_ == kTrueFlag; } - GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } - - //@} - - //!@name Object - //@{ - - //! Set this value as an empty object. - GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } - - //! Get the value associated with the object's name. - GenericValue& operator[](const Ch* name) { - if (Member* member = FindMember(name)) - return member->value; - else { - static GenericValue NullValue; - return NullValue; - } - } - const GenericValue& operator[](const Ch* name) const { return const_cast(*this)[name]; } - - //! Member iterators. - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.members; } - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.members + data_.o.size; } - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return data_.o.members; } - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return data_.o.members + data_.o.size; } - - //! Check whether a member exists in the object. - bool HasMember(const Ch* name) const { return FindMember(name) != 0; } - - //! Add a member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. - \return The value itself for fluent API. - \note The ownership of name and value will be transfered to this object if success. - */ - GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - Object& o = data_.o; - if (o.size >= o.capacity) { - if (o.capacity == 0) { - o.capacity = kDefaultObjectCapacity; - o.members = (Member*)allocator.Malloc(o.capacity * sizeof(Member)); - } - else { - SizeType oldCapacity = o.capacity; - o.capacity *= 2; - o.members = (Member*)allocator.Realloc(o.members, oldCapacity * sizeof(Member), o.capacity * sizeof(Member)); - } - } - o.members[o.size].name.RawAssign(name); - o.members[o.size].value.RawAssign(value); - o.size++; - return *this; - } - - GenericValue& AddMember(const Ch* name, Allocator& nameAllocator, GenericValue& value, Allocator& allocator) { - GenericValue n(name, internal::StrLen(name), nameAllocator); - return AddMember(n, value, allocator); - } - - GenericValue& AddMember(const Ch* name, GenericValue& value, Allocator& allocator) { - GenericValue n(name, internal::StrLen(name)); - return AddMember(n, value, allocator); - } - - template - GenericValue& AddMember(const Ch* name, T value, Allocator& allocator) { - GenericValue n(name, internal::StrLen(name)); - GenericValue v(value); - return AddMember(n, v, allocator); - } - - //! Remove a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note Removing member is implemented by moving the last member. So the ordering of members is changed. - */ - bool RemoveMember(const Ch* name) { - RAPIDJSON_ASSERT(IsObject()); - if (Member* m = FindMember(name)) { - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); - - Member* last = data_.o.members + (data_.o.size - 1); - if (data_.o.size > 1 && m != last) { - // Move the last one to this place - m->name = last->name; - m->value = last->value; - } - else { - // Only one left, just destroy - m->name.~GenericValue(); - m->value.~GenericValue(); - } - --data_.o.size; - return true; - } - return false; - } - - //@} - - //!@name Array - //@{ - - //! Set this value as an empty array. - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } - - //! Get the number of elements in array. - SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } - - //! Get the capacity of array. - SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } - - //! Check whether the array is empty. - bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } - - //! Remove all elements in the array. - /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. - */ - void Clear() { - RAPIDJSON_ASSERT(IsArray()); - for (SizeType i = 0; i < data_.a.size; ++i) - data_.a.elements[i].~GenericValue(); - data_.a.size = 0; - } - - //! Get an element from array by index. - /*! \param index Zero-based index of element. - \note -\code -Value a(kArrayType); -a.PushBack(123); -int x = a[0].GetInt(); // Error: operator[ is ambiguous, as 0 also mean a null pointer of const char* type. -int y = a[SizeType(0)].GetInt(); // Cast to SizeType will work. -int z = a[0u].GetInt(); // This works too. -\endcode - */ - GenericValue& operator[](SizeType index) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(index < data_.a.size); - return data_.a.elements[index]; - } - const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } - - //! Element iterator - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements; } - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements + data_.a.size; } - ConstValueIterator Begin() const { return const_cast(*this).Begin(); } - ConstValueIterator End() const { return const_cast(*this).End(); } - - //! Request the array to have enough capacity to store elements. - /*! \param newCapacity The capacity that the array at least need to have. - \param allocator The allocator for allocating memory. It must be the same one use previously. - \return The value itself for fluent API. - */ - GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (newCapacity > data_.a.capacity) { - data_.a.elements = (GenericValue*)allocator.Realloc(data_.a.elements, data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)); - data_.a.capacity = newCapacity; - } - return *this; - } - - //! Append a value at the end of the array. - /*! \param value The value to be appended. - \param allocator The allocator for allocating memory. It must be the same one use previously. - \return The value itself for fluent API. - \note The ownership of the value will be transfered to this object if success. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - */ - GenericValue& PushBack(GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (data_.a.size >= data_.a.capacity) - Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : data_.a.capacity * 2, allocator); - data_.a.elements[data_.a.size++].RawAssign(value); - return *this; - } - - template - GenericValue& PushBack(T value, Allocator& allocator) { - GenericValue v(value); - return PushBack(v, allocator); - } - - //! Remove the last element in the array. - GenericValue& PopBack() { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(!Empty()); - data_.a.elements[--data_.a.size].~GenericValue(); - return *this; - } - //@} - - //!@name Number - //@{ - - int GetInt() const { RAPIDJSON_ASSERT(flags_ & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(flags_ & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(flags_ & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(flags_ & kUint64Flag); return data_.n.u64; } - - double GetDouble() const { - RAPIDJSON_ASSERT(IsNumber()); - if ((flags_ & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((flags_ & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((flags_ & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((flags_ & kInt64Flag) != 0) return (double)data_.n.i64; // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((flags_ & kUint64Flag) != 0); return (double)data_.n.u64; // uint64_t -> double (may lose precision) - } - - GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } - GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } - GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } - GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } - GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } - - //@} - - //!@name String - //@{ - - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return data_.s.str; } - - //! Get the length of string. - /*! Since rapidjson permits "\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). - */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return data_.s.length; } - - //! Set this value as a string without copying source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string pointer. - \param length The length of source string, excluding the trailing null terminator. - \return The value itself for fluent API. - */ - GenericValue& SetString(const Ch* s, SizeType length) { this->~GenericValue(); SetStringRaw(s, length); return *this; } - - //! Set this value as a string without copying source string. - /*! \param s source string pointer. - \return The value itself for fluent API. - */ - GenericValue& SetString(const Ch* s) { return SetString(s, internal::StrLen(s)); } - - //! Set this value as a string by copying from source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string. - \param length The length of source string, excluding the trailing null terminator. - \param allocator Allocator for allocating copied buffer. Commonly use document.GetAllocator(). - \return The value itself for fluent API. - */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(s, length, allocator); return *this; } - - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use document.GetAllocator(). - \return The value itself for fluent API. - */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { SetString(s, internal::StrLen(s), allocator); return *this; } - - //@} - - //! Generate events of this value to a Handler. - /*! This function adopts the GoF visitor pattern. - Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. - It can also be used to deep clone this value via GenericDocument, which is also a Handler. - \tparam Handler type of handler. - \param handler An object implementing concept Handler. - */ - template - const GenericValue& Accept(Handler& handler) const { - switch(GetType()) { - case kNullType: handler.Null(); break; - case kFalseType: handler.Bool(false); break; - case kTrueType: handler.Bool(true); break; - - case kObjectType: - handler.StartObject(); - for (Member* m = data_.o.members; m != data_.o.members + data_.o.size; ++m) { - handler.String(m->name.data_.s.str, m->name.data_.s.length, false); - m->value.Accept(handler); - } - handler.EndObject(data_.o.size); - break; - - case kArrayType: - handler.StartArray(); - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - v->Accept(handler); - handler.EndArray(data_.a.size); - break; - - case kStringType: - handler.String(data_.s.str, data_.s.length, false); - break; - - case kNumberType: - if (IsInt()) handler.Int(data_.n.i.i); - else if (IsUint()) handler.Uint(data_.n.u.u); - else if (IsInt64()) handler.Int64(data_.n.i64); - else if (IsUint64()) handler.Uint64(data_.n.u64); - else handler.Double(data_.n.d); - break; - } - return *this; - } - -private: - template - friend class GenericDocument; - - enum { - kBoolFlag = 0x100, - kNumberFlag = 0x200, - kIntFlag = 0x400, - kUintFlag = 0x800, - kInt64Flag = 0x1000, - kUint64Flag = 0x2000, - kDoubleFlag = 0x4000, - kStringFlag = 0x100000, - kCopyFlag = 0x200000, - - // Initial flags of different types. - kNullFlag = kNullType, - kTrueFlag = kTrueType | kBoolFlag, - kFalseFlag = kFalseType | kBoolFlag, - kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, - kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, - kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, - kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, - kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, - kConstStringFlag = kStringType | kStringFlag, - kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, - kObjectFlag = kObjectType, - kArrayFlag = kArrayType, - - kTypeMask = 0xFF // bitwise-and with mask of 0xFF can be optimized by compiler - }; - - static const SizeType kDefaultArrayCapacity = 16; - static const SizeType kDefaultObjectCapacity = 16; - - struct String { - const Ch* str; - SizeType length; - unsigned hashcode; //!< reserved - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // By using proper binary layout, retrieval of different integer types do not need conversions. - union Number { -#if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN - struct I { - int i; - char padding[4]; - }i; - struct U { - unsigned u; - char padding2[4]; - }u; -#else - struct I { - char padding[4]; - int i; - }i; - struct U { - char padding2[4]; - unsigned u; - }u; -#endif - int64_t i64; - uint64_t u64; - double d; - }; // 8 bytes - - struct Object { - Member* members; - SizeType size; - SizeType capacity; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - struct Array { - GenericValue* elements; - SizeType size; - SizeType capacity; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - union Data { - String s; - Number n; - Object o; - Array a; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - //! Find member by name. - Member* FindMember(const Ch* name) { - RAPIDJSON_ASSERT(name); - RAPIDJSON_ASSERT(IsObject()); - - SizeType length = internal::StrLen(name); - - Object& o = data_.o; - for (Member* member = o.members; member != data_.o.members + data_.o.size; ++member) - if (length == member->name.data_.s.length && memcmp(member->name.data_.s.str, name, length * sizeof(Ch)) == 0) - return member; - - return 0; - } - const Member* FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } - - // Initialize this value as array with initial data, without calling destructor. - void SetArrayRaw(GenericValue* values, SizeType count, Allocator& alloctaor) { - flags_ = kArrayFlag; - data_.a.elements = (GenericValue*)alloctaor.Malloc(count * sizeof(GenericValue)); - memcpy(data_.a.elements, values, count * sizeof(GenericValue)); - data_.a.size = data_.a.capacity = count; - } - - //! Initialize this value as object with initial data, without calling destructor. - void SetObjectRaw(Member* members, SizeType count, Allocator& alloctaor) { - flags_ = kObjectFlag; - data_.o.members = (Member*)alloctaor.Malloc(count * sizeof(Member)); - memcpy(data_.o.members, members, count * sizeof(Member)); - data_.o.size = data_.o.capacity = count; - } - - //! Initialize this value as constant string, without calling destructor. - void SetStringRaw(const Ch* s, SizeType length) { - RAPIDJSON_ASSERT(s != NULL); - flags_ = kConstStringFlag; - data_.s.str = s; - data_.s.length = length; - } - - //! Initialize this value as copy string with initial data, without calling destructor. - void SetStringRaw(const Ch* s, SizeType length, Allocator& allocator) { - RAPIDJSON_ASSERT(s != NULL); - flags_ = kCopyStringFlag; - data_.s.str = (Ch *)allocator.Malloc((length + 1) * sizeof(Ch)); - data_.s.length = length; - memcpy(const_cast(data_.s.str), s, length * sizeof(Ch)); - const_cast(data_.s.str)[length] = '\0'; - } - - //! Assignment without calling destructor - void RawAssign(GenericValue& rhs) { - memcpy(this, &rhs, sizeof(GenericValue)); - rhs.flags_ = kNullFlag; - } - - Data data_; - unsigned flags_; -}; -#pragma pack (pop) - -//! Value with UTF8 encoding. -typedef GenericValue > Value; - -/////////////////////////////////////////////////////////////////////////////// -// GenericDocument - -//! A document for parsing JSON text as DOM. -/*! - \implements Handler - \tparam Encoding encoding for both parsing and string storage. - \tparam Alloactor allocator for allocating memory for the DOM, and the stack during parsing. -*/ -template > -class GenericDocument : public GenericValue { -public: - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericValue ValueType; //!< Value type of the document. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - - //! Constructor - /*! \param allocator Optional allocator for allocating stack memory. - \param stackCapacity Initial capacity of stack in bytes. - */ - GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseError_(0), errorOffset_(0) {} - - //! Parse JSON text from an input stream. - /*! \tparam parseFlags Combination of ParseFlag. - \param stream Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(Stream& stream) { - ValueType::SetNull(); // Remove existing root if exist - GenericReader reader; - if (reader.template Parse(stream, *this)) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - this->RawAssign(*stack_.template Pop(1)); // Add this-> to prevent issue 13. - parseError_ = 0; - errorOffset_ = 0; - } - else { - parseError_ = reader.GetParseError(); - errorOffset_ = reader.GetErrorOffset(); - ClearStack(); - } - return *this; - } - - //! Parse JSON text from a mutable string. - /*! \tparam parseFlags Combination of ParseFlag. - \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseInsitu(Ch* str) { - GenericInsituStringStream s(str); - return ParseStream(s); - } - - //! Parse JSON text from a read-only string. - /*! \tparam parseFlags Combination of ParseFlag (must not contain kParseInsituFlag). - \param str Read-only zero-terminated string to be parsed. - */ - template - GenericDocument& Parse(const Ch* str) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - GenericStringStream s(str); - return ParseStream(s); - } - - //! Whether a parse error was occured in the last parsing. - bool HasParseError() const { return parseError_ != 0; } - - //! Get the message of parsing error. - const char* GetParseError() const { return parseError_; } - - //! Get the offset in character of the parsing error. - size_t GetErrorOffset() const { return errorOffset_; } - - //! Get the allocator of this document. - Allocator& GetAllocator() { return stack_.GetAllocator(); } - - //! Get the capacity of stack in bytes. - size_t GetStackCapacity() const { return stack_.GetCapacity(); } - -private: - // Prohibit assignment - GenericDocument& operator=(const GenericDocument&); - - friend class GenericReader; // for Reader to call the following private handler functions - - // Implementation of Handler - void Null() { new (stack_.template Push()) ValueType(); } - void Bool(bool b) { new (stack_.template Push()) ValueType(b); } - void Int(int i) { new (stack_.template Push()) ValueType(i); } - void Uint(unsigned i) { new (stack_.template Push()) ValueType(i); } - void Int64(int64_t i) { new (stack_.template Push()) ValueType(i); } - void Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); } - void Double(double d) { new (stack_.template Push()) ValueType(d); } - - void String(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push()) ValueType(str, length); - } - - void StartObject() { new (stack_.template Push()) ValueType(kObjectType); } - - void EndObject(SizeType memberCount) { - typename ValueType::Member* members = stack_.template Pop(memberCount); - stack_.template Top()->SetObjectRaw(members, (SizeType)memberCount, GetAllocator()); - } - - void StartArray() { new (stack_.template Push()) ValueType(kArrayType); } - - void EndArray(SizeType elementCount) { - ValueType* elements = stack_.template Pop(elementCount); - stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); - } - - void ClearStack() { - if (Allocator::kNeedFree) - while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) - (stack_.template Pop(1))->~ValueType(); - else - stack_.Clear(); - } - - static const size_t kDefaultStackCapacity = 1024; - internal::Stack stack_; - const char* parseError_; - size_t errorOffset_; -}; - -typedef GenericDocument > Document; - -} // namespace rapidjson - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#endif // RAPIDJSON_DOCUMENT_H_ diff --git a/third-party/rapidjson/filestream.h b/third-party/rapidjson/filestream.h deleted file mode 100644 index 885894963f..0000000000 --- a/third-party/rapidjson/filestream.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef RAPIDJSON_FILESTREAM_H_ -#define RAPIDJSON_FILESTREAM_H_ - -#include - -namespace rapidjson { - -//! Wrapper of C file stream for input or output. -/*! - This simple wrapper does not check the validity of the stream. - \implements Stream -*/ -class FileStream { -public: - typedef char Ch; //!< Character type. Only support char. - - FileStream(FILE* fp) : fp_(fp), count_(0) { Read(); } - char Peek() const { return current_; } - char Take() { char c = current_; Read(); return c; } - size_t Tell() const { return count_; } - void Put(char c) { fputc(c, fp_); } - - // Not implemented - char* PutBegin() { return 0; } - size_t PutEnd(char*) { return 0; } - -private: - void Read() { - RAPIDJSON_ASSERT(fp_ != 0); - int c = fgetc(fp_); - if (c != EOF) { - current_ = (char)c; - count_++; - } - else - current_ = '\0'; - } - - FILE* fp_; - char current_; - size_t count_; -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/third-party/rapidjson/internal/pow10.h b/third-party/rapidjson/internal/pow10.h deleted file mode 100644 index bf3a9afb04..0000000000 --- a/third-party/rapidjson/internal/pow10.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef RAPIDJSON_POW10_ -#define RAPIDJSON_POW10_ - -namespace rapidjson { -namespace internal { - -//! Computes integer powers of 10 in double (10.0^n). -/*! This function uses lookup table for fast and accurate results. - \param n positive/negative exponent. Must <= 308. - \return 10.0^n -*/ -inline double Pow10(int n) { - static const double e[] = { // 1e-308...1e308: 617 * 8 bytes = 4936 bytes - 1e-308,1e-307,1e-306,1e-305,1e-304,1e-303,1e-302,1e-301,1e-300, - 1e-299,1e-298,1e-297,1e-296,1e-295,1e-294,1e-293,1e-292,1e-291,1e-290,1e-289,1e-288,1e-287,1e-286,1e-285,1e-284,1e-283,1e-282,1e-281,1e-280, - 1e-279,1e-278,1e-277,1e-276,1e-275,1e-274,1e-273,1e-272,1e-271,1e-270,1e-269,1e-268,1e-267,1e-266,1e-265,1e-264,1e-263,1e-262,1e-261,1e-260, - 1e-259,1e-258,1e-257,1e-256,1e-255,1e-254,1e-253,1e-252,1e-251,1e-250,1e-249,1e-248,1e-247,1e-246,1e-245,1e-244,1e-243,1e-242,1e-241,1e-240, - 1e-239,1e-238,1e-237,1e-236,1e-235,1e-234,1e-233,1e-232,1e-231,1e-230,1e-229,1e-228,1e-227,1e-226,1e-225,1e-224,1e-223,1e-222,1e-221,1e-220, - 1e-219,1e-218,1e-217,1e-216,1e-215,1e-214,1e-213,1e-212,1e-211,1e-210,1e-209,1e-208,1e-207,1e-206,1e-205,1e-204,1e-203,1e-202,1e-201,1e-200, - 1e-199,1e-198,1e-197,1e-196,1e-195,1e-194,1e-193,1e-192,1e-191,1e-190,1e-189,1e-188,1e-187,1e-186,1e-185,1e-184,1e-183,1e-182,1e-181,1e-180, - 1e-179,1e-178,1e-177,1e-176,1e-175,1e-174,1e-173,1e-172,1e-171,1e-170,1e-169,1e-168,1e-167,1e-166,1e-165,1e-164,1e-163,1e-162,1e-161,1e-160, - 1e-159,1e-158,1e-157,1e-156,1e-155,1e-154,1e-153,1e-152,1e-151,1e-150,1e-149,1e-148,1e-147,1e-146,1e-145,1e-144,1e-143,1e-142,1e-141,1e-140, - 1e-139,1e-138,1e-137,1e-136,1e-135,1e-134,1e-133,1e-132,1e-131,1e-130,1e-129,1e-128,1e-127,1e-126,1e-125,1e-124,1e-123,1e-122,1e-121,1e-120, - 1e-119,1e-118,1e-117,1e-116,1e-115,1e-114,1e-113,1e-112,1e-111,1e-110,1e-109,1e-108,1e-107,1e-106,1e-105,1e-104,1e-103,1e-102,1e-101,1e-100, - 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, 1e-83, 1e-82, 1e-81, 1e-80, - 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, - 1e-59, 1e-58, 1e-57, 1e-56, 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, - 1e-39, 1e-38, 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, - 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e+0, - 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, - 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, - 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, - 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, - 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, - 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, - 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, - 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, - 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, - 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, - 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, - 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, - 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, - 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, - 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, - 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 - }; - RAPIDJSON_ASSERT(n <= 308); - return n < -308 ? 0.0 : e[n + 308]; -} - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_POW10_ diff --git a/third-party/rapidjson/internal/stack.h b/third-party/rapidjson/internal/stack.h deleted file mode 100644 index 966893b3fc..0000000000 --- a/third-party/rapidjson/internal/stack.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef RAPIDJSON_INTERNAL_STACK_H_ -#define RAPIDJSON_INTERNAL_STACK_H_ - -namespace rapidjson { -namespace internal { - -/////////////////////////////////////////////////////////////////////////////// -// Stack - -//! A type-unsafe stack for storing different types of data. -/*! \tparam Allocator Allocator for allocating stack memory. -*/ -template -class Stack { -public: - Stack(Allocator* allocator, size_t stack_capacity) : allocator_(allocator), own_allocator_(0), stack_(0), stack_top_(0), stack_end_(0), stack_capacity_(stack_capacity) { - RAPIDJSON_ASSERT(stack_capacity_ > 0); - if (!allocator_) - own_allocator_ = allocator_ = new Allocator(); - stack_top_ = stack_ = (char*)allocator_->Malloc(stack_capacity_); - stack_end_ = stack_ + stack_capacity_; - } - - ~Stack() { - Allocator::Free(stack_); - delete own_allocator_; // Only delete if it is owned by the stack - } - - void Clear() { /*stack_top_ = 0;*/ stack_top_ = stack_; } - - template - T* Push(size_t count = 1) { - // Expand the stack if needed - if (stack_top_ + sizeof(T) * count >= stack_end_) { - size_t new_capacity = stack_capacity_ * 2; - size_t size = GetSize(); - size_t new_size = GetSize() + sizeof(T) * count; - if (new_capacity < new_size) - new_capacity = new_size; - stack_ = (char*)allocator_->Realloc(stack_, stack_capacity_, new_capacity); - stack_capacity_ = new_capacity; - stack_top_ = stack_ + size; - stack_end_ = stack_ + stack_capacity_; - } - T* ret = (T*)stack_top_; - stack_top_ += sizeof(T) * count; - return ret; - } - - template - T* Pop(size_t count) { - RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); - stack_top_ -= count * sizeof(T); - return (T*)stack_top_; - } - - template - T* Top() { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return (T*)(stack_top_ - sizeof(T)); - } - - template - T* Bottom() { return (T*)stack_; } - - Allocator& GetAllocator() { return *allocator_; } - size_t GetSize() const { return stack_top_ - stack_; } - size_t GetCapacity() const { return stack_capacity_; } - -private: - Allocator* allocator_; - Allocator* own_allocator_; - char *stack_; - char *stack_top_; - char *stack_end_; - size_t stack_capacity_; -}; - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_STACK_H_ diff --git a/third-party/rapidjson/internal/strfunc.h b/third-party/rapidjson/internal/strfunc.h deleted file mode 100644 index bbf444fe6d..0000000000 --- a/third-party/rapidjson/internal/strfunc.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ -#define RAPIDJSON_INTERNAL_STRFUNC_H_ - -namespace rapidjson { -namespace internal { - -//! Custom strlen() which works on different character types. -/*! \tparam Ch Character type (e.g. char, wchar_t, short) - \param s Null-terminated input string. - \return Number of characters in the string. - \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. -*/ -template -inline SizeType StrLen(const Ch* s) { - const Ch* p = s; - while (*p != '\0') - ++p; - return SizeType(p - s); -} - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/third-party/rapidjson/license.txt b/third-party/rapidjson/license.txt deleted file mode 100644 index 03d97d163e..0000000000 --- a/third-party/rapidjson/license.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2011 Milo Yip - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/third-party/rapidjson/prettywriter.h b/third-party/rapidjson/prettywriter.h deleted file mode 100644 index 238ff5ff62..0000000000 --- a/third-party/rapidjson/prettywriter.h +++ /dev/null @@ -1,156 +0,0 @@ -#ifndef RAPIDJSON_PRETTYWRITER_H_ -#define RAPIDJSON_PRETTYWRITER_H_ - -#include "writer.h" - -namespace rapidjson { - -//! Writer with indentation and spacing. -/*! - \tparam Stream Type of ouptut stream. - \tparam Encoding Encoding of both source strings and output. - \tparam Allocator Type of allocator for allocating memory of stack. -*/ -template, typename Allocator = MemoryPoolAllocator<> > -class PrettyWriter : public Writer { -public: - typedef Writer Base; - typedef typename Base::Ch Ch; - - //! Constructor - /*! \param stream Output stream. - \param allocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of - */ - PrettyWriter(Stream& stream, Allocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(stream, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} - - //! Set custom indentation. - /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\t', '\n', '\r'). - \param indentCharCount Number of indent characters for each indentation level. - \note The default indentation is 4 spaces. - */ - PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { - RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); - indentChar_ = indentChar; - indentCharCount_ = indentCharCount; - return *this; - } - - //@name Implementation of Handler. - //@{ - - PrettyWriter& Null() { PrettyPrefix(kNullType); Base::WriteNull(); return *this; } - PrettyWriter& Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); Base::WriteBool(b); return *this; } - PrettyWriter& Int(int i) { PrettyPrefix(kNumberType); Base::WriteInt(i); return *this; } - PrettyWriter& Uint(unsigned u) { PrettyPrefix(kNumberType); Base::WriteUint(u); return *this; } - PrettyWriter& Int64(int64_t i64) { PrettyPrefix(kNumberType); Base::WriteInt64(i64); return *this; } - PrettyWriter& Uint64(uint64_t u64) { PrettyPrefix(kNumberType); Base::WriteUint64(u64); return *this; } - PrettyWriter& Double(double d) { PrettyPrefix(kNumberType); Base::WriteDouble(d); return *this; } - - PrettyWriter& String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - PrettyPrefix(kStringType); - Base::WriteString(str, length); - return *this; - } - - PrettyWriter& StartObject() { - PrettyPrefix(kObjectType); - new (Base::level_stack_.template Push()) typename Base::Level(false); - Base::WriteStartObject(); - return *this; - } - - PrettyWriter& EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty) { - Base::stream_.Put('\n'); - WriteIndent(); - } - Base::WriteEndObject(); - return *this; - } - - PrettyWriter& StartArray() { - PrettyPrefix(kArrayType); - new (Base::level_stack_.template Push()) typename Base::Level(true); - Base::WriteStartArray(); - return *this; - } - - PrettyWriter& EndArray(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty) { - Base::stream_.Put('\n'); - WriteIndent(); - } - Base::WriteEndArray(); - return *this; - } - - //@} - - //! Simpler but slower overload. - PrettyWriter& String(const Ch* str) { return String(str, internal::StrLen(str)); } - -protected: - void PrettyPrefix(Type type) { - (void)type; - if (Base::level_stack_.GetSize() != 0) { // this value is not at root - typename Base::Level* level = Base::level_stack_.template Top(); - - if (level->inArray) { - if (level->valueCount > 0) { - Base::stream_.Put(','); // add comma if it is not the first element in array - Base::stream_.Put('\n'); - } - else - Base::stream_.Put('\n'); - WriteIndent(); - } - else { // in object - if (level->valueCount > 0) { - if (level->valueCount % 2 == 0) { - Base::stream_.Put(','); - Base::stream_.Put('\n'); - } - else { - Base::stream_.Put(':'); - Base::stream_.Put(' '); - } - } - else - Base::stream_.Put('\n'); - - if (level->valueCount % 2 == 0) - WriteIndent(); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else - RAPIDJSON_ASSERT(type == kObjectType || type == kArrayType); - } - - void WriteIndent() { - size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(Base::stream_, indentChar_, count); - } - - Ch indentChar_; - unsigned indentCharCount_; -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/third-party/rapidjson/rapidjson.h b/third-party/rapidjson/rapidjson.h deleted file mode 100644 index 7acb2aa4fd..0000000000 --- a/third-party/rapidjson/rapidjson.h +++ /dev/null @@ -1,525 +0,0 @@ -#ifndef RAPIDJSON_RAPIDJSON_H_ -#define RAPIDJSON_RAPIDJSON_H_ - -// Copyright (c) 2011-2012 Milo Yip (miloyip@gmail.com) -// Version 0.11 - -#include // malloc(), realloc(), free() -#include // memcpy() - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NO_INT64DEFINE - -// Here defines int64_t and uint64_t types in global namespace. -// If user have their own definition, can define RAPIDJSON_NO_INT64DEFINE to disable this. -#ifndef RAPIDJSON_NO_INT64DEFINE -#ifdef _MSC_VER -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -#else -#include -#endif -#endif // RAPIDJSON_NO_INT64TYPEDEF - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ENDIAN -#define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine -#define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine - -//! Endianness of the machine. -/*! GCC provided macro for detecting endianness of the target machine. But other - compilers may not have this. User can define RAPIDJSON_ENDIAN to either - RAPIDJSON_LITTLEENDIAN or RAPIDJSON_BIGENDIAN. -*/ -#ifndef RAPIDJSON_ENDIAN -#ifdef __BYTE_ORDER__ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -#else -#define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -#endif // __BYTE_ORDER__ -#else -#define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN // Assumes little endian otherwise. -#endif -#endif // RAPIDJSON_ENDIAN - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD - -// Enable SSE2 optimization. -//#define RAPIDJSON_SSE2 - -// Enable SSE4.2 optimization. -//#define RAPIDJSON_SSE42 - -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) -#define RAPIDJSON_SIMD -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NO_SIZETYPEDEFINE - -#ifndef RAPIDJSON_NO_SIZETYPEDEFINE -namespace rapidjson { -//! Use 32-bit array/string indices even for 64-bit platform, instead of using size_t. -/*! User may override the SizeType by defining RAPIDJSON_NO_SIZETYPEDEFINE. -*/ -typedef unsigned SizeType; -} // namespace rapidjson -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ASSERT - -//! Assertion. -/*! By default, rapidjson uses C assert() for assertion. - User can override it by defining RAPIDJSON_ASSERT(x) macro. -*/ -#ifndef RAPIDJSON_ASSERT -#include -#define RAPIDJSON_ASSERT(x) assert(x) -#endif // RAPIDJSON_ASSERT - -/////////////////////////////////////////////////////////////////////////////// -// Helpers - -#define RAPIDJSON_MULTILINEMACRO_BEGIN do { -#define RAPIDJSON_MULTILINEMACRO_END \ -} while((void)0, 0) - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// Allocator - -/*! \class rapidjson::Allocator - \brief Concept for allocating, resizing and freeing memory block. - - Note that Malloc() and Realloc() are non-static but Free() is static. - - So if an allocator need to support Free(), it needs to put its pointer in - the header of memory block. - -\code -concept Allocator { - static const bool kNeedFree; //!< Whether this allocator needs to call Free(). - - // Allocate a memory block. - // \param size of the memory block in bytes. - // \returns pointer to the memory block. - void* Malloc(size_t size); - - // Resize a memory block. - // \param originalPtr The pointer to current memory block. Null pointer is permitted. - // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) - // \param newSize the new size in bytes. - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); - - // Free a memory block. - // \param pointer to the memory block. Null pointer is permitted. - static void Free(void *ptr); -}; -\endcode -*/ - -/////////////////////////////////////////////////////////////////////////////// -// CrtAllocator - -//! C-runtime library allocator. -/*! This class is just wrapper for standard C library memory routines. - \implements Allocator -*/ -class CrtAllocator { -public: - static const bool kNeedFree = true; - void* Malloc(size_t size) { return malloc(size); } - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; return realloc(originalPtr, newSize); } - static void Free(void *ptr) { free(ptr); } -}; - -/////////////////////////////////////////////////////////////////////////////// -// MemoryPoolAllocator - -//! Default memory allocator used by the parser and DOM. -/*! This allocator allocate memory blocks from pre-allocated memory chunks. - - It does not free memory blocks. And Realloc() only allocate new memory. - - The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. - - User may also supply a buffer as the first chunk. - - If the user-buffer is full then additional chunks are allocated by BaseAllocator. - - The user-buffer is not deallocated by this allocator. - - \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. - \implements Allocator -*/ -template -class MemoryPoolAllocator { -public: - static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) - - //! Constructor with chunkSize. - /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = new BaseAllocator(); - AddChunk(chunk_capacity_); - } - - //! Constructor with user-supplied buffer. - /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. - - The user buffer will not be deallocated when this allocator is destructed. - - \param buffer User supplied buffer. - \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). - \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(char *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - RAPIDJSON_ASSERT(buffer != 0); - RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); - chunkHead_ = (ChunkHeader*)buffer; - chunkHead_->capacity = size - sizeof(ChunkHeader); - chunkHead_->size = 0; - chunkHead_->next = 0; - } - - //! Destructor. - /*! This deallocates all memory chunks, excluding the user-supplied buffer. - */ - ~MemoryPoolAllocator() { - Clear(); - delete ownBaseAllocator_; - } - - //! Deallocates all memory chunks, excluding the user-supplied buffer. - void Clear() { - while(chunkHead_ != 0 && chunkHead_ != (ChunkHeader *)userBuffer_) { - ChunkHeader* next = chunkHead_->next; - baseAllocator_->Free(chunkHead_); - chunkHead_ = next; - } - } - - //! Computes the total capacity of allocated memory chunks. - /*! \return total capacity in bytes. - */ - size_t Capacity() { - size_t capacity = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - capacity += c->capacity; - return capacity; - } - - //! Computes the memory blocks allocated. - /*! \return total used bytes. - */ - size_t Size() { - size_t size = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - size += c->size; - return size; - } - - //! Allocates a memory block. (concept Allocator) - void* Malloc(size_t size) { - size = (size + 3) & ~3; // Force aligning size to 4 - - if (chunkHead_->size + size > chunkHead_->capacity) - AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); - - char *buffer = (char *)(chunkHead_ + 1) + chunkHead_->size; - RAPIDJSON_ASSERT(((uintptr_t)buffer & 3) == 0); // returned buffer is aligned to 4 - chunkHead_->size += size; - - return buffer; - } - - //! Resizes a memory block (concept Allocator) - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - if (originalPtr == 0) - return Malloc(newSize); - - // Do not shrink if new size is smaller than original - if (originalSize >= newSize) - return originalPtr; - - // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { - size_t increment = newSize - originalSize; - increment = (increment + 3) & ~3; // Force aligning size to 4 - if (chunkHead_->size + increment <= chunkHead_->capacity) { - chunkHead_->size += increment; - RAPIDJSON_ASSERT(((uintptr_t)originalPtr & 3) == 0); // returned buffer is aligned to 4 - return originalPtr; - } - } - - // Realloc process: allocate and copy memory, do not free original buffer. - void* newBuffer = Malloc(newSize); - RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. - return memcpy(newBuffer, originalPtr, originalSize); - } - - //! Frees a memory block (concept Allocator) - static void Free(void *) {} // Do nothing - -private: - //! Creates a new chunk. - /*! \param capacity Capacity of the chunk in bytes. - */ - void AddChunk(size_t capacity) { - ChunkHeader* chunk = (ChunkHeader*)baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity); - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; - } - - static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. - - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. - }; - - ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. - size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - char *userBuffer_; //!< User supplied buffer. - BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. -}; - -/////////////////////////////////////////////////////////////////////////////// -// Encoding - -/*! \class rapidjson::Encoding - \brief Concept for encoding of Unicode characters. - -\code -concept Encoding { - typename Ch; //! Type of character. - - //! \brief Encode a Unicode codepoint to a buffer. - //! \param buffer pointer to destination buffer to store the result. It should have sufficient size of encoding one character. - //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. - //! \returns the pointer to the next character after the encoded data. - static Ch* Encode(Ch *buffer, unsigned codepoint); -}; -\endcode -*/ - -/////////////////////////////////////////////////////////////////////////////// -// UTF8 - -//! UTF-8 encoding. -/*! http://en.wikipedia.org/wiki/UTF-8 - \tparam CharType Type for storing 8-bit UTF-8 data. Default is char. - \implements Encoding -*/ -template -struct UTF8 { - typedef CharType Ch; - - static Ch* Encode(Ch *buffer, unsigned codepoint) { - if (codepoint <= 0x7F) - *buffer++ = codepoint & 0xFF; - else if (codepoint <= 0x7FF) { - *buffer++ = 0xC0 | ((codepoint >> 6) & 0xFF); - *buffer++ = 0x80 | ((codepoint & 0x3F)); - } - else if (codepoint <= 0xFFFF) { - *buffer++ = 0xE0 | ((codepoint >> 12) & 0xFF); - *buffer++ = 0x80 | ((codepoint >> 6) & 0x3F); - *buffer++ = 0x80 | (codepoint & 0x3F); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - *buffer++ = 0xF0 | ((codepoint >> 18) & 0xFF); - *buffer++ = 0x80 | ((codepoint >> 12) & 0x3F); - *buffer++ = 0x80 | ((codepoint >> 6) & 0x3F); - *buffer++ = 0x80 | (codepoint & 0x3F); - } - return buffer; - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// UTF16 - -//! UTF-16 encoding. -/*! http://en.wikipedia.org/wiki/UTF-16 - \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. - \implements Encoding -*/ -template -struct UTF16 { - typedef CharType Ch; - - static Ch* Encode(Ch* buffer, unsigned codepoint) { - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - *buffer++ = static_cast(codepoint); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - *buffer++ = static_cast((v >> 10) + 0xD800); - *buffer++ = (v & 0x3FF) + 0xDC00; - } - return buffer; - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// UTF32 - -//! UTF-32 encoding. -/*! http://en.wikipedia.org/wiki/UTF-32 - \tparam Ch Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. - \implements Encoding -*/ -template -struct UTF32 { - typedef CharType Ch; - - static Ch *Encode(Ch* buffer, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - *buffer++ = codepoint; - return buffer; - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// Stream - -/*! \class rapidjson::Stream - \brief Concept for reading and writing characters. - - For read-only stream, no need to implement PutBegin(), Put() and PutEnd(). - - For write-only stream, only need to implement Put(). - -\code -concept Stream { - typename Ch; //!< Character type of the stream. - - //! Read the current character from stream without moving the read cursor. - Ch Peek() const; - - //! Read the current character from stream and moving the read cursor to next character. - Ch Take(); - - //! Get the current read cursor. - //! \return Number of characters read from start. - size_t Tell(); - - //! Begin writing operation at the current read pointer. - //! \return The begin writer pointer. - Ch* PutBegin(); - - //! Write a character. - void Put(Ch c); - - //! End the writing operation. - //! \param begin The begin write pointer returned by PutBegin(). - //! \return Number of characters written. - size_t PutEnd(Ch* begin); -} -\endcode -*/ - -//! Put N copies of a character to a stream. -template -inline void PutN(Stream& stream, Ch c, size_t n) { - for (size_t i = 0; i < n; i++) - stream.Put(c); -} - -/////////////////////////////////////////////////////////////////////////////// -// StringStream - -//! Read-only string stream. -/*! \implements Stream -*/ -template -struct GenericStringStream { - typedef typename Encoding::Ch Ch; - - GenericStringStream(const Ch *src) : src_(src), head_(src) {} - - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return src_ - head_; } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. -}; - -typedef GenericStringStream > StringStream; - -/////////////////////////////////////////////////////////////////////////////// -// InsituStringStream - -//! A read-write string stream. -/*! This string stream is particularly designed for in-situ parsing. - \implements Stream -*/ -template -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; - - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return src_ - head_; } - - // Write - Ch* PutBegin() { return dst_ = src_; } - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - size_t PutEnd(Ch* begin) { return dst_ - begin; } - - Ch* src_; - Ch* dst_; - Ch* head_; -}; - -typedef GenericInsituStringStream > InsituStringStream; - -/////////////////////////////////////////////////////////////////////////////// -// Type - -//! Type of JSON value -enum Type { - kNullType = 0, //!< null - kFalseType = 1, //!< false - kTrueType = 2, //!< true - kObjectType = 3, //!< object - kArrayType = 4, //!< array - kStringType = 5, //!< string - kNumberType = 6, //!< number -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/third-party/rapidjson/reader.h b/third-party/rapidjson/reader.h deleted file mode 100644 index 78391add3c..0000000000 --- a/third-party/rapidjson/reader.h +++ /dev/null @@ -1,683 +0,0 @@ -#ifndef RAPIDJSON_READER_H_ -#define RAPIDJSON_READER_H_ - -// Copyright (c) 2011 Milo Yip (miloyip@gmail.com) -// Version 0.1 - -#include "rapidjson.h" -#include "internal/pow10.h" -#include "internal/stack.h" -#include - -#ifdef RAPIDJSON_SSE42 -#include -#elif defined(RAPIDJSON_SSE2) -#include -#endif - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4127) // conditional expression is constant -#endif - -#ifndef RAPIDJSON_PARSE_ERROR -#define RAPIDJSON_PARSE_ERROR(msg, offset) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - parseError_ = msg; \ - errorOffset_ = offset; \ - longjmp(jmpbuf_, 1); \ - RAPIDJSON_MULTILINEMACRO_END -#endif - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// ParseFlag - -//! Combination of parseFlags -enum ParseFlag { - kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer. - kParseInsituFlag = 1 //!< In-situ(destructive) parsing. -}; - -/////////////////////////////////////////////////////////////////////////////// -// Handler - -/*! \class rapidjson::Handler - \brief Concept for receiving events from GenericReader upon parsing. -\code -concept Handler { - typename Ch; - - void Null(); - void Bool(bool b); - void Int(int i); - void Uint(unsigned i); - void Int64(int64_t i); - void Uint64(uint64_t i); - void Double(double d); - void String(const Ch* str, SizeType length, bool copy); - void StartObject(); - void EndObject(SizeType memberCount); - void StartArray(); - void EndArray(SizeType elementCount); -}; -\endcode -*/ -/////////////////////////////////////////////////////////////////////////////// -// BaseReaderHandler - -//! Default implementation of Handler. -/*! This can be used as base class of any reader handler. - \implements Handler -*/ -template > -struct BaseReaderHandler { - typedef typename Encoding::Ch Ch; - - void Default() {} - void Null() { Default(); } - void Bool(bool) { Default(); } - void Int(int) { Default(); } - void Uint(unsigned) { Default(); } - void Int64(int64_t) { Default(); } - void Uint64(uint64_t) { Default(); } - void Double(double) { Default(); } - void String(const Ch*, SizeType, bool) { Default(); } - void StartObject() { Default(); } - void EndObject(SizeType) { Default(); } - void StartArray() { Default(); } - void EndArray(SizeType) { Default(); } -}; - -/////////////////////////////////////////////////////////////////////////////// -// SkipWhitespace - -//! Skip the JSON white spaces in a stream. -/*! \param stream A input stream for skipping white spaces. - \note This function has SSE2/SSE4.2 specialization. -*/ -template -void SkipWhitespace(Stream& stream) { - Stream s = stream; // Use a local copy for optimization - while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') - s.Take(); - stream = s; -} - -#ifdef RAPIDJSON_SSE42 -//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - static const char whitespace[16] = " \n\r\t"; - __m128i w = _mm_loadu_si128((const __m128i *)&whitespace[0]); - - for (;;) { - __m128i s = _mm_loadu_si128((const __m128i *)p); - unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r == 0) // all 16 characters are whitespace - p += 16; - else { // some of characters may be non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - if (_BitScanForward(&offset, r)) - return p + offset; -#else - if (r != 0) - return p + __builtin_ffs(r) - 1; -#endif - } - } -} - -#elif defined(RAPIDJSON_SSE2) - -//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - static const char whitespaces[4][17] = { - " ", - "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", - "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r", - "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"}; - - __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]); - __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]); - __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]); - __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]); - - for (;;) { - __m128i s = _mm_loadu_si128((const __m128i *)p); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = ~_mm_movemask_epi8(x); - if (r == 0) // all 16 characters are whitespace - p += 16; - else { // some of characters may be non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - if (_BitScanForward(&offset, r)) - return p + offset; -#else - if (r != 0) - return p + __builtin_ffs(r) - 1; -#endif - } - } -} - -#endif // RAPIDJSON_SSE2 - -#ifdef RAPIDJSON_SIMD -//! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& stream) { - stream.src_ = const_cast(SkipWhitespace_SIMD(stream.src_)); -} - -//! Template function specialization for StringStream -template<> inline void SkipWhitespace(StringStream& stream) { - stream.src_ = SkipWhitespace_SIMD(stream.src_); -} -#endif // RAPIDJSON_SIMD - -/////////////////////////////////////////////////////////////////////////////// -// GenericReader - -//! SAX-style JSON parser. Use Reader for UTF8 encoding and default allocator. -/*! GenericReader parses JSON text from a stream, and send events synchronously to an - object implementing Handler concept. - - It needs to allocate a stack for storing a single decoded string during - non-destructive parsing. - - For in-situ parsing, the decoded string is directly written to the source - text string, no temporary buffer is required. - - A GenericReader object can be reused for parsing multiple JSON text. - - \tparam Encoding Encoding of both the stream and the parse output. - \tparam Allocator Allocator type for stack. -*/ -template > -class GenericReader { -public: - typedef typename Encoding::Ch Ch; - - //! Constructor. - /*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) - \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) - */ - GenericReader(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseError_(0), errorOffset_(0) {} - - //! Parse JSON text. - /*! \tparam parseFlags Combination of ParseFlag. - \tparam Stream Type of input stream. - \tparam Handler Type of handler which must implement Handler concept. - \param stream Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template - bool Parse(Stream& stream, Handler& handler) { - parseError_ = 0; - errorOffset_ = 0; - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4611) // interaction between '_setjmp' and C++ object destruction is non-portable -#endif - if (setjmp(jmpbuf_)) { -#ifdef _MSC_VER -#pragma warning(pop) -#endif - stack_.Clear(); - return false; - } - - SkipWhitespace(stream); - - if (stream.Peek() == '\0') - RAPIDJSON_PARSE_ERROR("Text only contains white space(s)", stream.Tell()); - else { - switch (stream.Peek()) { - case '{': ParseObject(stream, handler); break; - case '[': ParseArray(stream, handler); break; - default: RAPIDJSON_PARSE_ERROR("Expect either an object or array at root", stream.Tell()); - } - SkipWhitespace(stream); - - if (stream.Peek() != '\0') - RAPIDJSON_PARSE_ERROR("Nothing should follow the root object or array.", stream.Tell()); - } - - return true; - } - - bool HasParseError() const { return parseError_ != 0; } - const char* GetParseError() const { return parseError_; } - size_t GetErrorOffset() const { return errorOffset_; } - -private: - // Parse object: { string : value, ... } - template - void ParseObject(Stream& stream, Handler& handler) { - RAPIDJSON_ASSERT(stream.Peek() == '{'); - stream.Take(); // Skip '{' - handler.StartObject(); - SkipWhitespace(stream); - - if (stream.Peek() == '}') { - stream.Take(); - handler.EndObject(0); // empty object - return; - } - - for (SizeType memberCount = 0;;) { - if (stream.Peek() != '"') { - RAPIDJSON_PARSE_ERROR("Name of an object member must be a string", stream.Tell()); - break; - } - - ParseString(stream, handler); - SkipWhitespace(stream); - - if (stream.Take() != ':') { - RAPIDJSON_PARSE_ERROR("There must be a colon after the name of object member", stream.Tell()); - break; - } - SkipWhitespace(stream); - - ParseValue(stream, handler); - SkipWhitespace(stream); - - ++memberCount; - - switch(stream.Take()) { - case ',': SkipWhitespace(stream); break; - case '}': handler.EndObject(memberCount); return; - default: RAPIDJSON_PARSE_ERROR("Must be a comma or '}' after an object member", stream.Tell()); - } - } - } - - // Parse array: [ value, ... ] - template - void ParseArray(Stream& stream, Handler& handler) { - RAPIDJSON_ASSERT(stream.Peek() == '['); - stream.Take(); // Skip '[' - handler.StartArray(); - SkipWhitespace(stream); - - if (stream.Peek() == ']') { - stream.Take(); - handler.EndArray(0); // empty array - return; - } - - for (SizeType elementCount = 0;;) { - ParseValue(stream, handler); - ++elementCount; - SkipWhitespace(stream); - - switch (stream.Take()) { - case ',': SkipWhitespace(stream); break; - case ']': handler.EndArray(elementCount); return; - default: RAPIDJSON_PARSE_ERROR("Must be a comma or ']' after an array element.", stream.Tell()); - } - } - } - - template - void ParseNull(Stream& stream, Handler& handler) { - RAPIDJSON_ASSERT(stream.Peek() == 'n'); - stream.Take(); - - if (stream.Take() == 'u' && stream.Take() == 'l' && stream.Take() == 'l') - handler.Null(); - else - RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell() - 1); - } - - template - void ParseTrue(Stream& stream, Handler& handler) { - RAPIDJSON_ASSERT(stream.Peek() == 't'); - stream.Take(); - - if (stream.Take() == 'r' && stream.Take() == 'u' && stream.Take() == 'e') - handler.Bool(true); - else - RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell()); - } - - template - void ParseFalse(Stream& stream, Handler& handler) { - RAPIDJSON_ASSERT(stream.Peek() == 'f'); - stream.Take(); - - if (stream.Take() == 'a' && stream.Take() == 'l' && stream.Take() == 's' && stream.Take() == 'e') - handler.Bool(false); - else - RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell() - 1); - } - - // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). - template - unsigned ParseHex4(Stream& stream) { - Stream s = stream; // Use a local copy for optimization - unsigned codepoint = 0; - for (int i = 0; i < 4; i++) { - Ch c = s.Take(); - codepoint <<= 4; - codepoint += c; - if (c >= '0' && c <= '9') - codepoint -= '0'; - else if (c >= 'A' && c <= 'F') - codepoint -= 'A' - 10; - else if (c >= 'a' && c <= 'f') - codepoint -= 'a' - 10; - else - RAPIDJSON_PARSE_ERROR("Incorrect hex digit after \\u escape", s.Tell() - 1); - } - stream = s; // Restore stream - return codepoint; - } - - // Parse string, handling the prefix and suffix double quotes and escaping. - template - void ParseString(Stream& stream, Handler& handler) { -#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - static const Ch escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 - }; -#undef Z16 - - Stream s = stream; // Use a local copy for optimization - RAPIDJSON_ASSERT(s.Peek() == '\"'); - s.Take(); // Skip '\"' - Ch *head; - SizeType len; - if (parseFlags & kParseInsituFlag) - head = s.PutBegin(); - else - len = 0; - -#define RAPIDJSON_PUT(x) \ - do { \ - if (parseFlags & kParseInsituFlag) \ - s.Put(x); \ - else { \ - *stack_.template Push() = x; \ - ++len; \ - } \ - } while(false) - - for (;;) { - Ch c = s.Take(); - if (c == '\\') { // Escape - Ch e = s.Take(); - if ((sizeof(Ch) == 1 || (int)e < 256) && escape[(unsigned char)e]) - RAPIDJSON_PUT(escape[(unsigned char)e]); - else if (e == 'u') { // Unicode - unsigned codepoint = ParseHex4(s); - if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { // Handle UTF-16 surrogate pair - if (s.Take() != '\\' || s.Take() != 'u') { - RAPIDJSON_PARSE_ERROR("Missing the second \\u in surrogate pair", s.Tell() - 2); - return; - } - unsigned codepoint2 = ParseHex4(s); - if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) { - RAPIDJSON_PARSE_ERROR("The second \\u in surrogate pair is invalid", s.Tell() - 2); - return; - } - codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; - } - - Ch buffer[4]; - SizeType count = SizeType(Encoding::Encode(buffer, codepoint) - &buffer[0]); - - if (parseFlags & kParseInsituFlag) - for (SizeType i = 0; i < count; i++) - s.Put(buffer[i]); - else { - memcpy(stack_.template Push(count), buffer, count * sizeof(Ch)); - len += count; - } - } - else { - RAPIDJSON_PARSE_ERROR("Unknown escape character", stream.Tell() - 1); - return; - } - } - else if (c == '"') { // Closing double quote - if (parseFlags & kParseInsituFlag) { - size_t length = s.PutEnd(head); - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - RAPIDJSON_PUT('\0'); // null-terminate the string - handler.String(head, SizeType(length), false); - } - else { - RAPIDJSON_PUT('\0'); - handler.String(stack_.template Pop(len), len - 1, true); - } - stream = s; // restore stream - return; - } - else if (c == '\0') { - RAPIDJSON_PARSE_ERROR("lacks ending quotation before the end of string", stream.Tell() - 1); - return; - } - else if ((unsigned)c < 0x20) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - RAPIDJSON_PARSE_ERROR("Incorrect unescaped character in string", stream.Tell() - 1); - return; - } - else - RAPIDJSON_PUT(c); // Normal character, just copy - } -#undef RAPIDJSON_PUT - } - - template - void ParseNumber(Stream& stream, Handler& handler) { - Stream s = stream; // Local copy for optimization - // Parse minus - bool minus = false; - if (s.Peek() == '-') { - minus = true; - s.Take(); - } - - // Parse int: zero / ( digit1-9 *DIGIT ) - unsigned i; - bool try64bit = false; - if (s.Peek() == '0') { - i = 0; - s.Take(); - } - else if (s.Peek() >= '1' && s.Peek() <= '9') { - i = s.Take() - '0'; - - if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 214748364) { // 2^31 = 2147483648 - if (i != 214748364 || s.Peek() > '8') { - try64bit = true; - break; - } - } - i = i * 10 + (s.Take() - '0'); - } - else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 429496729) { // 2^32 - 1 = 4294967295 - if (i != 429496729 || s.Peek() > '5') { - try64bit = true; - break; - } - } - i = i * 10 + (s.Take() - '0'); - } - } - else { - RAPIDJSON_PARSE_ERROR("Expect a value here.", stream.Tell()); - return; - } - - // Parse 64bit int - uint64_t i64 = 0; - bool useDouble = false; - if (try64bit) { - i64 = i; - if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= 922337203685477580uLL) // 2^63 = 9223372036854775808 - if (i64 != 922337203685477580uLL || s.Peek() > '8') { - useDouble = true; - break; - } - i64 = i64 * 10 + (s.Take() - '0'); - } - else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= 1844674407370955161uLL) // 2^64 - 1 = 18446744073709551615 - if (i64 != 1844674407370955161uLL || s.Peek() > '5') { - useDouble = true; - break; - } - i64 = i64 * 10 + (s.Take() - '0'); - } - } - - // Force double for big integer - double d = 0.0; - if (useDouble) { - d = (double)i64; - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (d >= 1E307) { - RAPIDJSON_PARSE_ERROR("Number too big to store in double", stream.Tell()); - return; - } - d = d * 10 + (s.Take() - '0'); - } - } - - // Parse frac = decimal-point 1*DIGIT - int expFrac = 0; - if (s.Peek() == '.') { - if (!useDouble) { - d = try64bit ? (double)i64 : (double)i; - useDouble = true; - } - s.Take(); - - if (s.Peek() >= '0' && s.Peek() <= '9') { - d = d * 10 + (s.Take() - '0'); - --expFrac; - } - else { - RAPIDJSON_PARSE_ERROR("At least one digit in fraction part", stream.Tell()); - return; - } - - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (expFrac > -16) { - d = d * 10 + (s.Peek() - '0'); - --expFrac; - } - s.Take(); - } - } - - // Parse exp = e [ minus / plus ] 1*DIGIT - int exp = 0; - if (s.Peek() == 'e' || s.Peek() == 'E') { - if (!useDouble) { - d = try64bit ? (double)i64 : (double)i; - useDouble = true; - } - s.Take(); - - bool expMinus = false; - if (s.Peek() == '+') - s.Take(); - else if (s.Peek() == '-') { - s.Take(); - expMinus = true; - } - - if (s.Peek() >= '0' && s.Peek() <= '9') { - exp = s.Take() - '0'; - while (s.Peek() >= '0' && s.Peek() <= '9') { - exp = exp * 10 + (s.Take() - '0'); - if (exp > 308) { - RAPIDJSON_PARSE_ERROR("Number too big to store in double", stream.Tell()); - return; - } - } - } - else { - RAPIDJSON_PARSE_ERROR("At least one digit in exponent", s.Tell()); - return; - } - - if (expMinus) - exp = -exp; - } - - // Finish parsing, call event according to the type of number. - if (useDouble) { - d *= internal::Pow10(exp + expFrac); - handler.Double(minus ? -d : d); - } - else { - if (try64bit) { - if (minus) - handler.Int64(-(int64_t)i64); - else - handler.Uint64(i64); - } - else { - if (minus) - handler.Int(-(int)i); - else - handler.Uint(i); - } - } - - stream = s; // restore stream - } - - // Parse any JSON value - template - void ParseValue(Stream& stream, Handler& handler) { - switch (stream.Peek()) { - case 'n': ParseNull (stream, handler); break; - case 't': ParseTrue (stream, handler); break; - case 'f': ParseFalse (stream, handler); break; - case '"': ParseString(stream, handler); break; - case '{': ParseObject(stream, handler); break; - case '[': ParseArray (stream, handler); break; - default : ParseNumber(stream, handler); - } - } - - static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. - internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. - jmp_buf jmpbuf_; //!< setjmp buffer for fast exit from nested parsing function calls. - const char* parseError_; - size_t errorOffset_; -}; // class GenericReader - -//! Reader with UTF8 encoding and default allocator. -typedef GenericReader > Reader; - -} // namespace rapidjson - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#endif // RAPIDJSON_READER_H_ diff --git a/third-party/rapidjson/stringbuffer.h b/third-party/rapidjson/stringbuffer.h deleted file mode 100644 index 269ae10761..0000000000 --- a/third-party/rapidjson/stringbuffer.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef RAPIDJSON_STRINGBUFFER_H_ -#define RAPIDJSON_STRINGBUFFER_H_ - -#include "rapidjson.h" -#include "internal/stack.h" - -namespace rapidjson { - -//! Represents an in-memory output stream. -/*! - \tparam Encoding Encoding of the stream. - \tparam Allocator type for allocating memory buffer. - \implements Stream -*/ -template -struct GenericStringBuffer { - typedef typename Encoding::Ch Ch; - - GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - - void Put(Ch c) { *stack_.template Push() = c; } - - void Clear() { stack_.Clear(); } - - const char* GetString() const { - // Push and pop a null terminator. This is safe. - *stack_.template Push() = '\0'; - stack_.template Pop(1); - - return stack_.template Bottom(); - } - - size_t Size() const { return stack_.GetSize(); } - - static const size_t kDefaultCapacity = 256; - mutable internal::Stack stack_; -}; - -typedef GenericStringBuffer > StringBuffer; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { - memset(stream.stack_.Push(n), c, n * sizeof(c)); -} - -} // namespace rapidjson - -#endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/third-party/rapidjson/writer.h b/third-party/rapidjson/writer.h deleted file mode 100644 index d96f2081a9..0000000000 --- a/third-party/rapidjson/writer.h +++ /dev/null @@ -1,241 +0,0 @@ -#ifndef RAPIDJSON_WRITER_H_ -#define RAPIDJSON_WRITER_H_ - -#include "rapidjson.h" -#include "internal/stack.h" -#include "internal/strfunc.h" -#include // snprintf() or _sprintf_s() -#include // placement new - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4127) // conditional expression is constant -#endif - -namespace rapidjson { - -//! JSON writer -/*! Writer implements the concept Handler. - It generates JSON text by events to an output stream. - - User may programmatically calls the functions of a writer to generate JSON text. - - On the other side, a writer can also be passed to objects that generates events, - - for example Reader::Parse() and Document::Accept(). - - \tparam Stream Type of ouptut stream. - \tparam Encoding Encoding of both source strings and output. - \implements Handler -*/ -template, typename Allocator = MemoryPoolAllocator<> > -class Writer { -public: - typedef typename Encoding::Ch Ch; - - Writer(Stream& stream, Allocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - stream_(stream), level_stack_(allocator, levelDepth * sizeof(Level)) {} - - //@name Implementation of Handler - //@{ - Writer& Null() { Prefix(kNullType); WriteNull(); return *this; } - Writer& Bool(bool b) { Prefix(b ? kTrueType : kFalseType); WriteBool(b); return *this; } - Writer& Int(int i) { Prefix(kNumberType); WriteInt(i); return *this; } - Writer& Uint(unsigned u) { Prefix(kNumberType); WriteUint(u); return *this; } - Writer& Int64(int64_t i64) { Prefix(kNumberType); WriteInt64(i64); return *this; } - Writer& Uint64(uint64_t u64) { Prefix(kNumberType); WriteUint64(u64); return *this; } - Writer& Double(double d) { Prefix(kNumberType); WriteDouble(d); return *this; } - - Writer& String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - Prefix(kStringType); - WriteString(str, length); - return *this; - } - - Writer& StartObject() { - Prefix(kObjectType); - new (level_stack_.template Push()) Level(false); - WriteStartObject(); - return *this; - } - - Writer& EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); - level_stack_.template Pop(1); - WriteEndObject(); - return *this; - } - - Writer& StartArray() { - Prefix(kArrayType); - new (level_stack_.template Push()) Level(true); - WriteStartArray(); - return *this; - } - - Writer& EndArray(SizeType elementCount = 0) { - (void)elementCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); - level_stack_.template Pop(1); - WriteEndArray(); - return *this; - } - //@} - - //! Simpler but slower overload. - Writer& String(const Ch* str) { return String(str, internal::StrLen(str)); } - -protected: - //! Information for each nested level - struct Level { - Level(bool inArray_) : inArray(inArray_), valueCount(0) {} - bool inArray; //!< true if in array, otherwise in object - size_t valueCount; //!< number of values in this level - }; - - static const size_t kDefaultLevelDepth = 32; - - void WriteNull() { - stream_.Put('n'); stream_.Put('u'); stream_.Put('l'); stream_.Put('l'); - } - - void WriteBool(bool b) { - if (b) { - stream_.Put('t'); stream_.Put('r'); stream_.Put('u'); stream_.Put('e'); - } - else { - stream_.Put('f'); stream_.Put('a'); stream_.Put('l'); stream_.Put('s'); stream_.Put('e'); - } - } - - void WriteInt(int i) { - if (i < 0) { - stream_.Put('-'); - i = -i; - } - WriteUint((unsigned)i); - } - - void WriteUint(unsigned u) { - char buffer[10]; - char *p = buffer; - do { - *p++ = (u % 10) + '0'; - u /= 10; - } while (u > 0); - - do { - --p; - stream_.Put(*p); - } while (p != buffer); - } - - void WriteInt64(int64_t i64) { - if (i64 < 0) { - stream_.Put('-'); - i64 = -i64; - } - WriteUint64((uint64_t)i64); - } - - void WriteUint64(uint64_t u64) { - char buffer[20]; - char *p = buffer; - do { - *p++ = char(u64 % 10) + '0'; - u64 /= 10; - } while (u64 > 0); - - do { - --p; - stream_.Put(*p); - } while (p != buffer); - } - - //! \todo Optimization with custom double-to-string converter. - void WriteDouble(double d) { - char buffer[100]; -#if _MSC_VER - int ret = sprintf_s(buffer, sizeof(buffer), "%g", d); -#else - int ret = snprintf(buffer, sizeof(buffer), "%g", d); -#endif - RAPIDJSON_ASSERT(ret >= 1); - for (int i = 0; i < ret; i++) - stream_.Put(buffer[i]); - } - - void WriteString(const Ch* str, SizeType length) { - static const char hexDigits[] = "0123456789ABCDEF"; - static const char escape[256] = { -#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - //0 1 2 3 4 5 6 7 8 9 A B C D E F - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 - 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 - Z16, Z16, // 30~4F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF -#undef Z16 - }; - - stream_.Put('\"'); - for (const Ch* p = str; p != str + length; ++p) { - if ((sizeof(Ch) == 1 || *p < 256) && escape[(unsigned char)*p]) { - stream_.Put('\\'); - stream_.Put(escape[(unsigned char)*p]); - if (escape[(unsigned char)*p] == 'u') { - stream_.Put('0'); - stream_.Put('0'); - stream_.Put(hexDigits[(*p) >> 4]); - stream_.Put(hexDigits[(*p) & 0xF]); - } - } - else - stream_.Put(*p); - } - stream_.Put('\"'); - } - - void WriteStartObject() { stream_.Put('{'); } - void WriteEndObject() { stream_.Put('}'); } - void WriteStartArray() { stream_.Put('['); } - void WriteEndArray() { stream_.Put(']'); } - - void Prefix(Type type) { - (void)type; - if (level_stack_.GetSize() != 0) { // this value is not at root - Level* level = level_stack_.template Top(); - if (level->valueCount > 0) { - if (level->inArray) - stream_.Put(','); // add comma if it is not the first element in array - else // in object - stream_.Put((level->valueCount % 2 == 0) ? ',' : ':'); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else - RAPIDJSON_ASSERT(type == kObjectType || type == kArrayType); - } - - Stream& stream_; - internal::Stack level_stack_; - -private: - // Prohibit assignment for VC C4512 warning - Writer& operator=(const Writer& w); -}; - -} // namespace rapidjson - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/utilities/document/document_db.cc b/utilities/document/document_db.cc index d762df8136..9e398153a3 100644 --- a/utilities/document/document_db.cc +++ b/utilities/document/document_db.cc @@ -64,43 +64,45 @@ class Filter { static Filter* ParseFilter(const JSONDocument& filter); struct Interval { - const JSONDocument* upper_bound; - const JSONDocument* lower_bound; + JSONDocument upper_bound; + JSONDocument lower_bound; bool upper_inclusive; bool lower_inclusive; Interval() - : upper_bound(nullptr), - lower_bound(nullptr), + : upper_bound(), + lower_bound(), upper_inclusive(false), lower_inclusive(false) {} - Interval(const JSONDocument* ub, const JSONDocument* lb, bool ui, bool li) + Interval(const JSONDocument& ub, const JSONDocument& lb, bool ui, bool li) : upper_bound(ub), lower_bound(lb), upper_inclusive(ui), - lower_inclusive(li) {} - void UpdateUpperBound(const JSONDocument* ub, bool inclusive); - void UpdateLowerBound(const JSONDocument* lb, bool inclusive); + lower_inclusive(li) { + } + + void UpdateUpperBound(const JSONDocument& ub, bool inclusive); + void UpdateLowerBound(const JSONDocument& lb, bool inclusive); }; bool SatisfiesFilter(const JSONDocument& document) const; const Interval* GetInterval(const std::string& field) const; private: - explicit Filter(const JSONDocument& filter) : filter_(filter) {} + explicit Filter(const JSONDocument& filter) : filter_(filter.Copy()) { + assert(filter_.IsOwner()); + } // copied from the parameter const JSONDocument filter_; - // upper_bound and lower_bound point to JSONDocuments in filter_, so no need - // to free them // constant after construction std::unordered_map intervals_; }; -void Filter::Interval::UpdateUpperBound(const JSONDocument* ub, +void Filter::Interval::UpdateUpperBound(const JSONDocument& ub, bool inclusive) { - bool update = (upper_bound == nullptr); + bool update = upper_bound.IsNull(); if (!update) { - int cmp = DocumentCompare(*upper_bound, *ub); + int cmp = DocumentCompare(upper_bound, ub); update = (cmp > 0) || (cmp == 0 && !inclusive); } if (update) { @@ -109,11 +111,11 @@ void Filter::Interval::UpdateUpperBound(const JSONDocument* ub, } } -void Filter::Interval::UpdateLowerBound(const JSONDocument* lb, +void Filter::Interval::UpdateLowerBound(const JSONDocument& lb, bool inclusive) { - bool update = (lower_bound == nullptr); + bool update = lower_bound.IsNull(); if (!update) { - int cmp = DocumentCompare(*lower_bound, *lb); + int cmp = DocumentCompare(lower_bound, lb); update = (cmp < 0) || (cmp == 0 && !inclusive); } if (update) { @@ -135,14 +137,14 @@ Filter* Filter::ParseFilter(const JSONDocument& filter) { continue; } assert(f->intervals_.find(items.first) == f->intervals_.end()); - if (items.second->IsObject()) { - if (items.second->Count() == 0) { + if (items.second.IsObject()) { + if (items.second.Count() == 0) { // uhm...? return nullptr; } Interval interval; - for (const auto& condition : items.second->Items()) { - if (condition.second->IsObject() || condition.second->IsArray()) { + for (const auto& condition : items.second.Items()) { + if (condition.second.IsObject() || condition.second.IsArray()) { // comparison operators not defined on objects. invalid array return nullptr; } @@ -164,7 +166,8 @@ Filter* Filter::ParseFilter(const JSONDocument& filter) { } else { // equality f->intervals_.insert( - {items.first, Interval(items.second, items.second, true, true)}); + {items.first, Interval(items.second, + items.second, true, true)}); } } @@ -182,30 +185,30 @@ const Filter::Interval* Filter::GetInterval(const std::string& field) const { bool Filter::SatisfiesFilter(const JSONDocument& document) const { for (const auto& interval : intervals_) { - auto value = document.Get(interval.first); - if (value == nullptr) { + if (!document.Contains(interval.first)) { // doesn't have the value, doesn't satisfy the filter // (we don't support null queries yet) return false; } - if (interval.second.upper_bound != nullptr) { - if (value->type() != interval.second.upper_bound->type()) { + auto value = document[interval.first]; + if (!interval.second.upper_bound.IsNull()) { + if (value.type() != interval.second.upper_bound.type()) { // no cross-type queries yet // TODO(icanadi) do this at least for numbers! return false; } - int cmp = DocumentCompare(*interval.second.upper_bound, *value); + int cmp = DocumentCompare(interval.second.upper_bound, value); if (cmp < 0 || (cmp == 0 && interval.second.upper_inclusive == false)) { // bigger (or equal) than upper bound return false; } } - if (interval.second.lower_bound != nullptr) { - if (value->type() != interval.second.lower_bound->type()) { + if (!interval.second.lower_bound.IsNull()) { + if (value.type() != interval.second.lower_bound.type()) { // no cross-type queries yet return false; } - int cmp = DocumentCompare(*interval.second.lower_bound, *value); + int cmp = DocumentCompare(interval.second.lower_bound, value); if (cmp > 0 || (cmp == 0 && interval.second.lower_inclusive == false)) { // smaller (or equal) than the lower bound return false; @@ -386,13 +389,12 @@ class SimpleSortedIndex : public Index { virtual void GetIndexKey(const JSONDocument& document, std::string* key) const override { - auto value = document.Get(field_); - if (value == nullptr) { + if (!document.Contains(field_)) { if (!EncodeJSONPrimitive(JSONDocument(JSONDocument::kNull), key)) { assert(false); } } else { - if (!EncodeJSONPrimitive(*value, key)) { + if (!EncodeJSONPrimitive(document[field_], key)) { assert(false); } } @@ -412,11 +414,11 @@ class SimpleSortedIndex : public Index { Direction direction; const JSONDocument* limit; - if (interval->lower_bound != nullptr) { - limit = interval->lower_bound; + if (!interval->lower_bound.IsNull()) { + limit = &(interval->lower_bound); direction = kForwards; } else { - limit = interval->upper_bound; + limit = &(interval->upper_bound); direction = kBackwards; } @@ -434,14 +436,13 @@ class SimpleSortedIndex : public Index { Index::Direction direction) const override { auto interval = filter.GetInterval(field_); assert(interval != nullptr); // because index is useful - if (direction == kForwards) { - if (interval->upper_bound == nullptr) { + if (interval->upper_bound.IsNull()) { // continue looking, no upper bound return true; } std::string encoded_upper_bound; - if (!EncodeJSONPrimitive(*interval->upper_bound, &encoded_upper_bound)) { + if (!EncodeJSONPrimitive(interval->upper_bound, &encoded_upper_bound)) { // uhm...? // TODO(icanadi) store encoded upper and lower bounds in Filter*? assert(false); @@ -457,12 +458,12 @@ class SimpleSortedIndex : public Index { : true; } else { assert(direction == kBackwards); - if (interval->lower_bound == nullptr) { + if (interval->lower_bound.IsNull()) { // continue looking, no lower bound return true; } std::string encoded_lower_bound; - if (!EncodeJSONPrimitive(*interval->lower_bound, &encoded_lower_bound)) { + if (!EncodeJSONPrimitive(interval->lower_bound, &encoded_lower_bound)) { // uhm...? // TODO(icanadi) store encoded upper and lower bounds in Filter*? assert(false); @@ -495,7 +496,7 @@ Index* Index::CreateIndexFromDescription(const JSONDocument& description, return nullptr; } const auto& field = *description.Items().begin(); - if (field.second->IsInt64() == false || field.second->GetInt64() != 1) { + if (field.second.IsInt64() == false || field.second.GetInt64() != 1) { // not supported yet return nullptr; } @@ -585,6 +586,7 @@ class CursorWithFilterIndexed : public Cursor { } current_json_document_.reset( JSONDocument::Deserialize(primary_index_iter_->value())); + assert(current_json_document_->IsOwner()); if (current_json_document_.get() == nullptr) { status_ = Status::Corruption("JSON deserialization failed"); valid_ = false; @@ -802,16 +804,19 @@ class DocumentDBImpl : public DocumentDB { if (!document.IsObject()) { return Status::InvalidArgument("Document not an object"); } - auto primary_key = document.Get(kPrimaryKey); - if (primary_key == nullptr || primary_key->IsNull() || - (!primary_key->IsString() && !primary_key->IsInt64())) { + if (!document.Contains(kPrimaryKey)) { + return Status::InvalidArgument("No primary key"); + } + auto primary_key = document[kPrimaryKey]; + if (primary_key.IsNull() || + (!primary_key.IsString() && !primary_key.IsInt64())) { return Status::InvalidArgument( - "No primary key or primary key format error"); + "Primary key format error"); } std::string encoded_document; document.Serialize(&encoded_document); std::string primary_key_encoded; - if (!EncodeJSONPrimitive(*primary_key, &primary_key_encoded)) { + if (!EncodeJSONPrimitive(primary_key, &primary_key_encoded)) { // previous call should be guaranteed to pass because of all primary_key // conditions checked before assert(false); @@ -854,16 +859,19 @@ class DocumentDBImpl : public DocumentDB { if (!document.IsObject()) { return Status::Corruption("Document corruption"); } - auto primary_key = document.Get(kPrimaryKey); - if (primary_key == nullptr || primary_key->IsNull() || - (!primary_key->IsString() && !primary_key->IsInt64())) { + if (!document.Contains(kPrimaryKey)) { + return Status::Corruption("Document corruption"); + } + auto primary_key = document[kPrimaryKey]; + if (primary_key.IsNull() || + (!primary_key.IsString() && !primary_key.IsInt64())) { return Status::Corruption("Document corruption"); } // TODO(icanadi) Instead of doing this, just get primary key encoding from // cursor, as it already has this information std::string primary_key_encoded; - if (!EncodeJSONPrimitive(*primary_key, &primary_key_encoded)) { + if (!EncodeJSONPrimitive(primary_key, &primary_key_encoded)) { // previous call should be guaranteed to pass because of all primary_key // conditions checked before assert(false); @@ -894,6 +902,9 @@ class DocumentDBImpl : public DocumentDB { std::unique_ptr cursor( ConstructFilterCursor(read_options, nullptr, filter)); + if (!updates.IsObject()) { + return Status::Corruption("Bad update document format"); + } WriteBatch batch; for (; cursor->status().ok() && cursor->Valid(); cursor->Next()) { const auto& old_document = cursor->document(); @@ -904,12 +915,35 @@ class DocumentDBImpl : public DocumentDB { // TODO(icanadi) Make this nicer, something like class Filter for (const auto& update : updates.Items()) { if (update.first == "$set") { - for (const auto& itr : update.second->Items()) { + JSONDocumentBuilder builder; + bool res = builder.WriteStartObject(); + assert(res); + for (const auto& itr : update.second.Items()) { if (itr.first == kPrimaryKey) { return Status::NotSupported("Please don't change primary key"); } - new_document.Set(itr.first, *itr.second); + res = builder.WriteKeyValue(itr.first, itr.second); + assert(res); } + res = builder.WriteEndObject(); + assert(res); + JSONDocument update_document = builder.GetJSONDocument(); + builder.Reset(); + res = builder.WriteStartObject(); + assert(res); + for (const auto& itr : new_document.Items()) { + if (update_document.Contains(itr.first)) { + res = builder.WriteKeyValue(itr.first, + update_document[itr.first]); + } else { + res = builder.WriteKeyValue(itr.first, new_document[itr.first]); + } + assert(res); + } + res = builder.WriteEndObject(); + assert(res); + new_document = builder.GetJSONDocument(); + assert(new_document.IsOwner()); } else { // TODO(icanadi) more commands return Status::InvalidArgument("Can't understand update command"); @@ -917,9 +951,12 @@ class DocumentDBImpl : public DocumentDB { } // TODO(icanadi) reuse some of this code - auto primary_key = new_document.Get(kPrimaryKey); - if (primary_key == nullptr || primary_key->IsNull() || - (!primary_key->IsString() && !primary_key->IsInt64())) { + if (!new_document.Contains(kPrimaryKey)) { + return Status::Corruption("Corrupted document -- primary key missing"); + } + auto primary_key = new_document[kPrimaryKey]; + if (primary_key.IsNull() || + (!primary_key.IsString() && !primary_key.IsInt64())) { // This will happen when document on storage doesn't have primary key, // since we don't support any update operations on primary key. That's // why this is corruption error @@ -928,7 +965,7 @@ class DocumentDBImpl : public DocumentDB { std::string encoded_document; new_document.Serialize(&encoded_document); std::string primary_key_encoded; - if (!EncodeJSONPrimitive(*primary_key, &primary_key_encoded)) { + if (!EncodeJSONPrimitive(primary_key, &primary_key_encoded)) { // previous call should be guaranteed to pass because of all primary_key // conditions checked before assert(false); @@ -983,7 +1020,7 @@ class DocumentDBImpl : public DocumentDB { const auto& command = *command_doc.Items().begin(); if (command.first == "$filter") { - cursor = ConstructFilterCursor(read_options, cursor, *command.second); + cursor = ConstructFilterCursor(read_options, cursor, command.second); } else { // only filter is supported for now delete cursor; @@ -1032,12 +1069,12 @@ class DocumentDBImpl : public DocumentDB { IndexColumnFamily tmp_storage(nullptr, nullptr); if (cursor == nullptr) { - auto index_name = query.Get("$index"); IndexColumnFamily* index_column_family = nullptr; - if (index_name != nullptr && index_name->IsString()) { + if (query.Contains("$index") && query["$index"].IsString()) { { + auto index_name = query["$index"]; MutexLock l(&name_to_index_mutex_); - auto index_iter = name_to_index_.find(index_name->GetString()); + auto index_iter = name_to_index_.find(index_name.GetString()); if (index_iter != name_to_index_.end()) { tmp_storage = index_iter->second; index_column_family = &tmp_storage; diff --git a/utilities/document/document_db_test.cc b/utilities/document/document_db_test.cc index bacef9a50e..ff25eefee3 100644 --- a/utilities/document/document_db_test.cc +++ b/utilities/document/document_db_test.cc @@ -67,7 +67,7 @@ class DocumentDBTest { TEST(DocumentDBTest, SimpleQueryTest) { DocumentDBOptions options; DocumentDB::IndexDescriptor index; - index.description = Parse("{'name': 1}"); + index.description = Parse("{\"name\": 1}"); index.name = "name_index"; ASSERT_OK(DocumentDB::Open(options, dbname_, {}, &db_)); @@ -78,8 +78,8 @@ TEST(DocumentDBTest, SimpleQueryTest) { delete index.description; std::vector json_objects = { - "{'_id': 1, 'name': 'One'}", "{'_id': 2, 'name': 'Two'}", - "{'_id': 3, 'name': 'Three'}", "{'_id': 4, 'name': 'Four'}"}; + "{\"_id\': 1, \"name\": \"One\"}", "{\"_id\": 2, \"name\": \"Two\"}", + "{\"_id\": 3, \"name\": \"Three\"}", "{\"_id\": 4, \"name\": \"Four\"}"}; for (auto& json : json_objects) { std::unique_ptr document(Parse(json)); @@ -255,6 +255,24 @@ TEST(DocumentDBTest, ComplexQueryTest) { ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update)); } + // update twice: set priority to 15 where job_name is 'white' + { + std::unique_ptr query(Parse("{'job_name': 'white'}")); + std::unique_ptr update(Parse("{'$set': {'priority': 10}," + "'$set': {'priority': 15}}")); + ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update)); + } + + // update twice: set priority to 15 and + // progress to 40 where job_name is 'white' + { + std::unique_ptr query(Parse("{'job_name': 'white'}")); + std::unique_ptr update( + Parse("{'$set': {'priority': 10, 'progress': 35}," + "'$set': {'priority': 15, 'progress': 40}}")); + ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update)); + } + // priority < 0 { std::unique_ptr query( diff --git a/utilities/document/json_document.cc b/utilities/document/json_document.cc index c73283d0c1..34bc515aa9 100644 --- a/utilities/document/json_document.cc +++ b/utilities/document/json_document.cc @@ -10,509 +10,512 @@ #define __STDC_FORMAT_MACROS #endif +#include #include +#include -#include +#include +#include #include #include #include #include -#include "third-party/rapidjson/reader.h" + +#include "third-party/fbson/FbsonDocument.h" +#include "third-party/fbson/FbsonJsonParser.h" +#include "third-party/fbson/FbsonUtil.h" #include "util/coding.h" +using std::placeholders::_1; + +namespace { + +size_t ObjectNumElem(const fbson::ObjectVal& objectVal) { + size_t size = 0; + for (auto keyValuePair : objectVal) { + (void)keyValuePair; + ++size; + } + return size; +} + +template +void InitJSONDocument(std::unique_ptr* data, + fbson::FbsonValue** value, + Func f) { + // TODO(stash): maybe add function to FbsonDocument to avoid creating array? + fbson::FbsonWriter writer; + bool res = writer.writeStartArray(); + assert(res); + uint32_t bytesWritten __attribute__((unused)) = f(writer); + assert(bytesWritten != 0); + res = writer.writeEndArray(); + assert(res); + char* buf = new char[writer.getOutput()->getSize()]; + memcpy(buf, writer.getOutput()->getBuffer(), writer.getOutput()->getSize()); + + *value = ((fbson::FbsonDocument *)buf)->getValue(); + assert((*value)->isArray()); + assert(((fbson::ArrayVal*)*value)->numElem() == 1); + *value = ((fbson::ArrayVal*)*value)->get(0); + data->reset(buf); +} + +void InitString(std::unique_ptr* data, + fbson::FbsonValue** value, + const std::string& s) { + InitJSONDocument(data, value, std::bind( + [](fbson::FbsonWriter& writer, const std::string& str) -> uint32_t { + bool res = writer.writeStartString(); + assert(res); + auto bytesWritten = writer.writeString(str.c_str(), + static_cast(str.length())); + res = writer.writeEndString(); + assert(res); + // If the string is empty, then bytesWritten == 0, and assert in + // InitJsonDocument will fail. + return bytesWritten + static_cast(str.empty()); + }, + _1, s)); +} + +bool IsNumeric(fbson::FbsonValue* value) { + return value->isInt8() || value->isInt16() || + value->isInt32() || value->isInt64(); +} + +int64_t GetInt64ValFromFbsonNumericType(fbson::FbsonValue* value) { + switch (value->type()) { + case fbson::FbsonType::T_Int8: + return reinterpret_cast(value)->val(); + case fbson::FbsonType::T_Int16: + return reinterpret_cast(value)->val(); + case fbson::FbsonType::T_Int32: + return reinterpret_cast(value)->val(); + case fbson::FbsonType::T_Int64: + return reinterpret_cast(value)->val(); + default: + assert(false); + } + return 0; +} + +bool IsComparable(fbson::FbsonValue* left, fbson::FbsonValue* right) { + if (left->type() == right->type()) { + return true; + } + if (IsNumeric(left) && IsNumeric(right)) { + return true; + } + return false; +} + +void CreateArray(std::unique_ptr* data, fbson::FbsonValue** value) { + fbson::FbsonWriter writer; + bool res = writer.writeStartArray(); + assert(res); + res = writer.writeEndArray(); + assert(res); + data->reset(new char[writer.getOutput()->getSize()]); + memcpy(data->get(), + writer.getOutput()->getBuffer(), + writer.getOutput()->getSize()); + *value = reinterpret_cast(data->get())->getValue(); +} + +void CreateObject(std::unique_ptr* data, fbson::FbsonValue** value) { + fbson::FbsonWriter writer; + bool res = writer.writeStartObject(); + assert(res); + res = writer.writeEndObject(); + assert(res); + data->reset(new char[writer.getOutput()->getSize()]); + memcpy(data->get(), + writer.getOutput()->getBuffer(), + writer.getOutput()->getSize()); + *value = reinterpret_cast(data->get())->getValue(); +} + +} // namespace + namespace rocksdb { -JSONDocument::JSONDocument() : type_(kNull) {} -JSONDocument::JSONDocument(bool b) : type_(kBool) { data_.b = b; } -JSONDocument::JSONDocument(double d) : type_(kDouble) { data_.d = d; } -JSONDocument::JSONDocument(int64_t i) : type_(kInt64) { data_.i = i; } -JSONDocument::JSONDocument(const std::string& s) : type_(kString) { - new (&data_.s) std::string(s); + +// TODO(stash): find smth easier +JSONDocument::JSONDocument() { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeNull, _1)); } -JSONDocument::JSONDocument(const char* s) : type_(kString) { - new (&data_.s) std::string(s); + +JSONDocument::JSONDocument(bool b) { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeBool, _1, b)); } -JSONDocument::JSONDocument(Type _type) : type_(_type) { + +JSONDocument::JSONDocument(double d) { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeDouble, _1, d)); +} + +JSONDocument::JSONDocument(int8_t i) { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeInt8, _1, i)); +} + +JSONDocument::JSONDocument(int16_t i) { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeInt16, _1, i)); +} + +JSONDocument::JSONDocument(int32_t i) { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeInt32, _1, i)); +} + +JSONDocument::JSONDocument(int64_t i) { + InitJSONDocument(&data_, + &value_, + std::bind(&fbson::FbsonWriter::writeInt64, _1, i)); +} + +JSONDocument::JSONDocument(const std::string& s) { + InitString(&data_, &value_, s); +} + +JSONDocument::JSONDocument(const char* s) : JSONDocument(std::string(s)) { +} + +void JSONDocument::InitFromValue(const fbson::FbsonValue* val) { + data_.reset(new char[val->numPackedBytes()]); + memcpy(data_.get(), val, val->numPackedBytes()); + value_ = reinterpret_cast(data_.get()); +} + +// Private constructor +JSONDocument::JSONDocument(fbson::FbsonValue* val, bool makeCopy) { + if (makeCopy) { + InitFromValue(val); + } else { + value_ = val; + } +} + +JSONDocument::JSONDocument(Type _type) { // TODO(icanadi) make all of this better by using templates - switch (type_) { + switch (_type) { case kNull: + InitJSONDocument(&data_, &value_, + std::bind(&fbson::FbsonWriter::writeNull, _1)); break; case kObject: - new (&data_.o) Object; + CreateObject(&data_, &value_); break; case kBool: - data_.b = false; + InitJSONDocument(&data_, &value_, + std::bind(&fbson::FbsonWriter::writeBool, _1, false)); break; case kDouble: - data_.d = 0.0; + InitJSONDocument(&data_, &value_, + std::bind(&fbson::FbsonWriter::writeDouble, _1, 0.)); break; case kArray: - new (&data_.a) Array; + CreateArray(&data_, &value_); break; case kInt64: - data_.i = 0; + InitJSONDocument(&data_, &value_, + std::bind(&fbson::FbsonWriter::writeInt64, _1, 0)); break; case kString: - new (&data_.s) std::string(); + InitString(&data_, &value_, ""); break; default: assert(false); } } -JSONDocument::JSONDocument(const JSONDocument& json_document) - : JSONDocument(json_document.type_) { - switch (json_document.type_) { - case kNull: - break; - case kArray: - data_.a.reserve(json_document.data_.a.size()); - for (const auto& iter : json_document.data_.a) { - // deep copy - data_.a.push_back(new JSONDocument(*iter)); - } - break; - case kBool: - data_.b = json_document.data_.b; - break; - case kDouble: - data_.d = json_document.data_.d; - break; - case kInt64: - data_.i = json_document.data_.i; - break; - case kObject: { - for (const auto& iter : json_document.data_.o) { - // deep copy - data_.o.insert({iter.first, new JSONDocument(*iter.second)}); - } - break; - } - case kString: - data_.s = json_document.data_.s; - break; +JSONDocument::JSONDocument(const JSONDocument& jsonDocument) { + if (jsonDocument.IsOwner()) { + InitFromValue(jsonDocument.value_); + } else { + value_ = jsonDocument.value_; + } +} + +JSONDocument::JSONDocument(JSONDocument&& jsonDocument) { + value_ = jsonDocument.value_; + data_.swap(jsonDocument.data_); +} + +JSONDocument& JSONDocument::operator=(JSONDocument jsonDocument) { + value_ = jsonDocument.value_; + data_.swap(jsonDocument.data_); + return *this; +} + +JSONDocument::Type JSONDocument::type() const { + switch (value_->type()) { + case fbson::FbsonType::T_Null: + return JSONDocument::kNull; + + case fbson::FbsonType::T_True: + case fbson::FbsonType::T_False: + return JSONDocument::kBool; + + case fbson::FbsonType::T_Int8: + case fbson::FbsonType::T_Int16: + case fbson::FbsonType::T_Int32: + case fbson::FbsonType::T_Int64: + return JSONDocument::kInt64; + + case fbson::FbsonType::T_Double: + return JSONDocument::kDouble; + + case fbson::FbsonType::T_String: + return JSONDocument::kString; + + case fbson::FbsonType::T_Object: + return JSONDocument::kObject; + + case fbson::FbsonType::T_Array: + return JSONDocument::kArray; + + case fbson::FbsonType::T_Binary: + assert(false); default: assert(false); } + return JSONDocument::kNull; } -JSONDocument::~JSONDocument() { - switch (type_) { - case kObject: - for (auto iter : data_.o) { - delete iter.second; - } - (&data_.o)->~Object(); - break; - case kArray: - for (auto iter : data_.a) { - delete iter; - } - (&data_.a)->~Array(); - break; - case kString: - using std::string; - (&data_.s)->~string(); - break; - default: - // we're cool, no need for destructors for others - break; - } -} - -JSONDocument::Type JSONDocument::type() const { return type_; } - bool JSONDocument::Contains(const std::string& key) const { - assert(type_ == kObject); - auto iter = data_.o.find(key); - return iter != data_.o.end(); + assert(IsObject()); + auto objectVal = reinterpret_cast(value_); + return objectVal->find(key.c_str()) != nullptr; } -const JSONDocument* JSONDocument::Get(const std::string& key) const { - assert(type_ == kObject); - auto iter = data_.o.find(key); - if (iter == data_.o.end()) { - return nullptr; - } - return iter->second; -} - -JSONDocument& JSONDocument::operator[](const std::string& key) { - assert(type_ == kObject); - auto iter = data_.o.find(key); - assert(iter != data_.o.end()); - return *(iter->second); -} - -const JSONDocument& JSONDocument::operator[](const std::string& key) const { - assert(type_ == kObject); - auto iter = data_.o.find(key); - assert(iter != data_.o.end()); - return *(iter->second); -} - -JSONDocument* JSONDocument::Set(const std::string& key, const JSONDocument& value) { - assert(type_ == kObject); - auto itr = data_.o.find(key); - if (itr == data_.o.end()) { - // insert - data_.o.insert({key, new JSONDocument(value)}); - } else { - // overwrite - delete itr->second; - itr->second = new JSONDocument(value); - } - return this; +JSONDocument JSONDocument::operator[](const std::string& key) const { + assert(IsObject()); + auto objectVal = reinterpret_cast(value_); + auto foundValue = objectVal->find(key.c_str()); + assert(foundValue != nullptr); + // No need to save paths in const objects + JSONDocument ans(foundValue, false); + return std::move(ans); } size_t JSONDocument::Count() const { - assert(type_ == kArray || type_ == kObject); - if (type_ == kArray) { - return data_.a.size(); - } else if (type_ == kObject) { - return data_.o.size(); + assert(IsObject() || IsArray()); + if (IsObject()) { + // TODO(stash): add to fbson? + const fbson::ObjectVal& objectVal = + *reinterpret_cast(value_); + return ObjectNumElem(objectVal); + } else if (IsArray()) { + auto arrayVal = reinterpret_cast(value_); + return arrayVal->numElem(); } assert(false); return 0; } -const JSONDocument* JSONDocument::GetFromArray(size_t i) const { - assert(type_ == kArray); - return data_.a[i]; -} - -JSONDocument& JSONDocument::operator[](size_t i) { - assert(type_ == kArray && i < data_.a.size()); - return *data_.a[i]; -} - -const JSONDocument& JSONDocument::operator[](size_t i) const { - assert(type_ == kArray && i < data_.a.size()); - return *data_.a[i]; -} - -JSONDocument* JSONDocument::SetInArray(size_t i, const JSONDocument& value) { - assert(IsArray() && i < data_.a.size()); - delete data_.a[i]; - data_.a[i] = new JSONDocument(value); - return this; -} - -JSONDocument* JSONDocument::PushBack(const JSONDocument& value) { +JSONDocument JSONDocument::operator[](size_t i) const { assert(IsArray()); - data_.a.push_back(new JSONDocument(value)); - return this; + auto arrayVal = reinterpret_cast(value_); + auto foundValue = arrayVal->get(static_cast(i)); + JSONDocument ans(foundValue, false); + return std::move(ans); } -bool JSONDocument::IsNull() const { return type() == kNull; } -bool JSONDocument::IsArray() const { return type() == kArray; } -bool JSONDocument::IsBool() const { return type() == kBool; } -bool JSONDocument::IsDouble() const { return type() == kDouble; } -bool JSONDocument::IsInt64() const { return type() == kInt64; } -bool JSONDocument::IsObject() const { return type() == kObject; } -bool JSONDocument::IsString() const { return type() == kString; } +bool JSONDocument::IsNull() const { + return value_->isNull(); +} + +bool JSONDocument::IsArray() const { + return value_->isArray(); +} + +bool JSONDocument::IsBool() const { + return value_->isTrue() || value_->isFalse(); +} + +bool JSONDocument::IsDouble() const { + return value_->isDouble(); +} + +bool JSONDocument::IsInt64() const { + return value_->isInt8() || value_->isInt16() || + value_->isInt32() || value_->isInt64(); +} + +bool JSONDocument::IsObject() const { + return value_->isObject(); +} + +bool JSONDocument::IsString() const { + return value_->isString(); +} bool JSONDocument::GetBool() const { assert(IsBool()); - return data_.b; -} -double JSONDocument::GetDouble() const { - assert(IsDouble()); - return data_.d; -} -int64_t JSONDocument::GetInt64() const { - assert(IsInt64()); - return data_.i; -} -const std::string& JSONDocument::GetString() const { - assert(IsString()); - return data_.s; + return value_->isTrue(); } -bool JSONDocument::operator==(const JSONDocument& rhs) const { - if (type_ != rhs.type_) { +double JSONDocument::GetDouble() const { + assert(IsDouble()); + return ((fbson::DoubleVal*)value_)->val(); +} + +int64_t JSONDocument::GetInt64() const { + assert(IsInt64()); + return GetInt64ValFromFbsonNumericType(value_); +} + +std::string JSONDocument::GetString() const { + assert(IsString()); + fbson::StringVal* stringVal = (fbson::StringVal*)value_; + return std::string(stringVal->getBlob(), stringVal->getBlobLen()); +} + +namespace { + +// FbsonValue can be int8, int16, int32, int64 +bool CompareNumeric(fbson::FbsonValue* left, fbson::FbsonValue* right) { + assert(IsNumeric(left) && IsNumeric(right)); + return GetInt64ValFromFbsonNumericType(left) == + GetInt64ValFromFbsonNumericType(right); +} + +bool CompareSimpleTypes(fbson::FbsonValue* left, fbson::FbsonValue* right) { + if (IsNumeric(left)) { + return CompareNumeric(left, right); + } + if (left->numPackedBytes() != right->numPackedBytes()) { return false; } - switch (type_) { - case kNull: - return true; // null == null - case kArray: - if (data_.a.size() != rhs.data_.a.size()) { + return memcmp(left, right, left->numPackedBytes()) == 0; +} + +bool Compare(fbson::FbsonValue* left, fbson::FbsonValue* right) { + if (!IsComparable(left, right)) { + return false; + } + + switch (left->type()) { + case fbson::FbsonType::T_True: + case fbson::FbsonType::T_False: + case fbson::FbsonType::T_Null: + return true; + case fbson::FbsonType::T_Int8: + case fbson::FbsonType::T_Int16: + case fbson::FbsonType::T_Int32: + case fbson::FbsonType::T_Int64: + return CompareNumeric(left, right); + case fbson::FbsonType::T_String: + case fbson::FbsonType::T_Double: + return CompareSimpleTypes(left, right); + case fbson::FbsonType::T_Object: + { + auto leftObject = reinterpret_cast(left); + auto rightObject = reinterpret_cast(right); + if (ObjectNumElem(*leftObject) != ObjectNumElem(*rightObject)) { return false; } - for (size_t i = 0; i < data_.a.size(); ++i) { - if (!(*data_.a[i] == *rhs.data_.a[i])) { + for (auto && keyValue : *leftObject) { + std::string str(keyValue.getKeyStr(), keyValue.klen()); + if (rightObject->find(str.c_str()) == nullptr) { + return false; + } + if (!Compare(keyValue.value(), + rightObject->find(str.c_str()))) { return false; } } return true; - case kBool: - return data_.b == rhs.data_.b; - case kDouble: - return std::fabs(data_.d - rhs.data_.d) <= std::numeric_limits::epsilon(); - case kInt64: - return data_.i == rhs.data_.i; - case kObject: - if (data_.o.size() != rhs.data_.o.size()) { + } + case fbson::FbsonType::T_Array: + { + auto leftArr = reinterpret_cast(left); + auto rightArr = reinterpret_cast(right); + if (leftArr->numElem() != rightArr->numElem()) { return false; } - for (const auto& iter : data_.o) { - auto rhs_iter = rhs.data_.o.find(iter.first); - if (rhs_iter == rhs.data_.o.end() || - !(*(rhs_iter->second) == *iter.second)) { + for (int i = 0; i < static_cast(leftArr->numElem()); ++i) { + if (!Compare(leftArr->get(i), rightArr->get(i))) { return false; } } return true; - case kString: - return data_.s == rhs.data_.s; + } default: assert(false); } - // it can't come to here, but we don't want the compiler to complain return false; } +} // namespace + +bool JSONDocument::operator==(const JSONDocument& rhs) const { + return Compare(value_, rhs.value_); +} + +bool JSONDocument::operator!=(const JSONDocument& rhs) const { + return !(*this == rhs); +} + +JSONDocument JSONDocument::Copy() const { + return JSONDocument(value_, true); +} + +bool JSONDocument::IsOwner() const { + return data_.get() != nullptr; +} + std::string JSONDocument::DebugString() const { - std::string ret; - switch (type_) { - case kNull: - ret = "null"; - break; - case kArray: - ret = "["; - for (size_t i = 0; i < data_.a.size(); ++i) { - if (i) { - ret += ", "; - } - ret += data_.a[i]->DebugString(); - } - ret += "]"; - break; - case kBool: - ret = data_.b ? "true" : "false"; - break; - case kDouble: { - char buf[100]; - snprintf(buf, sizeof(buf), "%lf", data_.d); - ret = buf; - break; - } - case kInt64: { - char buf[100]; - snprintf(buf, sizeof(buf), "%" PRIi64, data_.i); - ret = buf; - break; - } - case kObject: { - bool first = true; - ret = "{"; - for (const auto& iter : data_.o) { - ret += first ? "" : ", "; - first = false; - ret += iter.first + ": "; - ret += iter.second->DebugString(); - } - ret += "}"; - break; - } - case kString: - ret = "\"" + data_.s + "\""; - break; - default: - assert(false); - } - return ret; + fbson::FbsonToJson fbsonToJson; + return fbsonToJson.json(value_); } JSONDocument::ItemsIteratorGenerator JSONDocument::Items() const { - assert(type_ == kObject); - return data_.o; + assert(IsObject()); + return ItemsIteratorGenerator(*(static_cast(value_))); } -// parsing with rapidjson // TODO(icanadi) (perf) allocate objects with arena JSONDocument* JSONDocument::ParseJSON(const char* json) { - class JSONDocumentBuilder { - public: - JSONDocumentBuilder() {} - - void Null() { stack_.push_back(new JSONDocument()); } - void Bool(bool b) { stack_.push_back(new JSONDocument(b)); } - void Int(int i) { Int64(static_cast(i)); } - void Uint(unsigned i) { Int64(static_cast(i)); } - void Int64(int64_t i) { stack_.push_back(new JSONDocument(i)); } - void Uint64(uint64_t i) { Int64(static_cast(i)); } - void Double(double d) { stack_.push_back(new JSONDocument(d)); } - void String(const char* str, size_t length, bool copy) { - assert(copy); - stack_.push_back(new JSONDocument(std::string(str, length))); - } - void StartObject() { stack_.push_back(new JSONDocument(kObject)); } - void EndObject(size_t member_count) { - assert(stack_.size() > 2 * member_count); - auto object_base_iter = stack_.end() - member_count * 2 - 1; - assert((*object_base_iter)->type_ == kObject); - auto& object_map = (*object_base_iter)->data_.o; - // iter will always be stack_.end() at some point (i.e. will not advance - // past it) because of the way we calculate object_base_iter - for (auto iter = object_base_iter + 1; iter != stack_.end(); iter += 2) { - assert((*iter)->type_ == kString); - object_map.insert({(*iter)->data_.s, *(iter + 1)}); - delete *iter; - } - stack_.erase(object_base_iter + 1, stack_.end()); - } - void StartArray() { stack_.push_back(new JSONDocument(kArray)); } - void EndArray(size_t element_count) { - assert(stack_.size() > element_count); - auto array_base_iter = stack_.end() - element_count - 1; - assert((*array_base_iter)->type_ == kArray); - (*array_base_iter)->data_.a.assign(array_base_iter + 1, stack_.end()); - stack_.erase(array_base_iter + 1, stack_.end()); - } - - JSONDocument* GetDocument() { - if (stack_.size() != 1) { - return nullptr; - } - return stack_.back(); - } - - void DeleteAllDocumentsOnStack() { - for (auto document : stack_) { - delete document; - } - stack_.clear(); - } - - private: - std::vector stack_; - }; - - rapidjson::StringStream stream(json); - rapidjson::Reader reader; - JSONDocumentBuilder handler; - bool ok = reader.Parse<0>(stream, handler); - if (!ok) { - handler.DeleteAllDocumentsOnStack(); + fbson::FbsonJsonParser parser; + if (!parser.parse(json)) { return nullptr; } - auto document = handler.GetDocument(); - assert(document != nullptr); - return document; -} -// serialization and deserialization -// format: -// ------ -// document ::= header(char) object -// object ::= varint32(n) key_value*(n times) -// key_value ::= string element -// element ::= 0x01 (kNull) -// | 0x02 array (kArray) -// | 0x03 byte (kBool) -// | 0x04 double (kDouble) -// | 0x05 int64 (kInt64) -// | 0x06 object (kObject) -// | 0x07 string (kString) -// array ::= varint32(n) element*(n times) -// TODO(icanadi) evaluate string vs cstring format -// string ::= varint32(n) byte*(n times) -// double ::= 64-bit IEEE 754 floating point (8 bytes) -// int64 ::= 8 bytes, 64-bit signed integer, little endian + auto fbsonVal = fbson::FbsonDocument::createValue( + parser.getWriter().getOutput()->getBuffer(), + static_cast(parser.getWriter().getOutput()->getSize())); -namespace { -inline char GetPrefixFromType(JSONDocument::Type type) { - static char types[] = {0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7}; - return types[type]; -} - -inline bool GetNextType(Slice* input, JSONDocument::Type* type) { - if (input->size() == 0) { - return false; + if (fbsonVal == nullptr) { + return nullptr; } - static JSONDocument::Type prefixes[] = { - JSONDocument::kNull, JSONDocument::kArray, JSONDocument::kBool, - JSONDocument::kDouble, JSONDocument::kInt64, JSONDocument::kObject, - JSONDocument::kString}; - size_t prefix = static_cast((*input)[0]); - if (prefix == 0 || prefix >= 0x8) { - return false; - } - input->remove_prefix(1); - *type = prefixes[static_cast(prefix - 1)]; - return true; -} -// TODO(icanadi): Make sure this works on all platforms we support. Some -// platforms may store double in different binary format (our specification says -// we need IEEE 754) -inline void PutDouble(std::string* dst, double d) { - dst->append(reinterpret_cast(&d), sizeof(d)); + return new JSONDocument(fbsonVal, true); } -bool DecodeDouble(Slice* input, double* d) { - if (input->size() < sizeof(double)) { - return false; - } - memcpy(d, input->data(), sizeof(double)); - input->remove_prefix(sizeof(double)); - - return true; -} -} // namespace - void JSONDocument::Serialize(std::string* dst) const { // first byte is reserved for header // currently, header is only version number. that will help us provide // backwards compatility. we might also store more information here if // necessary dst->push_back(kSerializationFormatVersion); - SerializeInternal(dst, false); + dst->push_back(FBSON_VER); + dst->append(reinterpret_cast(value_), value_->numPackedBytes()); } -void JSONDocument::SerializeInternal(std::string* dst, bool type_prefix) const { - if (type_prefix) { - dst->push_back(GetPrefixFromType(type_)); - } - switch (type_) { - case kNull: - // just the prefix is all we need - break; - case kArray: - PutVarint32(dst, static_cast(data_.a.size())); - for (const auto& element : data_.a) { - element->SerializeInternal(dst, true); - } - break; - case kBool: - dst->push_back(static_cast(data_.b)); - break; - case kDouble: - PutDouble(dst, data_.d); - break; - case kInt64: - PutFixed64(dst, static_cast(data_.i)); - break; - case kObject: { - PutVarint32(dst, static_cast(data_.o.size())); - for (const auto& iter : data_.o) { - PutLengthPrefixedSlice(dst, Slice(iter.first)); - iter.second->SerializeInternal(dst, true); - } - break; - } - case kString: - PutLengthPrefixedSlice(dst, Slice(data_.s)); - break; - default: - assert(false); - } -} - -const char JSONDocument::kSerializationFormatVersion = 1; +const char JSONDocument::kSerializationFormatVersion = 2; JSONDocument* JSONDocument::Deserialize(const Slice& src) { Slice input(src); @@ -520,102 +523,87 @@ JSONDocument* JSONDocument::Deserialize(const Slice& src) { return nullptr; } char header = input[0]; - if (header != kSerializationFormatVersion) { - // don't understand this header (possibly newer version format and we don't - // support downgrade) - return nullptr; + if (header == 1) { + assert(false); } input.remove_prefix(1); - auto root = new JSONDocument(kObject); - bool ok = root->DeserializeInternal(&input); - if (!ok || input.size() > 0) { - // parsing failure :( - delete root; + auto value = fbson::FbsonDocument::createValue(input.data(), + static_cast(input.size())); + if (value == nullptr) { return nullptr; } - return root; + + return new JSONDocument(value, true); } -bool JSONDocument::DeserializeInternal(Slice* input) { - switch (type_) { - case kNull: - break; - case kArray: { - uint32_t size; - if (!GetVarint32(input, &size)) { - return false; - } - data_.a.resize(size); - for (size_t i = 0; i < size; ++i) { - Type t; - if (!GetNextType(input, &t)) { - return false; - } - data_.a[i] = new JSONDocument(t); - if (!data_.a[i]->DeserializeInternal(input)) { - return false; - } - } - break; - } - case kBool: - if (input->size() < 1) { - return false; - } - data_.b = static_cast((*input)[0]); - input->remove_prefix(1); - break; - case kDouble: - if (!DecodeDouble(input, &data_.d)) { - return false; - } - break; - case kInt64: { - uint64_t tmp; - if (!GetFixed64(input, &tmp)) { - return false; - } - data_.i = static_cast(tmp); - break; - } - case kObject: { - uint32_t num_elements; - bool ok = GetVarint32(input, &num_elements); - for (uint32_t i = 0; ok && i < num_elements; ++i) { - Slice key; - ok = GetLengthPrefixedSlice(input, &key); - Type t; - ok = ok && GetNextType(input, &t); - if (ok) { - std::unique_ptr value(new JSONDocument(t)); - ok = value->DeserializeInternal(input); - if (ok) { - data_.o.insert({key.ToString(), value.get()}); - value.release(); - } - } - } - if (!ok) { - return false; - } - break; - } - case kString: { - Slice key; - if (!GetLengthPrefixedSlice(input, &key)) { - return false; - } - data_.s = key.ToString(); - break; - } - default: - // this is an assert and not a return because DeserializeInternal() will - // always be called with a valid type_. In case there has been data - // corruption, GetNextType() is the function that will detect that and - // return corruption - assert(false); +class JSONDocument::const_item_iterator::Impl { + public: + typedef fbson::ObjectVal::const_iterator It; + + explicit Impl(It it) : it_(it) {} + + const char* getKeyStr() const { + return it_->getKeyStr(); } - return true; + + uint8_t klen() const { + return it_->klen(); + } + + It& operator++() { + return ++it_; + } + + bool operator!=(const Impl& other) { + return it_ != other.it_; + } + + fbson::FbsonValue* value() const { + return it_->value(); + } + + private: + It it_; +}; + +JSONDocument::const_item_iterator::const_item_iterator(Impl* impl) +: it_(impl) {} + +JSONDocument::const_item_iterator::const_item_iterator(const_item_iterator&& a) +: it_(std::move(a.it_)) {} + +JSONDocument::const_item_iterator& + JSONDocument::const_item_iterator::operator++() { + ++(*it_); + return *this; +} + +bool JSONDocument::const_item_iterator::operator!=( + const const_item_iterator& other) { + return *it_ != *(other.it_); +} + +JSONDocument::const_item_iterator::~const_item_iterator() { +} + +JSONDocument::const_item_iterator::value_type + JSONDocument::const_item_iterator::operator*() { + return {std::string(it_->getKeyStr(), it_->klen()), + JSONDocument(it_->value(), false)}; +} + +JSONDocument::ItemsIteratorGenerator::ItemsIteratorGenerator( + const fbson::ObjectVal& object) + : object_(object) {} + +JSONDocument::const_item_iterator + JSONDocument::ItemsIteratorGenerator::begin() const { + return const_item_iterator(new const_item_iterator::Impl(object_.begin())); +} + +JSONDocument::const_item_iterator + JSONDocument::ItemsIteratorGenerator::end() const { + return const_item_iterator(new const_item_iterator::Impl(object_.end())); } } // namespace rocksdb diff --git a/utilities/document/json_document_builder.cc b/utilities/document/json_document_builder.cc new file mode 100644 index 0000000000..0dd4ce4646 --- /dev/null +++ b/utilities/document/json_document_builder.cc @@ -0,0 +1,115 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE +#include "rocksdb/utilities/json_document.h" +#include "third-party/fbson/FbsonWriter.h" + +namespace rocksdb { +JSONDocumentBuilder::JSONDocumentBuilder() +: writer_(new fbson::FbsonWriter()) { +} + +JSONDocumentBuilder::JSONDocumentBuilder(fbson::FbsonOutStream* out) +: writer_(new fbson::FbsonWriter(*out)) { +} + +void JSONDocumentBuilder::Reset() { + writer_->reset(); +} + +bool JSONDocumentBuilder::WriteStartArray() { + return writer_->writeStartArray(); +} + +bool JSONDocumentBuilder::WriteEndArray() { + return writer_->writeEndArray(); +} + +bool JSONDocumentBuilder::WriteStartObject() { + return writer_->writeStartObject(); +} + +bool JSONDocumentBuilder::WriteEndObject() { + return writer_->writeEndObject(); +} + +bool JSONDocumentBuilder::WriteKeyValue(const std::string& key, + const JSONDocument& value) { + size_t bytesWritten = writer_->writeKey(key.c_str(), key.size()); + if (bytesWritten == 0) { + return false; + } + return WriteJSONDocument(value); +} + +bool JSONDocumentBuilder::WriteJSONDocument(const JSONDocument& value) { + switch (value.type()) { + case JSONDocument::kNull: + return writer_->writeNull() != 0; + case JSONDocument::kInt64: + return writer_->writeInt64(value.GetInt64()); + case JSONDocument::kDouble: + return writer_->writeDouble(value.GetDouble()); + case JSONDocument::kBool: + return writer_->writeBool(value.GetBool()); + case JSONDocument::kString: + { + bool res = writer_->writeStartString(); + if (!res) { + return false; + } + const std::string& str = value.GetString(); + res = writer_->writeString(str.c_str(), + static_cast(str.size())); + if (!res) { + return false; + } + return writer_->writeEndString(); + } + case JSONDocument::kArray: + { + bool res = WriteStartArray(); + if (!res) { + return false; + } + for (size_t i = 0; i < value.Count(); ++i) { + res = WriteJSONDocument(value[i]); + if (!res) { + return false; + } + } + return WriteEndArray(); + } + case JSONDocument::kObject: + { + bool res = WriteStartObject(); + if (!res) { + return false; + } + for (auto keyValue : value.Items()) { + WriteKeyValue(keyValue.first, keyValue.second); + } + return WriteEndObject(); + } + default: + assert(false); + } + return false; +} + +JSONDocument JSONDocumentBuilder::GetJSONDocument() { + fbson::FbsonValue* value = + fbson::FbsonDocument::createValue(writer_->getOutput()->getBuffer(), + static_cast(writer_->getOutput()->getSize())); + return JSONDocument(value, true); +} + +JSONDocumentBuilder::~JSONDocumentBuilder() { +} + +} // namespace rocksdb + +#endif // ROCKSDB_LITE diff --git a/utilities/document/json_document_test.cc b/utilities/document/json_document_test.cc index 8d1967ed91..17646cb858 100644 --- a/utilities/document/json_document_test.cc +++ b/utilities/document/json_document_test.cc @@ -3,7 +3,9 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. +#include #include +#include #include "rocksdb/utilities/json_document.h" @@ -48,6 +50,10 @@ void AssertField(const JSONDocument& json, const std::string& field, class JSONDocumentTest { public: + JSONDocumentTest() + : rnd_(101) + {} + void AssertSampleJSON(const JSONDocument& json) { AssertField(json, "title", std::string("json")); AssertField(json, "type", std::string("object")); @@ -94,14 +100,95 @@ class JSONDocumentTest { "\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": 2}], " "\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], " "\"enabled\": true }"; + + Random rnd_; }; -TEST(JSONDocumentTest, Parsing) { - JSONDocument x(static_cast(5)); - ASSERT_TRUE(x.IsInt64()); +TEST(JSONDocumentTest, MakeNullTest) { + JSONDocument x; + ASSERT_TRUE(x.IsNull()); + ASSERT_TRUE(x.IsOwner()); + ASSERT_TRUE(!x.IsBool()); +} - // make sure it's correctly parsed - auto parsed_json = JSONDocument::ParseJSON(kSampleJSON.c_str()); +TEST(JSONDocumentTest, MakeBoolTest) { + { + JSONDocument x(true); + ASSERT_TRUE(x.IsOwner()); + ASSERT_TRUE(x.IsBool()); + ASSERT_TRUE(!x.IsInt64()); + ASSERT_EQ(x.GetBool(), true); + } + + { + JSONDocument x(false); + ASSERT_TRUE(x.IsOwner()); + ASSERT_TRUE(x.IsBool()); + ASSERT_TRUE(!x.IsInt64()); + ASSERT_EQ(x.GetBool(), false); + } +} + +TEST(JSONDocumentTest, MakeInt64Test) { + JSONDocument x(static_cast(16)); + ASSERT_TRUE(x.IsInt64()); + ASSERT_TRUE(x.IsInt64()); + ASSERT_TRUE(!x.IsBool()); + ASSERT_TRUE(x.IsOwner()); + ASSERT_EQ(x.GetInt64(), 16); +} + +TEST(JSONDocumentTest, MakeStringTest) { + JSONDocument x("string"); + ASSERT_TRUE(x.IsOwner()); + ASSERT_TRUE(x.IsString()); + ASSERT_TRUE(!x.IsBool()); + ASSERT_EQ(x.GetString(), "string"); +} + +TEST(JSONDocumentTest, MakeDoubleTest) { + JSONDocument x(5.6); + ASSERT_TRUE(x.IsOwner()); + ASSERT_TRUE(x.IsDouble()); + ASSERT_TRUE(!x.IsBool()); + ASSERT_EQ(x.GetDouble(), 5.6); +} + +TEST(JSONDocumentTest, MakeByTypeTest) { + { + JSONDocument x(JSONDocument::kNull); + ASSERT_TRUE(x.IsNull()); + } + { + JSONDocument x(JSONDocument::kBool); + ASSERT_TRUE(x.IsBool()); + } + { + JSONDocument x(JSONDocument::kString); + ASSERT_TRUE(x.IsString()); + } + { + JSONDocument x(JSONDocument::kInt64); + ASSERT_TRUE(x.IsInt64()); + } + { + JSONDocument x(JSONDocument::kDouble); + ASSERT_TRUE(x.IsDouble()); + } + { + JSONDocument x(JSONDocument::kObject); + ASSERT_TRUE(x.IsObject()); + } + { + JSONDocument x(JSONDocument::kArray); + ASSERT_TRUE(x.IsArray()); + } +} + +TEST(JSONDocumentTest, Parsing) { + std::unique_ptr parsed_json( + JSONDocument::ParseJSON(kSampleJSON.c_str())); + ASSERT_TRUE(parsed_json->IsOwner()); ASSERT_TRUE(parsed_json != nullptr); AssertSampleJSON(*parsed_json); @@ -109,13 +196,11 @@ TEST(JSONDocumentTest, Parsing) { JSONDocument copied_json_document(*parsed_json); AssertSampleJSON(copied_json_document); ASSERT_TRUE(copied_json_document == *parsed_json); - delete parsed_json; - auto parsed_different_sample = - JSONDocument::ParseJSON(kSampleJSONDifferent.c_str()); + std::unique_ptr parsed_different_sample( + JSONDocument::ParseJSON(kSampleJSONDifferent.c_str())); ASSERT_TRUE(parsed_different_sample != nullptr); ASSERT_TRUE(!(*parsed_different_sample == copied_json_document)); - delete parsed_different_sample; // parse error const std::string kFaultyJSON = @@ -124,45 +209,116 @@ TEST(JSONDocumentTest, Parsing) { } TEST(JSONDocumentTest, Serialization) { - auto parsed_json = JSONDocument::ParseJSON(kSampleJSON.c_str()); + std::unique_ptr parsed_json( + JSONDocument::ParseJSON(kSampleJSON.c_str())); ASSERT_TRUE(parsed_json != nullptr); + ASSERT_TRUE(parsed_json->IsOwner()); std::string serialized; parsed_json->Serialize(&serialized); - delete parsed_json; - auto deserialized_json = JSONDocument::Deserialize(Slice(serialized)); + std::unique_ptr deserialized_json( + JSONDocument::Deserialize(Slice(serialized))); ASSERT_TRUE(deserialized_json != nullptr); AssertSampleJSON(*deserialized_json); - delete deserialized_json; // deserialization failure ASSERT_TRUE(JSONDocument::Deserialize( Slice(serialized.data(), serialized.size() - 10)) == nullptr); } -TEST(JSONDocumentTest, Mutation) { - auto sample_json = JSONDocument::ParseJSON(kSampleJSON.c_str()); - ASSERT_TRUE(sample_json != nullptr); - auto different_json = JSONDocument::ParseJSON(kSampleJSONDifferent.c_str()); - ASSERT_TRUE(different_json != nullptr); +TEST(JSONDocumentTest, OperatorEqualsTest) { + // kNull + ASSERT_TRUE(JSONDocument() == JSONDocument()); - (*different_json)["properties"]["flags"][2].Set("status", JSONDocument()); + // kBool + ASSERT_TRUE(JSONDocument(false) != JSONDocument()); + ASSERT_TRUE(JSONDocument(false) == JSONDocument(false)); + ASSERT_TRUE(JSONDocument(true) == JSONDocument(true)); + ASSERT_TRUE(JSONDocument(false) != JSONDocument(true)); - ASSERT_TRUE(*different_json == *sample_json); + // kString + ASSERT_TRUE(JSONDocument("test") != JSONDocument()); + ASSERT_TRUE(JSONDocument("test") == JSONDocument("test")); - delete different_json; - delete sample_json; + // kInt64 + ASSERT_TRUE(JSONDocument(static_cast(15)) != JSONDocument()); + ASSERT_TRUE(JSONDocument(static_cast(15)) != + JSONDocument(static_cast(14))); + ASSERT_TRUE(JSONDocument(static_cast(15)) == + JSONDocument(static_cast(15))); - auto json1 = JSONDocument::ParseJSON("{\"a\": [1, 2, 3]}"); - auto json2 = JSONDocument::ParseJSON("{\"a\": [2, 2, 3, 4]}"); - ASSERT_TRUE(json1 != nullptr && json2 != nullptr); + unique_ptr arrayWithInt8Doc(JSONDocument::ParseJSON("[8]")); + ASSERT_TRUE(arrayWithInt8Doc != nullptr); + ASSERT_TRUE(arrayWithInt8Doc->IsArray()); + ASSERT_TRUE((*arrayWithInt8Doc)[0].IsInt64()); + ASSERT_TRUE((*arrayWithInt8Doc)[0] == JSONDocument(static_cast(8))); - (*json1)["a"].SetInArray(0, static_cast(2))->PushBack( - static_cast(4)); - ASSERT_TRUE(*json1 == *json2); + unique_ptr arrayWithInt16Doc(JSONDocument::ParseJSON("[512]")); + ASSERT_TRUE(arrayWithInt16Doc != nullptr); + ASSERT_TRUE(arrayWithInt16Doc->IsArray()); + ASSERT_TRUE((*arrayWithInt16Doc)[0].IsInt64()); + ASSERT_TRUE((*arrayWithInt16Doc)[0] == + JSONDocument(static_cast(512))); - delete json1; - delete json2; + unique_ptr arrayWithInt32Doc( + JSONDocument::ParseJSON("[1000000]")); + ASSERT_TRUE(arrayWithInt32Doc != nullptr); + ASSERT_TRUE(arrayWithInt32Doc->IsArray()); + ASSERT_TRUE((*arrayWithInt32Doc)[0].IsInt64()); + ASSERT_TRUE((*arrayWithInt32Doc)[0] == + JSONDocument(static_cast(1000000))); + + // kDouble + ASSERT_TRUE(JSONDocument(15.) != JSONDocument()); + ASSERT_TRUE(JSONDocument(15.) != JSONDocument(14.)); + ASSERT_TRUE(JSONDocument(15.) == JSONDocument(15.)); +} + +TEST(JSONDocumentTest, JSONDocumentBuilderTest) { + unique_ptr parsedArray( + JSONDocument::ParseJSON("[1, [123, \"a\", \"b\"], {\"b\":\"c\"}]")); + ASSERT_TRUE(parsedArray != nullptr); + + JSONDocumentBuilder builder; + ASSERT_TRUE(builder.WriteStartArray()); + ASSERT_TRUE(builder.WriteJSONDocument(1)); + + ASSERT_TRUE(builder.WriteStartArray()); + ASSERT_TRUE(builder.WriteJSONDocument(123)); + ASSERT_TRUE(builder.WriteJSONDocument("a")); + ASSERT_TRUE(builder.WriteJSONDocument("b")); + ASSERT_TRUE(builder.WriteEndArray()); + + ASSERT_TRUE(builder.WriteStartObject()); + ASSERT_TRUE(builder.WriteKeyValue("b", "c")); + ASSERT_TRUE(builder.WriteEndObject()); + + ASSERT_TRUE(builder.WriteEndArray()); + + ASSERT_TRUE(*parsedArray == builder.GetJSONDocument()); +} + +TEST(JSONDocumentTest, OwnershipTest) { + std::unique_ptr parsed( + JSONDocument::ParseJSON(kSampleJSON.c_str())); + ASSERT_TRUE(parsed != nullptr); + ASSERT_TRUE(parsed->IsOwner()); + + // Copy constructor from owner -> owner + JSONDocument copy_constructor(*parsed); + ASSERT_TRUE(copy_constructor.IsOwner()); + + // Copy constructor from non-owner -> non-owner + JSONDocument non_owner((*parsed)["properties"]); + ASSERT_TRUE(!non_owner.IsOwner()); + + // Move constructor from owner -> owner + JSONDocument moved_from_owner(std::move(copy_constructor)); + ASSERT_TRUE(moved_from_owner.IsOwner()); + + // Move constructor from non-owner -> non-owner + JSONDocument moved_from_non_owner(std::move(non_owner)); + ASSERT_TRUE(!moved_from_non_owner.IsOwner()); } } // namespace rocksdb