diff --git a/HISTORY.md b/HISTORY.md index c909f0b9ef..6046bb1e78 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,7 @@ * Add support for trace filtering. ### Public API Change +* Remove bundled fbson library. * statistics.stats_level_ becomes atomic. It is preferred to use statistics.set_stats_level() and statistics.get_stats_level() to access it. ### Bug Fixes diff --git a/third-party/fbson/COMMIT.md b/third-party/fbson/COMMIT.md deleted file mode 100644 index b38b5424d3..0000000000 --- a/third-party/fbson/COMMIT.md +++ /dev/null @@ -1,5 +0,0 @@ -fbson commit: -https://github.com/facebook/mysql-5.6/commit/55ef9ff25c934659a70b4094e9b406c48e9dd43d - -# TODO. -* Had to convert zero sized array to [1] sized arrays due to the fact that MS Compiler complains about it not being standard. At some point need to contribute this change back to MySql where this code was taken from. diff --git a/third-party/fbson/FbsonDocument.h b/third-party/fbson/FbsonDocument.h deleted file mode 100644 index c69fcb45f5..0000000000 --- a/third-party/fbson/FbsonDocument.h +++ /dev/null @@ -1,890 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -/* - * This header defines FbsonDocument, FbsonKeyValue, and various value classes - * which are derived from FbsonValue, and a forward iterator for container - * values - essentially everything that is related to FBSON binary data - * structures. - * - * Implementation notes: - * - * None of the classes in this header file can be instantiated directly (i.e. - * you cannot create a FbsonKeyValue or FbsonValue object - all constructors - * are declared non-public). We use the classes as wrappers on the packed FBSON - * bytes (serialized), and cast the classes (types) to the underlying packed - * byte array. - * - * For the same reason, we cannot define any FBSON value class to be virtual, - * since we never call constructors, and will not instantiate vtbl and vptrs. - * - * Therefore, the classes are defined as packed structures (i.e. no data - * alignment and padding), and the private member variables of the classes are - * defined precisely in the same order as the FBSON spec. This ensures we - * access the packed FBSON bytes correctly. - * - * The packed structures are highly optimized for in-place operations with low - * overhead. The reads (and in-place writes) are performed directly on packed - * bytes. There is no memory allocation at all at runtime. - * - * For updates/writes of values that will expand the original FBSON size, the - * write will fail, and the caller needs to handle buffer increase. - * - * ** Iterator ** - * Both ObjectVal class and ArrayVal class have iterator type that you can use - * to declare an iterator on a container object to go through the key-value - * pairs or value list. The iterator has both non-const and const types. - * - * Note: iterators are forward direction only. - * - * ** Query ** - * Querying into containers is through the member functions find (for key/value - * pairs) and get (for array elements), and is in streaming style. We don't - * need to read/scan the whole FBSON packed bytes in order to return results. - * Once the key/index is found, we will stop search. You can use text to query - * both objects and array (for array, text will be converted to integer index), - * and use index to retrieve from array. Array index is 0-based. - * - * ** External dictionary ** - * During query processing, you can also pass a callback function, so the - * search will first try to check if the key string exists in the dictionary. - * If so, search will be based on the id instead of the key string. - * - * @author Tian Xia - */ - -#pragma once - -#include -#include -#include - -namespace fbson { - -#pragma pack(push, 1) - -#define FBSON_VER 1 - -// forward declaration -class FbsonValue; -class ObjectVal; - -/* - * FbsonDocument is the main object that accesses and queries FBSON packed - * bytes. NOTE: FbsonDocument only allows object container as the top level - * FBSON value. However, you can use the static method "createValue" to get any - * FbsonValue object from the packed bytes. - * - * FbsonDocument object also dereferences to an object container value - * (ObjectVal) once FBSON is loaded. - * - * ** Load ** - * FbsonDocument is usable after loading packed bytes (memory location) into - * the object. We only need the header and first few bytes of the payload after - * header to verify the FBSON. - * - * Note: creating an FbsonDocument (through createDocument) does not allocate - * any memory. The document object is an efficient wrapper on the packed bytes - * which is accessed directly. - * - * ** Query ** - * Query is through dereferencing into ObjectVal. - */ -class FbsonDocument { - public: - // create an FbsonDocument object from FBSON packed bytes - static FbsonDocument* createDocument(const char* pb, uint32_t size); - - // create an FbsonValue from FBSON packed bytes - static FbsonValue* createValue(const char* pb, uint32_t size); - - uint8_t version() { return header_.ver_; } - - FbsonValue* getValue() { return ((FbsonValue*)payload_); } - - ObjectVal* operator->() { return ((ObjectVal*)payload_); } - - const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); } - - private: - /* - * FbsonHeader class defines FBSON header (internal to FbsonDocument). - * - * Currently it only contains version information (1-byte). We may expand the - * header to include checksum of the FBSON binary for more security. - */ - struct FbsonHeader { - uint8_t ver_; - } header_; - - char payload_[1]; - - FbsonDocument(); - - FbsonDocument(const FbsonDocument&) = delete; - FbsonDocument& operator=(const FbsonDocument&) = delete; -}; - -/* - * FbsonFwdIteratorT implements FBSON's iterator template. - * - * Note: it is an FORWARD iterator only due to the design of FBSON format. - */ -template -class FbsonFwdIteratorT { - typedef Iter_Type iterator; - typedef typename std::iterator_traits::pointer pointer; - typedef typename std::iterator_traits::reference reference; - - public: - explicit FbsonFwdIteratorT(const iterator& i) : current_(i) {} - - // allow non-const to const iterator conversion (same container type) - template - FbsonFwdIteratorT(const FbsonFwdIteratorT& rhs) - : current_(rhs.base()) {} - - bool operator==(const FbsonFwdIteratorT& rhs) const { - return (current_ == rhs.current_); - } - - bool operator!=(const FbsonFwdIteratorT& rhs) const { - return !operator==(rhs); - } - - bool operator<(const FbsonFwdIteratorT& rhs) const { - return (current_ < rhs.current_); - } - - bool operator>(const FbsonFwdIteratorT& rhs) const { return !operator<(rhs); } - - FbsonFwdIteratorT& operator++() { - current_ = (iterator)(((char*)current_) + current_->numPackedBytes()); - return *this; - } - - FbsonFwdIteratorT operator++(int) { - auto tmp = *this; - current_ = (iterator)(((char*)current_) + current_->numPackedBytes()); - return tmp; - } - - explicit operator pointer() { return current_; } - - reference operator*() const { return *current_; } - - pointer operator->() const { return current_; } - - iterator base() const { return current_; } - - private: - iterator current_; -}; - -typedef int (*hDictInsert)(const char* key, unsigned len); -typedef int (*hDictFind)(const char* key, unsigned len); - -/* - * FbsonType defines 10 primitive types and 2 container types, as described - * below. - * - * primitive_value ::= - * 0x00 //null value (0 byte) - * | 0x01 //boolean true (0 byte) - * | 0x02 //boolean false (0 byte) - * | 0x03 int8 //char/int8 (1 byte) - * | 0x04 int16 //int16 (2 bytes) - * | 0x05 int32 //int32 (4 bytes) - * | 0x06 int64 //int64 (8 bytes) - * | 0x07 double //floating point (8 bytes) - * | 0x08 string //variable length string - * | 0x09 binary //variable length binary - * - * container ::= - * 0x0A int32 key_value_list //object, int32 is the total bytes of the object - * | 0x0B int32 value_list //array, int32 is the total bytes of the array - */ -enum class FbsonType : char { - T_Null = 0x00, - T_True = 0x01, - T_False = 0x02, - T_Int8 = 0x03, - T_Int16 = 0x04, - T_Int32 = 0x05, - T_Int64 = 0x06, - T_Double = 0x07, - T_String = 0x08, - T_Binary = 0x09, - T_Object = 0x0A, - T_Array = 0x0B, - NUM_TYPES, -}; - -typedef std::underlying_type::type FbsonTypeUnder; - -/* - * FbsonKeyValue class defines FBSON key type, as described below. - * - * key ::= - * 0x00 int8 //1-byte dictionary id - * | int8 (byte*) //int8 (>0) is the size of the key string - * - * value ::= primitive_value | container - * - * FbsonKeyValue can be either an id mapping to the key string in an external - * dictionary, or it is the original key string. Whether to read an id or a - * string is decided by the first byte (size_). - * - * Note: a key object must be followed by a value object. Therefore, a key - * object implicitly refers to a key-value pair, and you can get the value - * object right after the key object. The function numPackedBytes hence - * indicates the total size of the key-value pair, so that we will be able go - * to next pair from the key. - * - * ** Dictionary size ** - * By default, the dictionary size is 255 (1-byte). Users can define - * "USE_LARGE_DICT" to increase the dictionary size to 655535 (2-byte). - */ -class FbsonKeyValue { - public: -#ifdef USE_LARGE_DICT - static const int sMaxKeyId = 65535; - typedef uint16_t keyid_type; -#else - static const int sMaxKeyId = 255; - typedef uint8_t keyid_type; -#endif // #ifdef USE_LARGE_DICT - - static const uint8_t sMaxKeyLen = 64; - - // size of the key. 0 indicates it is stored as id - uint8_t klen() const { return size_; } - - // get the key string. Note the string may not be null terminated. - const char* getKeyStr() const { return key_.str_; } - - keyid_type getKeyId() const { return key_.id_; } - - unsigned int keyPackedBytes() const { - return size_ ? (sizeof(size_) + size_) - : (sizeof(size_) + sizeof(keyid_type)); - } - - FbsonValue* value() const { - return (FbsonValue*)(((char*)this) + keyPackedBytes()); - } - - // size of the total packed bytes (key+value) - unsigned int numPackedBytes() const; - - private: - uint8_t size_; - - union key_ { - keyid_type id_; - char str_[1]; - } key_; - - FbsonKeyValue(); -}; - -/* - * FbsonValue is the base class of all FBSON types. It contains only one member - * variable - type info, which can be retrieved by member functions is[Type]() - * or type(). - */ -class FbsonValue { - public: - static const uint32_t sMaxValueLen = 1 << 24; // 16M - - bool isNull() const { return (type_ == FbsonType::T_Null); } - bool isTrue() const { return (type_ == FbsonType::T_True); } - bool isFalse() const { return (type_ == FbsonType::T_False); } - bool isInt8() const { return (type_ == FbsonType::T_Int8); } - bool isInt16() const { return (type_ == FbsonType::T_Int16); } - bool isInt32() const { return (type_ == FbsonType::T_Int32); } - bool isInt64() const { return (type_ == FbsonType::T_Int64); } - bool isDouble() const { return (type_ == FbsonType::T_Double); } - bool isString() const { return (type_ == FbsonType::T_String); } - bool isBinary() const { return (type_ == FbsonType::T_Binary); } - bool isObject() const { return (type_ == FbsonType::T_Object); } - bool isArray() const { return (type_ == FbsonType::T_Array); } - - FbsonType type() const { return type_; } - - // size of the total packed bytes - unsigned int numPackedBytes() const; - - // size of the value in bytes - unsigned int size() const; - - // get the raw byte array of the value - const char* getValuePtr() const; - - // find the FBSON value by a key path string (null terminated) - FbsonValue* findPath(const char* key_path, - const char* delim = ".", - hDictFind handler = nullptr) { - return findPath(key_path, (unsigned int)strlen(key_path), delim, handler); - } - - // find the FBSON value by a key path string (with length) - FbsonValue* findPath(const char* key_path, - unsigned int len, - const char* delim, - hDictFind handler); - - protected: - FbsonType type_; // type info - - FbsonValue(); -}; - -/* - * NumerValT is the template class (derived from FbsonValue) of all number - * types (integers and double). - */ -template -class NumberValT : public FbsonValue { - public: - T val() const { return num_; } - - unsigned int numPackedBytes() const { return sizeof(FbsonValue) + sizeof(T); } - - // catch all unknow specialization of the template class - bool setVal(T /*value*/) { return false; } - - private: - T num_; - - NumberValT(); -}; - -typedef NumberValT Int8Val; - -// override setVal for Int8Val -template <> -inline bool Int8Val::setVal(int8_t value) { - if (!isInt8()) { - return false; - } - - num_ = value; - return true; -} - -typedef NumberValT Int16Val; - -// override setVal for Int16Val -template <> -inline bool Int16Val::setVal(int16_t value) { - if (!isInt16()) { - return false; - } - - num_ = value; - return true; -} - -typedef NumberValT Int32Val; - -// override setVal for Int32Val -template <> -inline bool Int32Val::setVal(int32_t value) { - if (!isInt32()) { - return false; - } - - num_ = value; - return true; -} - -typedef NumberValT Int64Val; - -// override setVal for Int64Val -template <> -inline bool Int64Val::setVal(int64_t value) { - if (!isInt64()) { - return false; - } - - num_ = value; - return true; -} - -typedef NumberValT DoubleVal; - -// override setVal for DoubleVal -template <> -inline bool DoubleVal::setVal(double value) { - if (!isDouble()) { - return false; - } - - num_ = value; - return true; -} - -/* - * BlobVal is the base class (derived from FbsonValue) for string and binary - * types. The size_ indicates the total bytes of the payload_. - */ -class BlobVal : public FbsonValue { - public: - // size of the blob payload only - unsigned int getBlobLen() const { return size_; } - - // return the blob as byte array - const char* getBlob() const { return payload_; } - - // size of the total packed bytes - unsigned int numPackedBytes() const { - return sizeof(FbsonValue) + sizeof(size_) + size_; - } - - protected: - uint32_t size_; - char payload_[1]; - - // set new blob bytes - bool internalSetVal(const char* blob, uint32_t blobSize) { - // if we cannot fit the new blob, fail the operation - if (blobSize > size_) { - return false; - } - - memcpy(payload_, blob, blobSize); - - // Set the reset of the bytes to 0. Note we cannot change the size_ of the - // current payload, as all values are packed. - memset(payload_ + blobSize, 0, size_ - blobSize); - - return true; - } - - BlobVal(); - - private: - // Disable as this class can only be allocated dynamically - BlobVal(const BlobVal&) = delete; - BlobVal& operator=(const BlobVal&) = delete; -}; - -/* - * Binary type - */ -class BinaryVal : public BlobVal { - public: - bool setVal(const char* blob, uint32_t blobSize) { - if (!isBinary()) { - return false; - } - - return internalSetVal(blob, blobSize); - } - - private: - BinaryVal(); -}; - -/* - * String type - * Note: FBSON string may not be a c-string (NULL-terminated) - */ -class StringVal : public BlobVal { - public: - bool setVal(const char* str, uint32_t blobSize) { - if (!isString()) { - return false; - } - - return internalSetVal(str, blobSize); - } - - private: - StringVal(); -}; - -/* - * ContainerVal is the base class (derived from FbsonValue) for object and - * array types. The size_ indicates the total bytes of the payload_. - */ -class ContainerVal : public FbsonValue { - public: - // size of the container payload only - unsigned int getContainerSize() const { return size_; } - - // return the container payload as byte array - const char* getPayload() const { return payload_; } - - // size of the total packed bytes - unsigned int numPackedBytes() const { - return sizeof(FbsonValue) + sizeof(size_) + size_; - } - - protected: - uint32_t size_; - char payload_[1]; - - ContainerVal(); - - ContainerVal(const ContainerVal&) = delete; - ContainerVal& operator=(const ContainerVal&) = delete; -}; - -/* - * Object type - */ -class ObjectVal : public ContainerVal { - public: - // find the FBSON value by a key string (null terminated) - FbsonValue* find(const char* key, hDictFind handler = nullptr) const { - if (!key) - return nullptr; - - return find(key, (unsigned int)strlen(key), handler); - } - - // find the FBSON value by a key string (with length) - FbsonValue* find(const char* key, - unsigned int klen, - hDictFind handler = nullptr) const { - if (!key || !klen) - return nullptr; - - int key_id = -1; - if (handler && (key_id = handler(key, klen)) >= 0) { - return find(key_id); - } - - return internalFind(key, klen); - } - - // find the FBSON value by a key dictionary ID - FbsonValue* find(int key_id) const { - if (key_id < 0 || key_id > FbsonKeyValue::sMaxKeyId) - return nullptr; - - const char* pch = payload_; - const char* fence = payload_ + size_; - - while (pch < fence) { - FbsonKeyValue* pkey = (FbsonKeyValue*)(pch); - if (!pkey->klen() && key_id == pkey->getKeyId()) { - return pkey->value(); - } - pch += pkey->numPackedBytes(); - } - - assert(pch == fence); - - return nullptr; - } - - typedef FbsonKeyValue value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef FbsonFwdIteratorT iterator; - typedef FbsonFwdIteratorT const_iterator; - - iterator begin() { return iterator((pointer)payload_); } - - const_iterator begin() const { return const_iterator((pointer)payload_); } - - iterator end() { return iterator((pointer)(payload_ + size_)); } - - const_iterator end() const { - return const_iterator((pointer)(payload_ + size_)); - } - - private: - FbsonValue* internalFind(const char* key, unsigned int klen) const { - const char* pch = payload_; - const char* fence = payload_ + size_; - - while (pch < fence) { - FbsonKeyValue* pkey = (FbsonKeyValue*)(pch); - if (klen == pkey->klen() && strncmp(key, pkey->getKeyStr(), klen) == 0) { - return pkey->value(); - } - pch += pkey->numPackedBytes(); - } - - assert(pch == fence); - - return nullptr; - } - - private: - ObjectVal(); -}; - -/* - * Array type - */ -class ArrayVal : public ContainerVal { - public: - // get the FBSON value at index - FbsonValue* get(int idx) const { - if (idx < 0) - return nullptr; - - const char* pch = payload_; - const char* fence = payload_ + size_; - - while (pch < fence && idx-- > 0) - pch += ((FbsonValue*)pch)->numPackedBytes(); - - if (idx == -1) - return (FbsonValue*)pch; - else { - assert(pch == fence); - return nullptr; - } - } - - // Get number of elements in array - unsigned int numElem() const { - const char* pch = payload_; - const char* fence = payload_ + size_; - - unsigned int num = 0; - while (pch < fence) { - ++num; - pch += ((FbsonValue*)pch)->numPackedBytes(); - } - - assert(pch == fence); - - return num; - } - - typedef FbsonValue value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef FbsonFwdIteratorT iterator; - typedef FbsonFwdIteratorT const_iterator; - - iterator begin() { return iterator((pointer)payload_); } - - const_iterator begin() const { return const_iterator((pointer)payload_); } - - iterator end() { return iterator((pointer)(payload_ + size_)); } - - const_iterator end() const { - return const_iterator((pointer)(payload_ + size_)); - } - - private: - ArrayVal(); -}; - -inline FbsonDocument* FbsonDocument::createDocument(const char* pb, - uint32_t size) { - if (!pb || size < sizeof(FbsonHeader) + sizeof(FbsonValue)) { - return nullptr; - } - - FbsonDocument* doc = (FbsonDocument*)pb; - if (doc->header_.ver_ != FBSON_VER) { - return nullptr; - } - - FbsonValue* val = (FbsonValue*)doc->payload_; - if (!val->isObject() || size != sizeof(FbsonHeader) + val->numPackedBytes()) { - return nullptr; - } - - return doc; -} - -inline FbsonValue* FbsonDocument::createValue(const char* pb, uint32_t size) { - if (!pb || size < sizeof(FbsonHeader) + sizeof(FbsonValue)) { - return nullptr; - } - - FbsonDocument* doc = (FbsonDocument*)pb; - if (doc->header_.ver_ != FBSON_VER) { - return nullptr; - } - - FbsonValue* val = (FbsonValue*)doc->payload_; - if (size != sizeof(FbsonHeader) + val->numPackedBytes()) { - return nullptr; - } - - return val; -} - -inline unsigned int FbsonKeyValue::numPackedBytes() const { - unsigned int ks = keyPackedBytes(); - FbsonValue* val = (FbsonValue*)(((char*)this) + ks); - return ks + val->numPackedBytes(); -} - -// Poor man's "virtual" function FbsonValue::numPackedBytes -inline unsigned int FbsonValue::numPackedBytes() const { - switch (type_) { - case FbsonType::T_Null: - case FbsonType::T_True: - case FbsonType::T_False: { - return sizeof(type_); - } - - case FbsonType::T_Int8: { - return sizeof(type_) + sizeof(int8_t); - } - case FbsonType::T_Int16: { - return sizeof(type_) + sizeof(int16_t); - } - case FbsonType::T_Int32: { - return sizeof(type_) + sizeof(int32_t); - } - case FbsonType::T_Int64: { - return sizeof(type_) + sizeof(int64_t); - } - case FbsonType::T_Double: { - return sizeof(type_) + sizeof(double); - } - case FbsonType::T_String: - case FbsonType::T_Binary: { - return ((BlobVal*)(this))->numPackedBytes(); - } - - case FbsonType::T_Object: - case FbsonType::T_Array: { - return ((ContainerVal*)(this))->numPackedBytes(); - } - default: - return 0; - } -} - -inline unsigned int FbsonValue::size() const { - switch (type_) { - case FbsonType::T_Int8: { - return sizeof(int8_t); - } - case FbsonType::T_Int16: { - return sizeof(int16_t); - } - case FbsonType::T_Int32: { - return sizeof(int32_t); - } - case FbsonType::T_Int64: { - return sizeof(int64_t); - } - case FbsonType::T_Double: { - return sizeof(double); - } - case FbsonType::T_String: - case FbsonType::T_Binary: { - return ((BlobVal*)(this))->getBlobLen(); - } - - case FbsonType::T_Object: - case FbsonType::T_Array: { - return ((ContainerVal*)(this))->getContainerSize(); - } - case FbsonType::T_Null: - case FbsonType::T_True: - case FbsonType::T_False: - default: - return 0; - } -} - -inline const char* FbsonValue::getValuePtr() const { - switch (type_) { - case FbsonType::T_Int8: - case FbsonType::T_Int16: - case FbsonType::T_Int32: - case FbsonType::T_Int64: - case FbsonType::T_Double: - return ((char*)this) + sizeof(FbsonType); - - case FbsonType::T_String: - case FbsonType::T_Binary: - return ((BlobVal*)(this))->getBlob(); - - case FbsonType::T_Object: - case FbsonType::T_Array: - return ((ContainerVal*)(this))->getPayload(); - - case FbsonType::T_Null: - case FbsonType::T_True: - case FbsonType::T_False: - default: - return nullptr; - } -} - -inline FbsonValue* FbsonValue::findPath(const char* key_path, - unsigned int kp_len, - const char* delim = ".", - hDictFind handler = nullptr) { - if (!key_path || !kp_len) - return nullptr; - - if (!delim) - delim = "."; // default delimiter - - FbsonValue* pval = this; - const char* fence = key_path + kp_len; - char idx_buf[21]; // buffer to parse array index (integer value) - - while (pval && key_path < fence) { - const char* key = key_path; - unsigned int klen = 0; - // find the current key - for (; key_path != fence && *key_path != *delim; ++key_path, ++klen) - ; - - if (!klen) - return nullptr; - - switch (pval->type_) { - case FbsonType::T_Object: { - pval = ((ObjectVal*)pval)->find(key, klen, handler); - break; - } - - case FbsonType::T_Array: { - // parse string into an integer (array index) - if (klen >= sizeof(idx_buf)) - return nullptr; - - memcpy(idx_buf, key, klen); - idx_buf[klen] = 0; - - char* end = nullptr; - int index = (int)strtol(idx_buf, &end, 10); - if (end && !*end) - pval = ((fbson::ArrayVal*)pval)->get(index); - else - // incorrect index string - return nullptr; - break; - } - - default: - return nullptr; - } - - // skip the delimiter - if (key_path < fence) { - ++key_path; - if (key_path == fence) - // we have a trailing delimiter at the end - return nullptr; - } - } - - return pval; -} - -#pragma pack(pop) - -} // namespace fbson diff --git a/third-party/fbson/FbsonJsonParser.h b/third-party/fbson/FbsonJsonParser.h deleted file mode 100644 index f4b8ed2515..0000000000 --- a/third-party/fbson/FbsonJsonParser.h +++ /dev/null @@ -1,742 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -/* - * This file defines FbsonJsonParserT (template) and FbsonJsonParser. - * - * FbsonJsonParserT is a template class which implements a JSON parser. - * FbsonJsonParserT parses JSON text, and serialize it to FBSON binary format - * by using FbsonWriterT object. By default, FbsonJsonParserT creates a new - * FbsonWriterT object with an output stream object. However, you can also - * pass in your FbsonWriterT or any stream object that implements some basic - * interface of std::ostream (see FbsonStream.h). - * - * FbsonJsonParser specializes FbsonJsonParserT with FbsonOutStream type (see - * FbsonStream.h). So unless you want to provide own a different output stream - * type, use FbsonJsonParser object. - * - * ** Parsing JSON ** - * FbsonJsonParserT parses JSON string, and directly serializes into FBSON - * packed bytes. There are three ways to parse a JSON string: (1) using - * c-string, (2) using string with len, (3) using std::istream object. You can - * use custome streambuf to redirect output. FbsonOutBuffer is a streambuf used - * internally if the input is raw character buffer. - * - * You can reuse an FbsonJsonParserT object to parse/serialize multiple JSON - * strings, and the previous FBSON will be overwritten. - * - * If parsing fails (returned false), the error code will be set to one of - * FbsonErrType, and can be retrieved by calling getErrorCode(). - * - * ** External dictionary ** - * During parsing a JSON string, you can pass a callback function to map a key - * string to an id, and store the dictionary id in FBSON to save space. The - * purpose of using an external dictionary is more towards a collection of - * documents (which has common keys) rather than a single document, so that - * space saving will be significant. - * - * ** Endianness ** - * Note: FBSON serialization doesn't assume endianness of the server. However - * you will need to ensure that the endianness at the reader side is the same - * as that at the writer side (if they are on different machines). Otherwise, - * proper conversion is needed when a number value is returned to the - * caller/writer. - * - * @author Tian Xia - */ - -#pragma once - -#include -#include -#include "FbsonDocument.h" -#include "FbsonWriter.h" - -namespace fbson { - -const char* const kJsonDelim = " ,]}\t\r\n"; -const char* const kWhiteSpace = " \t\n\r"; - -/* - * Error codes - */ -enum class FbsonErrType { - E_NONE = 0, - E_INVALID_VER, - E_EMPTY_STR, - E_OUTPUT_FAIL, - E_INVALID_DOCU, - E_INVALID_VALUE, - E_INVALID_KEY, - E_INVALID_STR, - E_INVALID_OBJ, - E_INVALID_ARR, - E_INVALID_HEX, - E_INVALID_OCTAL, - E_INVALID_DECIMAL, - E_INVALID_EXPONENT, - E_HEX_OVERFLOW, - E_OCTAL_OVERFLOW, - E_DECIMAL_OVERFLOW, - E_DOUBLE_OVERFLOW, - E_EXPONENT_OVERFLOW, -}; - -/* - * Template FbsonJsonParserT - */ -template -class FbsonJsonParserT { - public: - FbsonJsonParserT() : err_(FbsonErrType::E_NONE) {} - - explicit FbsonJsonParserT(OS_TYPE& os) - : writer_(os), err_(FbsonErrType::E_NONE) {} - - // parse a UTF-8 JSON string - bool parse(const std::string& str, hDictInsert handler = nullptr) { - return parse(str.c_str(), (unsigned int)str.size(), handler); - } - - // parse a UTF-8 JSON c-style string (NULL terminated) - bool parse(const char* c_str, hDictInsert handler = nullptr) { - return parse(c_str, (unsigned int)strlen(c_str), handler); - } - - // parse a UTF-8 JSON string with length - bool parse(const char* pch, unsigned int len, hDictInsert handler = nullptr) { - if (!pch || len == 0) { - err_ = FbsonErrType::E_EMPTY_STR; - return false; - } - - FbsonInBuffer sb(pch, len); - std::istream in(&sb); - return parse(in, handler); - } - - // parse UTF-8 JSON text from an input stream - bool parse(std::istream& in, hDictInsert handler = nullptr) { - bool res = false; - - // reset output stream - writer_.reset(); - - trim(in); - - if (in.peek() == '{') { - in.ignore(); - res = parseObject(in, handler); - } else if (in.peek() == '[') { - in.ignore(); - res = parseArray(in, handler); - } else { - err_ = FbsonErrType::E_INVALID_DOCU; - } - - trim(in); - if (res && !in.eof()) { - err_ = FbsonErrType::E_INVALID_DOCU; - return false; - } - - return res; - } - - FbsonWriterT& getWriter() { return writer_; } - - FbsonErrType getErrorCode() { return err_; } - - // clear error code - void clearErr() { err_ = FbsonErrType::E_NONE; } - - private: - // parse a JSON object (comma-separated list of key-value pairs) - bool parseObject(std::istream& in, hDictInsert handler) { - if (!writer_.writeStartObject()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - trim(in); - - if (in.peek() == '}') { - in.ignore(); - // empty object - if (!writer_.writeEndObject()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - return true; - } - - while (in.good()) { - if (in.get() != '"') { - err_ = FbsonErrType::E_INVALID_KEY; - return false; - } - - if (!parseKVPair(in, handler)) { - return false; - } - - trim(in); - - char ch = in.get(); - if (ch == '}') { - // end of the object - if (!writer_.writeEndObject()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - return true; - } else if (ch != ',') { - err_ = FbsonErrType::E_INVALID_OBJ; - return false; - } - - trim(in); - } - - err_ = FbsonErrType::E_INVALID_OBJ; - return false; - } - - // parse a JSON array (comma-separated list of values) - bool parseArray(std::istream& in, hDictInsert handler) { - if (!writer_.writeStartArray()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - trim(in); - - if (in.peek() == ']') { - in.ignore(); - // empty array - if (!writer_.writeEndArray()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - return true; - } - - while (in.good()) { - if (!parseValue(in, handler)) { - return false; - } - - trim(in); - - char ch = in.get(); - if (ch == ']') { - // end of the array - if (!writer_.writeEndArray()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - return true; - } else if (ch != ',') { - err_ = FbsonErrType::E_INVALID_ARR; - return false; - } - - trim(in); - } - - err_ = FbsonErrType::E_INVALID_ARR; - return false; - } - - // parse a key-value pair, separated by ":" - bool parseKVPair(std::istream& in, hDictInsert handler) { - if (parseKey(in, handler) && parseValue(in, handler)) { - return true; - } - - return false; - } - - // parse a key (must be string) - bool parseKey(std::istream& in, hDictInsert handler) { - char key[FbsonKeyValue::sMaxKeyLen]; - int i = 0; - while (in.good() && in.peek() != '"' && i < FbsonKeyValue::sMaxKeyLen) { - key[i++] = in.get(); - } - - if (!in.good() || in.peek() != '"' || i == 0) { - err_ = FbsonErrType::E_INVALID_KEY; - return false; - } - - in.ignore(); // discard '"' - - int key_id = -1; - if (handler) { - key_id = handler(key, i); - } - - if (key_id < 0) { - writer_.writeKey(key, i); - } else { - writer_.writeKey(key_id); - } - - trim(in); - - if (in.get() != ':') { - err_ = FbsonErrType::E_INVALID_OBJ; - return false; - } - - return true; - } - - // parse a value - bool parseValue(std::istream& in, hDictInsert handler) { - bool res = false; - - trim(in); - - switch (in.peek()) { - case 'N': - case 'n': { - in.ignore(); - res = parseNull(in); - break; - } - case 'T': - case 't': { - in.ignore(); - res = parseTrue(in); - break; - } - case 'F': - case 'f': { - in.ignore(); - res = parseFalse(in); - break; - } - case '"': { - in.ignore(); - res = parseString(in); - break; - } - case '{': { - in.ignore(); - res = parseObject(in, handler); - break; - } - case '[': { - in.ignore(); - res = parseArray(in, handler); - break; - } - default: { - res = parseNumber(in); - break; - } - } - - return res; - } - - // parse NULL value - bool parseNull(std::istream& in) { - if (tolower(in.get()) == 'u' && tolower(in.get()) == 'l' && - tolower(in.get()) == 'l') { - writer_.writeNull(); - return true; - } - - err_ = FbsonErrType::E_INVALID_VALUE; - return false; - } - - // parse TRUE value - bool parseTrue(std::istream& in) { - if (tolower(in.get()) == 'r' && tolower(in.get()) == 'u' && - tolower(in.get()) == 'e') { - writer_.writeBool(true); - return true; - } - - err_ = FbsonErrType::E_INVALID_VALUE; - return false; - } - - // parse FALSE value - bool parseFalse(std::istream& in) { - if (tolower(in.get()) == 'a' && tolower(in.get()) == 'l' && - tolower(in.get()) == 's' && tolower(in.get()) == 'e') { - writer_.writeBool(false); - return true; - } - - err_ = FbsonErrType::E_INVALID_VALUE; - return false; - } - - // parse a string - bool parseString(std::istream& in) { - if (!writer_.writeStartString()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - bool escaped = false; - char buffer[4096]; // write 4KB at a time - int nread = 0; - while (in.good()) { - char ch = in.get(); - if (ch != '"' || escaped) { - buffer[nread++] = ch; - if (nread == 4096) { - // flush buffer - if (!writer_.writeString(buffer, nread)) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - nread = 0; - } - // set/reset escape - if (ch == '\\' || escaped) { - escaped = !escaped; - } - } else { - // write all remaining bytes in the buffer - if (nread > 0) { - if (!writer_.writeString(buffer, nread)) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - } - // end writing string - if (!writer_.writeEndString()) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - return true; - } - } - - err_ = FbsonErrType::E_INVALID_STR; - return false; - } - - // parse a number - // Number format can be hex, octal, or decimal (including float). - // Only decimal can have (+/-) sign prefix. - bool parseNumber(std::istream& in) { - bool ret = false; - switch (in.peek()) { - case '0': { - in.ignore(); - - if (in.peek() == 'x' || in.peek() == 'X') { - in.ignore(); - ret = parseHex(in); - } else if (in.peek() == '.') { - in.ignore(); - ret = parseDouble(in, 0, 0, 1); - } else { - ret = parseOctal(in); - } - - break; - } - case '-': { - in.ignore(); - ret = parseDecimal(in, -1); - break; - } - case '+': - in.ignore(); -#if defined(__clang__) - [[clang::fallthrough]]; -#elif defined(__GNUC__) && __GNUC__ >= 7 - [[gnu::fallthrough]]; -#endif - default: - ret = parseDecimal(in, 1); - break; - } - - return ret; - } - - // parse a number in hex format - bool parseHex(std::istream& in) { - uint64_t val = 0; - int num_digits = 0; - char ch = tolower(in.peek()); - while (in.good() && !strchr(kJsonDelim, ch) && (++num_digits) <= 16) { - if (ch >= '0' && ch <= '9') { - val = (val << 4) + (ch - '0'); - } else if (ch >= 'a' && ch <= 'f') { - val = (val << 4) + (ch - 'a' + 10); - } else { // unrecognized hex digit - err_ = FbsonErrType::E_INVALID_HEX; - return false; - } - - in.ignore(); - ch = tolower(in.peek()); - } - - int size = 0; - if (num_digits <= 2) { - size = writer_.writeInt8((int8_t)val); - } else if (num_digits <= 4) { - size = writer_.writeInt16((int16_t)val); - } else if (num_digits <= 8) { - size = writer_.writeInt32((int32_t)val); - } else if (num_digits <= 16) { - size = writer_.writeInt64(val); - } else { - err_ = FbsonErrType::E_HEX_OVERFLOW; - return false; - } - - if (size == 0) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - return true; - } - - // parse a number in octal format - bool parseOctal(std::istream& in) { - int64_t val = 0; - char ch = in.peek(); - while (in.good() && !strchr(kJsonDelim, ch)) { - if (ch >= '0' && ch <= '7') { - val = val * 8 + (ch - '0'); - } else { - err_ = FbsonErrType::E_INVALID_OCTAL; - return false; - } - - // check if the number overflows - if (val < 0) { - err_ = FbsonErrType::E_OCTAL_OVERFLOW; - return false; - } - - in.ignore(); - ch = in.peek(); - } - - int size = 0; - if (val <= std::numeric_limits::max()) { - size = writer_.writeInt8((int8_t)val); - } else if (val <= std::numeric_limits::max()) { - size = writer_.writeInt16((int16_t)val); - } else if (val <= std::numeric_limits::max()) { - size = writer_.writeInt32((int32_t)val); - } else { // val <= INT64_MAX - size = writer_.writeInt64(val); - } - - if (size == 0) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - return true; - } - - // parse a number in decimal (including float) - bool parseDecimal(std::istream& in, int sign) { - int64_t val = 0; - int precision = 0; - - char ch = 0; - while (in.good() && (ch = in.peek()) == '0') - in.ignore(); - - while (in.good() && !strchr(kJsonDelim, ch)) { - if (ch >= '0' && ch <= '9') { - val = val * 10 + (ch - '0'); - ++precision; - } else if (ch == '.') { - // note we don't pop out '.' - return parseDouble(in, static_cast(val), precision, sign); - } else { - err_ = FbsonErrType::E_INVALID_DECIMAL; - return false; - } - - in.ignore(); - - // if the number overflows int64_t, first parse it as double iff we see a - // decimal point later. Otherwise, will treat it as overflow - if (val < 0 && val > std::numeric_limits::min()) { - return parseDouble(in, static_cast(val), precision, sign); - } - - ch = in.peek(); - } - - if (sign < 0) { - val = -val; - } - - int size = 0; - if (val >= std::numeric_limits::min() && - val <= std::numeric_limits::max()) { - size = writer_.writeInt8((int8_t)val); - } else if (val >= std::numeric_limits::min() && - val <= std::numeric_limits::max()) { - size = writer_.writeInt16((int16_t)val); - } else if (val >= std::numeric_limits::min() && - val <= std::numeric_limits::max()) { - size = writer_.writeInt32((int32_t)val); - } else { // val <= INT64_MAX - size = writer_.writeInt64(val); - } - - if (size == 0) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - return true; - } - - // parse IEEE745 double precision: - // Significand precision length - 15 - // Maximum exponent value - 308 - // - // "If a decimal string with at most 15 significant digits is converted to - // IEEE 754 double precision representation and then converted back to a - // string with the same number of significant digits, then the final string - // should match the original" - bool parseDouble(std::istream& in, double val, int precision, int sign) { - int integ = precision; - int frac = 0; - bool is_frac = false; - - char ch = in.peek(); - if (ch == '.') { - is_frac = true; - in.ignore(); - ch = in.peek(); - } - - int exp = 0; - while (in.good() && !strchr(kJsonDelim, ch)) { - if (ch >= '0' && ch <= '9') { - if (precision < 15) { - val = val * 10 + (ch - '0'); - if (is_frac) { - ++frac; - } else { - ++integ; - } - ++precision; - } else if (!is_frac) { - ++exp; - } - } else if (ch == 'e' || ch == 'E') { - in.ignore(); - int exp2; - if (!parseExponent(in, exp2)) { - return false; - } - - exp += exp2; - // check if exponent overflows - if (exp > 308 || exp < -308) { - err_ = FbsonErrType::E_EXPONENT_OVERFLOW; - return false; - } - - is_frac = true; - break; - } - - in.ignore(); - ch = in.peek(); - } - - if (!is_frac) { - err_ = FbsonErrType::E_DECIMAL_OVERFLOW; - return false; - } - - val *= std::pow(10, exp - frac); - if (std::isnan(val) || std::isinf(val)) { - err_ = FbsonErrType::E_DOUBLE_OVERFLOW; - return false; - } - - if (sign < 0) { - val = -val; - } - - if (writer_.writeDouble(val) == 0) { - err_ = FbsonErrType::E_OUTPUT_FAIL; - return false; - } - - return true; - } - - // parse the exponent part of a double number - bool parseExponent(std::istream& in, int& exp) { - bool neg = false; - - char ch = in.peek(); - if (ch == '+') { - in.ignore(); - ch = in.peek(); - } else if (ch == '-') { - neg = true; - in.ignore(); - ch = in.peek(); - } - - exp = 0; - while (in.good() && !strchr(kJsonDelim, ch)) { - if (ch >= '0' && ch <= '9') { - exp = exp * 10 + (ch - '0'); - } else { - err_ = FbsonErrType::E_INVALID_EXPONENT; - return false; - } - - if (exp > 308) { - err_ = FbsonErrType::E_EXPONENT_OVERFLOW; - return false; - } - - in.ignore(); - ch = in.peek(); - } - - if (neg) { - exp = -exp; - } - - return true; - } - - void trim(std::istream& in) { - while (in.good() && strchr(kWhiteSpace, in.peek())) { - in.ignore(); - } - } - - private: - FbsonWriterT writer_; - FbsonErrType err_; -}; - -typedef FbsonJsonParserT FbsonJsonParser; - -} // namespace fbson diff --git a/third-party/fbson/FbsonStream.h b/third-party/fbson/FbsonStream.h deleted file mode 100644 index b20cb1c3bf..0000000000 --- a/third-party/fbson/FbsonStream.h +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -/* - * This header file defines FbsonInBuffer and FbsonOutStream classes. - * - * ** Input Buffer ** - * FbsonInBuffer is a customer input buffer to wrap raw character buffer. Its - * object instances are used to create std::istream objects interally. - * - * ** Output Stream ** - * FbsonOutStream is a custom output stream classes, to contain the FBSON - * serialized binary. The class is conveniently used to specialize templates of - * FbsonParser and FbsonWriter. - * - * @author Tian Xia - */ - -#pragma once - -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#if defined OS_WIN && !defined snprintf -#define snprintf _snprintf -#endif - -#include -#include - -namespace fbson { - -// lengths includes sign -#define MAX_INT_DIGITS 11 -#define MAX_INT64_DIGITS 20 -#define MAX_DOUBLE_DIGITS 23 // 1(sign)+16(significant)+1(decimal)+5(exponent) - -/* - * FBSON's implementation of input buffer - */ -class FbsonInBuffer : public std::streambuf { - public: - FbsonInBuffer(const char* str, uint32_t len) { - // this is read buffer and the str will not be changed - // so we use const_cast (ugly!) to remove constness - char* pch(const_cast(str)); - setg(pch, pch, pch + len); - } -}; - -/* - * FBSON's implementation of output stream. - * - * This is a wrapper of a char buffer. By default, the buffer capacity is 1024 - * bytes. We will double the buffer if realloc is needed for writes. - */ -class FbsonOutStream : public std::ostream { - public: - explicit FbsonOutStream(uint32_t capacity = 1024) - : std::ostream(nullptr), - head_(nullptr), - size_(0), - capacity_(capacity), - alloc_(true) { - if (capacity_ == 0) { - capacity_ = 1024; - } - - head_ = (char*)malloc(capacity_); - } - - FbsonOutStream(char* buffer, uint32_t capacity) - : std::ostream(nullptr), - head_(buffer), - size_(0), - capacity_(capacity), - alloc_(false) { - assert(buffer && capacity_ > 0); - } - - ~FbsonOutStream() { - if (alloc_) { - free(head_); - } - } - - void put(char c) { write(&c, 1); } - - void write(const char* c_str) { write(c_str, (uint32_t)strlen(c_str)); } - - void write(const char* bytes, uint32_t len) { - if (len == 0) - return; - - if (size_ + len > capacity_) { - realloc(len); - } - - memcpy(head_ + size_, bytes, len); - size_ += len; - } - - // write the integer to string - void write(int i) { - // snprintf automatically adds a NULL, so we need one more char - if (size_ + MAX_INT_DIGITS + 1 > capacity_) { - realloc(MAX_INT_DIGITS + 1); - } - - int len = snprintf(head_ + size_, MAX_INT_DIGITS + 1, "%d", i); - assert(len > 0); - size_ += len; - } - - // write the 64bit integer to string - void write(int64_t l) { - // snprintf automatically adds a NULL, so we need one more char - if (size_ + MAX_INT64_DIGITS + 1 > capacity_) { - realloc(MAX_INT64_DIGITS + 1); - } - - int len = snprintf(head_ + size_, MAX_INT64_DIGITS + 1, "%" PRIi64, l); - assert(len > 0); - size_ += len; - } - - // write the double to string - void write(double d) { - // snprintf automatically adds a NULL, so we need one more char - if (size_ + MAX_DOUBLE_DIGITS + 1 > capacity_) { - realloc(MAX_DOUBLE_DIGITS + 1); - } - - int len = snprintf(head_ + size_, MAX_DOUBLE_DIGITS + 1, "%.15g", d); - assert(len > 0); - size_ += len; - } - - pos_type tellp() const { return size_; } - - void seekp(pos_type pos) { size_ = (uint32_t)pos; } - - const char* getBuffer() const { return head_; } - - pos_type getSize() const { return tellp(); } - - private: - void realloc(uint32_t len) { - assert(capacity_ > 0); - - capacity_ *= 2; - while (capacity_ < size_ + len) { - capacity_ *= 2; - } - - if (alloc_) { - char* new_buf = (char*)::realloc(head_, capacity_); - assert(new_buf); - head_ = new_buf; - } else { - char* new_buf = (char*)::malloc(capacity_); - assert(new_buf); - memcpy(new_buf, head_, size_); - head_ = new_buf; - alloc_ = true; - } - } - - private: - char* head_; - uint32_t size_; - uint32_t capacity_; - bool alloc_; -}; - -} // namespace fbson diff --git a/third-party/fbson/FbsonUtil.h b/third-party/fbson/FbsonUtil.h deleted file mode 100644 index 70ac6cb2ba..0000000000 --- a/third-party/fbson/FbsonUtil.h +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -/* - * This header file defines miscellaneous utility classes. - * - * @author Tian Xia - */ - -#pragma once - -#include -#include "FbsonDocument.h" - -namespace fbson { - -#define OUT_BUF_SIZE 1024 - -/* - * FbsonToJson converts an FbsonValue object to a JSON string. - */ -class FbsonToJson { - public: - FbsonToJson() : os_(buffer_, OUT_BUF_SIZE) {} - - // get json string - const char* json(const FbsonValue* pval) { - os_.clear(); - os_.seekp(0); - - if (pval) { - intern_json(pval); - } - - os_.put(0); - return os_.getBuffer(); - } - - private: - // recursively convert FbsonValue - void intern_json(const FbsonValue* val) { - switch (val->type()) { - case FbsonType::T_Null: { - os_.write("null", 4); - break; - } - case FbsonType::T_True: { - os_.write("true", 4); - break; - } - case FbsonType::T_False: { - os_.write("false", 5); - break; - } - case FbsonType::T_Int8: { - os_.write(((Int8Val*)val)->val()); - break; - } - case FbsonType::T_Int16: { - os_.write(((Int16Val*)val)->val()); - break; - } - case FbsonType::T_Int32: { - os_.write(((Int32Val*)val)->val()); - break; - } - case FbsonType::T_Int64: { - os_.write(((Int64Val*)val)->val()); - break; - } - case FbsonType::T_Double: { - os_.write(((DoubleVal*)val)->val()); - break; - } - case FbsonType::T_String: { - os_.put('"'); - os_.write(((StringVal*)val)->getBlob(), ((StringVal*)val)->getBlobLen()); - os_.put('"'); - break; - } - case FbsonType::T_Binary: { - os_.write("\"", 9); - os_.write(((BinaryVal*)val)->getBlob(), ((BinaryVal*)val)->getBlobLen()); - os_.write("\"", 9); - break; - } - case FbsonType::T_Object: { - object_to_json((ObjectVal*)val); - break; - } - case FbsonType::T_Array: { - array_to_json((ArrayVal*)val); - break; - } - default: - break; - } - } - - // convert object - void object_to_json(const ObjectVal* val) { - os_.put('{'); - - auto iter = val->begin(); - auto iter_fence = val->end(); - - while (iter < iter_fence) { - // write key - if (iter->klen()) { - os_.put('"'); - os_.write(iter->getKeyStr(), iter->klen()); - os_.put('"'); - } else { - os_.write(iter->getKeyId()); - } - os_.put(':'); - - // convert value - intern_json(iter->value()); - - ++iter; - if (iter != iter_fence) { - os_.put(','); - } - } - - assert(iter == iter_fence); - - os_.put('}'); - } - - // convert array to json - void array_to_json(const ArrayVal* val) { - os_.put('['); - - auto iter = val->begin(); - auto iter_fence = val->end(); - - while (iter != iter_fence) { - // convert value - intern_json((const FbsonValue*)iter); - ++iter; - if (iter != iter_fence) { - os_.put(','); - } - } - - assert(iter == iter_fence); - - os_.put(']'); - } - - private: - FbsonOutStream os_; - char buffer_[OUT_BUF_SIZE]; -}; - -} // namespace fbson diff --git a/third-party/fbson/FbsonWriter.h b/third-party/fbson/FbsonWriter.h deleted file mode 100644 index e5010fade2..0000000000 --- a/third-party/fbson/FbsonWriter.h +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -/* - * This file defines FbsonWriterT (template) and FbsonWriter. - * - * FbsonWriterT is a template class which implements an FBSON serializer. - * Users call various write functions of FbsonWriterT object to write values - * directly to FBSON packed bytes. All write functions of value or key return - * the number of bytes written to FBSON, or 0 if there is an error. To write an - * object, an array, or a string, you must call writeStart[..] before writing - * values or key, and call writeEnd[..] after finishing at the end. - * - * By default, an FbsonWriterT object creates an output stream buffer. - * Alternatively, you can also pass any output stream object to a writer, as - * long as the stream object implements some basic functions of std::ostream - * (such as FbsonOutStream, see FbsonStream.h). - * - * FbsonWriter specializes FbsonWriterT with FbsonOutStream type (see - * FbsonStream.h). So unless you want to provide own a different output stream - * type, use FbsonParser object. - * - * @author Tian Xia - */ - -#pragma once - -#include -#include "FbsonDocument.h" -#include "FbsonStream.h" - -// conversion' conversion from 'type1' to 'type2', possible loss of data -// Can not restore at the header end as the warnings are emitted at the point of -// template instantiation -#if defined(_MSC_VER) -#pragma warning(disable : 4244) -#endif - -namespace fbson { - -template -class FbsonWriterT { - public: - FbsonWriterT() - : alloc_(true), hasHdr_(false), kvState_(WS_Value), str_pos_(0) { - os_ = new OS_TYPE(); - } - - explicit FbsonWriterT(OS_TYPE& os) - : os_(&os), - alloc_(false), - hasHdr_(false), - kvState_(WS_Value), - str_pos_(0) {} - - ~FbsonWriterT() { - if (alloc_) { - delete os_; - } - } - - void reset() { - os_->clear(); - os_->seekp(0); - hasHdr_ = false; - kvState_ = WS_Value; - for (; !stack_.empty(); stack_.pop()) - ; - } - - // write a key string (or key id if an external dict is provided) - uint32_t writeKey(const char* key, - uint8_t len, - hDictInsert handler = nullptr) { - if (len && !stack_.empty() && verifyKeyState()) { - int key_id = -1; - if (handler) { - key_id = handler(key, len); - } - - uint32_t size = sizeof(uint8_t); - if (key_id < 0) { - os_->put(len); - os_->write(key, len); - size += len; - } else if (key_id <= FbsonKeyValue::sMaxKeyId) { - FbsonKeyValue::keyid_type idx = key_id; - os_->put(0); - os_->write((char*)&idx, sizeof(FbsonKeyValue::keyid_type)); - size += sizeof(FbsonKeyValue::keyid_type); - } else { // key id overflow - assert(0); - return 0; - } - - kvState_ = WS_Key; - return size; - } - - return 0; - } - - // write a key id - uint32_t writeKey(FbsonKeyValue::keyid_type idx) { - if (!stack_.empty() && verifyKeyState()) { - os_->put(0); - os_->write((char*)&idx, sizeof(FbsonKeyValue::keyid_type)); - kvState_ = WS_Key; - return sizeof(uint8_t) + sizeof(FbsonKeyValue::keyid_type); - } - - return 0; - } - - uint32_t writeNull() { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Null); - kvState_ = WS_Value; - return sizeof(FbsonValue); - } - - return 0; - } - - uint32_t writeBool(bool b) { - if (!stack_.empty() && verifyValueState()) { - if (b) { - os_->put((FbsonTypeUnder)FbsonType::T_True); - } else { - os_->put((FbsonTypeUnder)FbsonType::T_False); - } - - kvState_ = WS_Value; - return sizeof(FbsonValue); - } - - return 0; - } - - uint32_t writeInt8(int8_t v) { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Int8); - os_->put(v); - kvState_ = WS_Value; - return sizeof(Int8Val); - } - - return 0; - } - - uint32_t writeInt16(int16_t v) { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Int16); - os_->write((char*)&v, sizeof(int16_t)); - kvState_ = WS_Value; - return sizeof(Int16Val); - } - - return 0; - } - - uint32_t writeInt32(int32_t v) { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Int32); - os_->write((char*)&v, sizeof(int32_t)); - kvState_ = WS_Value; - return sizeof(Int32Val); - } - - return 0; - } - - uint32_t writeInt64(int64_t v) { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Int64); - os_->write((char*)&v, sizeof(int64_t)); - kvState_ = WS_Value; - return sizeof(Int64Val); - } - - return 0; - } - - uint32_t writeDouble(double v) { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Double); - os_->write((char*)&v, sizeof(double)); - kvState_ = WS_Value; - return sizeof(DoubleVal); - } - - return 0; - } - - // must call writeStartString before writing a string val - bool writeStartString() { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_String); - str_pos_ = os_->tellp(); - - // fill the size bytes with 0 for now - uint32_t size = 0; - os_->write((char*)&size, sizeof(uint32_t)); - - kvState_ = WS_String; - return true; - } - - return false; - } - - // finish writing a string val - bool writeEndString() { - if (kvState_ == WS_String) { - std::streampos cur_pos = os_->tellp(); - int32_t size = (int32_t)(cur_pos - str_pos_ - sizeof(uint32_t)); - assert(size >= 0); - - os_->seekp(str_pos_); - os_->write((char*)&size, sizeof(uint32_t)); - os_->seekp(cur_pos); - - kvState_ = WS_Value; - return true; - } - - return false; - } - - uint32_t writeString(const char* str, uint32_t len) { - if (kvState_ == WS_String) { - os_->write(str, len); - return len; - } - - return 0; - } - - uint32_t writeString(char ch) { - if (kvState_ == WS_String) { - os_->put(ch); - return 1; - } - - return 0; - } - - // must call writeStartBinary before writing a binary val - bool writeStartBinary() { - if (!stack_.empty() && verifyValueState()) { - os_->put((FbsonTypeUnder)FbsonType::T_Binary); - str_pos_ = os_->tellp(); - - // fill the size bytes with 0 for now - uint32_t size = 0; - os_->write((char*)&size, sizeof(uint32_t)); - - kvState_ = WS_Binary; - return true; - } - - return false; - } - - // finish writing a binary val - bool writeEndBinary() { - if (kvState_ == WS_Binary) { - std::streampos cur_pos = os_->tellp(); - int32_t size = (int32_t)(cur_pos - str_pos_ - sizeof(uint32_t)); - assert(size >= 0); - - os_->seekp(str_pos_); - os_->write((char*)&size, sizeof(uint32_t)); - os_->seekp(cur_pos); - - kvState_ = WS_Value; - return true; - } - - return false; - } - - uint32_t writeBinary(const char* bin, uint32_t len) { - if (kvState_ == WS_Binary) { - os_->write(bin, len); - return len; - } - - return 0; - } - - // must call writeStartObject before writing an object val - bool writeStartObject() { - if (stack_.empty() || verifyValueState()) { - if (stack_.empty()) { - // if this is a new FBSON, write the header - if (!hasHdr_) { - writeHeader(); - } else - return false; - } - - os_->put((FbsonTypeUnder)FbsonType::T_Object); - // save the size position - stack_.push(WriteInfo({WS_Object, os_->tellp()})); - - // fill the size bytes with 0 for now - uint32_t size = 0; - os_->write((char*)&size, sizeof(uint32_t)); - - kvState_ = WS_Value; - return true; - } - - return false; - } - - // finish writing an object val - bool writeEndObject() { - if (!stack_.empty() && stack_.top().state == WS_Object && - kvState_ == WS_Value) { - WriteInfo& ci = stack_.top(); - std::streampos cur_pos = os_->tellp(); - int32_t size = (int32_t)(cur_pos - ci.sz_pos - sizeof(uint32_t)); - assert(size >= 0); - - os_->seekp(ci.sz_pos); - os_->write((char*)&size, sizeof(uint32_t)); - os_->seekp(cur_pos); - stack_.pop(); - - return true; - } - - return false; - } - - // must call writeStartArray before writing an array val - bool writeStartArray() { - if (stack_.empty() || verifyValueState()) { - if (stack_.empty()) { - // if this is a new FBSON, write the header - if (!hasHdr_) { - writeHeader(); - } else - return false; - } - - os_->put((FbsonTypeUnder)FbsonType::T_Array); - // save the size position - stack_.push(WriteInfo({WS_Array, os_->tellp()})); - - // fill the size bytes with 0 for now - uint32_t size = 0; - os_->write((char*)&size, sizeof(uint32_t)); - - kvState_ = WS_Value; - return true; - } - - return false; - } - - // finish writing an array val - bool writeEndArray() { - if (!stack_.empty() && stack_.top().state == WS_Array && - kvState_ == WS_Value) { - WriteInfo& ci = stack_.top(); - std::streampos cur_pos = os_->tellp(); - int32_t size = (int32_t)(cur_pos - ci.sz_pos - sizeof(uint32_t)); - assert(size >= 0); - - os_->seekp(ci.sz_pos); - os_->write((char*)&size, sizeof(uint32_t)); - os_->seekp(cur_pos); - stack_.pop(); - - return true; - } - - return false; - } - - OS_TYPE* getOutput() { return os_; } - - private: - // verify we are in the right state before writing a value - bool verifyValueState() { - assert(!stack_.empty()); - return (stack_.top().state == WS_Object && kvState_ == WS_Key) || - (stack_.top().state == WS_Array && kvState_ == WS_Value); - } - - // verify we are in the right state before writing a key - bool verifyKeyState() { - assert(!stack_.empty()); - return stack_.top().state == WS_Object && kvState_ == WS_Value; - } - - void writeHeader() { - os_->put(FBSON_VER); - hasHdr_ = true; - } - - private: - enum WriteState { - WS_NONE, - WS_Array, - WS_Object, - WS_Key, - WS_Value, - WS_String, - WS_Binary, - }; - - struct WriteInfo { - WriteState state; - std::streampos sz_pos; - }; - - private: - OS_TYPE* os_; - bool alloc_; - bool hasHdr_; - WriteState kvState_; // key or value state - std::streampos str_pos_; - std::stack stack_; -}; - -typedef FbsonWriterT FbsonWriter; - -} // namespace fbson