From 1542f34f0e0fc53324f6fdc5905f4b77b252a789 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 26 Nov 2018 09:06:31 +0100 Subject: update nlohmann modernjson to v3.4 (with bson support) --- .../modernjson/detail/conversions/from_json.hpp | 56 +- .../lib/modernjson/detail/conversions/to_chars.hpp | 19 +- .../lib/modernjson/detail/conversions/to_json.hpp | 10 +- include/lib/modernjson/detail/exceptions.hpp | 31 +- .../lib/modernjson/detail/input/binary_reader.hpp | 1431 ++++++++++++-------- .../lib/modernjson/detail/input/input_adapters.hpp | 169 ++- include/lib/modernjson/detail/input/json_sax.hpp | 105 +- include/lib/modernjson/detail/input/lexer.hpp | 237 +++- include/lib/modernjson/detail/input/parser.hpp | 60 +- include/lib/modernjson/detail/input/position_t.hpp | 27 + .../detail/iterators/internal_iterator.hpp | 4 +- .../lib/modernjson/detail/iterators/iter_impl.hpp | 4 +- .../detail/iterators/iteration_proxy.hpp | 7 +- .../detail/iterators/json_reverse_iterator.hpp | 4 +- .../detail/iterators/primitive_iterator.hpp | 4 +- include/lib/modernjson/detail/json_pointer.hpp | 12 +- include/lib/modernjson/detail/json_ref.hpp | 18 +- include/lib/modernjson/detail/macro_scope.hpp | 33 +- include/lib/modernjson/detail/meta/cpp_future.hpp | 4 +- include/lib/modernjson/detail/meta/detected.hpp | 4 +- include/lib/modernjson/detail/meta/is_sax.hpp | 4 +- include/lib/modernjson/detail/meta/type_traits.hpp | 196 ++- include/lib/modernjson/detail/meta/void_t.hpp | 4 +- .../lib/modernjson/detail/output/binary_writer.hpp | 630 +++++++-- .../modernjson/detail/output/output_adapters.hpp | 16 +- .../lib/modernjson/detail/output/serializer.hpp | 150 +- include/lib/modernjson/detail/value_t.hpp | 4 +- 27 files changed, 2217 insertions(+), 1026 deletions(-) create mode 100644 include/lib/modernjson/detail/input/position_t.hpp (limited to 'include/lib/modernjson/detail') diff --git a/include/lib/modernjson/detail/conversions/from_json.hpp b/include/lib/modernjson/detail/conversions/from_json.hpp index 8281dda..817d4b7 100644 --- a/include/lib/modernjson/detail/conversions/from_json.hpp +++ b/include/lib/modernjson/detail/conversions/from_json.hpp @@ -84,13 +84,13 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) } template < - typename BasicJsonType, typename CompatibleStringType, + typename BasicJsonType, typename ConstructibleStringType, enable_if_t < - is_compatible_string_type::value and + is_constructible_string_type::value and not std::is_same::value, + ConstructibleStringType>::value, int > = 0 > -void from_json(const BasicJsonType& j, CompatibleStringType& s) +void from_json(const BasicJsonType& j, ConstructibleStringType& s) { if (JSON_UNLIKELY(not j.is_string())) { @@ -173,11 +173,11 @@ auto from_json_array_impl(const BasicJsonType& j, std::array& arr, } } -template -auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/) +template +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) -> decltype( - arr.reserve(std::declval()), - j.template get(), + arr.reserve(std::declval()), + j.template get(), void()) { using std::end; @@ -188,12 +188,12 @@ auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, prio { // get() returns *this, this won't call a from_json // method when value_type is BasicJsonType - return i.template get(); + return i.template get(); }); } -template -void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, +template +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<0> /*unused*/) { using std::end; @@ -204,21 +204,21 @@ void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, { // get() returns *this, this won't call a from_json // method when value_type is BasicJsonType - return i.template get(); + return i.template get(); }); } -template ::value and - not is_compatible_object_type::value and - not is_compatible_string_type::value and - not is_basic_json::value, + is_constructible_array_type::value and + not is_constructible_object_type::value and + not is_constructible_string_type::value and + not is_basic_json::value, int > = 0 > -auto from_json(const BasicJsonType& j, CompatibleArrayType& arr) +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) -> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), -j.template get(), +j.template get(), void()) { if (JSON_UNLIKELY(not j.is_array())) @@ -230,9 +230,9 @@ void()) from_json_array_impl(j, arr, priority_tag<3> {}); } -template::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleObjectType& obj) +template::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) { if (JSON_UNLIKELY(not j.is_object())) { @@ -240,13 +240,13 @@ void from_json(const BasicJsonType& j, CompatibleObjectType& obj) } auto inner_object = j.template get_ptr(); - using value_type = typename CompatibleObjectType::value_type; + using value_type = typename ConstructibleObjectType::value_type; std::transform( inner_object->begin(), inner_object->end(), std::inserter(obj, obj.begin()), [](typename BasicJsonType::object_t::value_type const & p) { - return value_type(p.first, p.second.template get()); + return value_type(p.first, p.second.template get()); }); } @@ -299,7 +299,7 @@ void from_json(const BasicJsonType& j, std::pair& p) } template -void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence) +void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence /*unused*/) { t = std::make_tuple(j.at(Idx).template get::type>()...); } @@ -358,7 +358,7 @@ struct from_json_fn return from_json(j, val); } }; -} +} // namespace detail /// namespace to hold default `from_json` function /// to see why this is required: @@ -366,5 +366,5 @@ struct from_json_fn namespace { constexpr const auto& from_json = detail::static_const::value; -} -} +} // namespace +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/conversions/to_chars.hpp b/include/lib/modernjson/detail/conversions/to_chars.hpp index a13d258..b32e176 100644 --- a/include/lib/modernjson/detail/conversions/to_chars.hpp +++ b/include/lib/modernjson/detail/conversions/to_chars.hpp @@ -47,10 +47,9 @@ struct diyfp // f * 2^e { static constexpr int kPrecision = 64; // = q - uint64_t f; - int e; + uint64_t f = 0; + int e = 0; - constexpr diyfp() noexcept : f(0), e(0) {} constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} /*! @@ -62,7 +61,7 @@ struct diyfp // f * 2^e assert(x.e == y.e); assert(x.f >= y.f); - return diyfp(x.f - y.f, x.e); + return {x.f - y.f, x.e}; } /*! @@ -127,7 +126,7 @@ struct diyfp // f * 2^e const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); - return diyfp(h, x.e + y.e + 64); + return {h, x.e + y.e + 64}; } /*! @@ -158,7 +157,7 @@ struct diyfp // f * 2^e assert(delta >= 0); assert(((x.f << delta) >> delta) == x.f); - return diyfp(x.f << delta, target_exponent); + return {x.f << delta, target_exponent}; } }; @@ -461,7 +460,7 @@ inline cached_power get_cached_power_for_binary_exponent(int e) assert(e >= -1500); assert(e <= 1500); const int f = kAlpha - e - 1; - const int k = (f * 78913) / (1 << 18) + (f > 0); + const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; assert(index >= 0); @@ -609,7 +608,7 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); - uint32_t p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + auto p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e // 1) @@ -928,7 +927,7 @@ inline char* append_exponent(char* buf, int e) *buf++ = '+'; } - uint32_t k = static_cast(e); + auto k = static_cast(e); if (k < 10) { // Always print at least two digits in the exponent. @@ -1046,7 +1045,7 @@ format. Returns an iterator pointing past-the-end of the decimal representation. @note The result is NOT null-terminated. */ template -char* to_chars(char* first, char* last, FloatType value) +char* to_chars(char* first, const char* last, FloatType value) { static_cast(last); // maybe unused - fix warning assert(std::isfinite(value)); diff --git a/include/lib/modernjson/detail/conversions/to_json.hpp b/include/lib/modernjson/detail/conversions/to_json.hpp index 37210ad..5c3669c 100644 --- a/include/lib/modernjson/detail/conversions/to_json.hpp +++ b/include/lib/modernjson/detail/conversions/to_json.hpp @@ -306,13 +306,13 @@ void to_json(BasicJsonType& j, const std::pair& p) // for https://github.com/nlohmann/json/pull/1134 template::iteration_proxy_internal>::value, int> = 0> -void to_json(BasicJsonType& j, T b) noexcept +void to_json(BasicJsonType& j, const T& b) { j = {{b.key(), b.value()}}; } template -void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence) +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence /*unused*/) { j = {std::get(t)...}; } @@ -332,11 +332,11 @@ struct to_json_fn return to_json(j, std::forward(val)); } }; -} +} // namespace detail /// namespace to hold default `to_json` function namespace { constexpr const auto& to_json = detail::static_const::value; -} -} +} // namespace +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/exceptions.hpp b/include/lib/modernjson/detail/exceptions.hpp index b73d7b1..bcb1f2e 100644 --- a/include/lib/modernjson/detail/exceptions.hpp +++ b/include/lib/modernjson/detail/exceptions.hpp @@ -4,6 +4,8 @@ #include // runtime_error #include // to_string +#include + namespace nlohmann { namespace detail @@ -91,6 +93,7 @@ json.exception.parse_error.109 | parse error: array index 'one' is not a number json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). @note For an input with n bytes, 1 is the index of the first character and n+1 is the index of the terminating null byte or the end of file. This also @@ -114,15 +117,23 @@ class parse_error : public exception /*! @brief create a parse error exception @param[in] id_ the id of the exception - @param[in] byte_ the byte index where the error occurred (or 0 if the - position cannot be determined) + @param[in] position the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) @param[in] what_arg the explanatory string @return parse_error object */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) { std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + ": " + what_arg; return parse_error(id_, byte_, w.c_str()); } @@ -141,6 +152,12 @@ class parse_error : public exception private: parse_error(int id_, std::size_t byte_, const char* what_arg) : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } }; /*! @@ -220,6 +237,7 @@ json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | @liveexample{The following code shows how a `type_error` exception can be caught.,type_error} @@ -262,8 +280,9 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON only supports integers numbers up to 9223372036854775807. | +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} @@ -326,5 +345,5 @@ class other_error : public exception private: other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/input/binary_reader.hpp b/include/lib/modernjson/detail/input/binary_reader.hpp index 103cbf7..637569a 100644 --- a/include/lib/modernjson/detail/input/binary_reader.hpp +++ b/include/lib/modernjson/detail/input/binary_reader.hpp @@ -68,6 +68,10 @@ class binary_reader switch (format) { + case input_format_t::bson: + result = parse_bson_internal(); + break; + case input_format_t::cbor: result = parse_cbor_internal(); break; @@ -100,7 +104,8 @@ class binary_reader if (JSON_UNLIKELY(current != std::char_traits::eof())) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, "expected end of input")); + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); } } @@ -120,6 +125,225 @@ class binary_reader } private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size; + get_number(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast(current); + } + + return true; + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + } + + return get_string(input_format_t::bson, len - static_cast(1), result) and get() != std::char_traits::eof(); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const int element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number; + return get_number(input_format_t::bson, number) and sax->number_float(static_cast(number), ""); + } + + case 0x02: // string + { + std::int32_t len; + string_t value; + return get_number(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value; + return get_number(input_format_t::bson, value) and sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value; + return get_number(input_format_t::bson, value) and sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + char cr[3]; + (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast(element_type)); + return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr))); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + while (int element_type = get()) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_UNLIKELY(not get_bson_cstr(key))) + { + return false; + } + + if (not is_array) + { + if (not sax->key(key)) + { + return false; + } + } + + if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size; + get_number(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + /*! @param[in] get_char whether a new character should be retrieved from the input (true, default) or whether the last read @@ -133,7 +357,7 @@ class binary_reader { // EOF case std::char_traits::eof(): - return unexpect_eof(); + return unexpect_eof(input_format_t::cbor, "value"); // Integer 0x00..0x17 (0..23) case 0x00: @@ -165,25 +389,25 @@ class binary_reader case 0x18: // Unsigned integer (one-byte uint8_t follows) { uint8_t number; - return get_number(number) and sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); } case 0x19: // Unsigned integer (two-byte uint16_t follows) { uint16_t number; - return get_number(number) and sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); } case 0x1A: // Unsigned integer (four-byte uint32_t follows) { uint32_t number; - return get_number(number) and sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); } case 0x1B: // Unsigned integer (eight-byte uint64_t follows) { uint64_t number; - return get_number(number) and sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); } // Negative integer -1-0x00..-1-0x17 (-1..-24) @@ -216,25 +440,25 @@ class binary_reader case 0x38: // Negative integer (one-byte uint8_t follows) { uint8_t number; - return get_number(number) and sax->number_integer(static_cast(-1) - number); + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); } case 0x39: // Negative integer -1-n (two-byte uint16_t follows) { uint16_t number; - return get_number(number) and sax->number_integer(static_cast(-1) - number); + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); } case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) { uint32_t number; - return get_number(number) and sax->number_integer(static_cast(-1) - number); + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); } case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) { uint64_t number; - return get_number(number) and sax->number_integer(static_cast(-1) + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - static_cast(number)); } @@ -303,25 +527,25 @@ class binary_reader case 0x98: // array (one-byte uint8_t for n follows) { uint8_t len; - return get_number(len) and get_cbor_array(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { uint16_t len; - return get_number(len) and get_cbor_array(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { uint32_t len; - return get_number(len) and get_cbor_array(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); } case 0x9B: // array (eight-byte uint64_t for n follow) { uint64_t len; - return get_number(len) and get_cbor_array(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); } case 0x9F: // array (indefinite length) @@ -357,25 +581,25 @@ class binary_reader case 0xB8: // map (one-byte uint8_t for n follows) { uint8_t len; - return get_number(len) and get_cbor_object(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { uint16_t len; - return get_number(len) and get_cbor_object(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { uint32_t len; - return get_number(len) and get_cbor_object(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); } case 0xBB: // map (eight-byte uint64_t for n follow) { uint64_t len; - return get_number(len) and get_cbor_object(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); } case 0xBF: // map (indefinite length) @@ -392,17 +616,20 @@ class binary_reader case 0xF9: // Half-Precision Float (two-byte IEEE 754) { - const int byte1 = get(); - if (JSON_UNLIKELY(not unexpect_eof())) + const int byte1_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) { return false; } - const int byte2 = get(); - if (JSON_UNLIKELY(not unexpect_eof())) + const int byte2_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) { return false; } + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + // code from RFC 7049, Appendix D, Figure 3: // As half-precision floating-point numbers were only added // to IEEE 754 in 2008, today's programming platforms often @@ -438,131 +665,44 @@ class binary_reader case 0xFA: // Single-Precision Float (four-byte IEEE 754) { float number; - return get_number(number) and sax->number_float(static_cast(number), ""); + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast(number), ""); } case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { double number; - return get_number(number) and sax->number_float(static_cast(number), ""); + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast(number), ""); } default: // anything else (0xFF is handled inside the other types) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); } } } /*! - @return whether a valid MessagePack value was passed to the SAX parser + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed */ - bool parse_msgpack_internal() + bool get_cbor_string(string_t& result) { - switch (get()) + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) { - // EOF - case std::char_traits::eof(): - return unexpect_eof(); + return false; + } - // positive fixint - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - case 0x38: - case 0x39: - case 0x3A: - case 0x3B: - case 0x3C: - case 0x3D: - case 0x3E: - case 0x3F: - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: - case 0x59: - case 0x5A: - case 0x5B: - case 0x5C: - case 0x5D: - case 0x5E: - case 0x5F: + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) case 0x60: case 0x61: case 0x62: @@ -587,368 +727,254 @@ class binary_reader case 0x75: case 0x76: case 0x77: - case 0x78: - case 0x79: - case 0x7A: - case 0x7B: - case 0x7C: - case 0x7D: - case 0x7E: - case 0x7F: - return sax->number_unsigned(static_cast(current)); + { + return get_string(input_format_t::cbor, current & 0x1F, result); + } - // fixmap - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - return get_msgpack_object(static_cast(current & 0x0F)); - - // fixarray - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - case 0x98: - case 0x99: - case 0x9A: - case 0x9B: - case 0x9C: - case 0x9D: - case 0x9E: - case 0x9F: - return get_msgpack_array(static_cast(current & 0x0F)); - - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - { - string_t s; - return get_msgpack_string(s) and sax->string(s); - } - - case 0xC0: // nil - return sax->null(); - - case 0xC2: // false - return sax->boolean(false); - - case 0xC3: // true - return sax->boolean(true); - - case 0xCA: // float 32 - { - float number; - return get_number(number) and sax->number_float(static_cast(number), ""); - } - - case 0xCB: // float 64 - { - double number; - return get_number(number) and sax->number_float(static_cast(number), ""); - } - - case 0xCC: // uint 8 - { - uint8_t number; - return get_number(number) and sax->number_unsigned(number); - } - - case 0xCD: // uint 16 - { - uint16_t number; - return get_number(number) and sax->number_unsigned(number); - } - - case 0xCE: // uint 32 - { - uint32_t number; - return get_number(number) and sax->number_unsigned(number); - } - - case 0xCF: // uint 64 - { - uint64_t number; - return get_number(number) and sax->number_unsigned(number); - } - - case 0xD0: // int 8 - { - int8_t number; - return get_number(number) and sax->number_integer(number); - } - - case 0xD1: // int 16 - { - int16_t number; - return get_number(number) and sax->number_integer(number); - } - - case 0xD2: // int 32 - { - int32_t number; - return get_number(number) and sax->number_integer(number); - } - - case 0xD3: // int 64 - { - int64_t number; - return get_number(number) and sax->number_integer(number); - } - - case 0xD9: // str 8 - case 0xDA: // str 16 - case 0xDB: // str 32 + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) { - string_t s; - return get_msgpack_string(s) and sax->string(s); + uint8_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); } - case 0xDC: // array 16 + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) { uint16_t len; - return get_number(len) and get_msgpack_array(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); } - case 0xDD: // array 32 + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) { uint32_t len; - return get_number(len) and get_msgpack_array(static_cast(len)); + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); } - case 0xDE: // map 16 + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) { - uint16_t len; - return get_number(len) and get_msgpack_object(static_cast(len)); + uint64_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); } - case 0xDF: // map 32 + case 0x7F: // UTF-8 string (indefinite length) { - uint32_t len; - return get_number(len) and get_msgpack_object(static_cast(len)); + while (get() != 0xFF) + { + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; } - // negative fixint - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - case 0xF0: - case 0xF1: - case 0xF2: - case 0xF3: - case 0xF4: - case 0xF5: - case 0xF6: - case 0xF7: - case 0xF8: - case 0xF9: - case 0xFA: - case 0xFB: - case 0xFC: - case 0xFD: - case 0xFE: - case 0xFF: - return sax->number_integer(static_cast(current)); - - default: // anything else + default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "error reading MessagePack; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); } } } /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead - - @return whether a valid UBJSON value was passed to the SAX parser - */ - bool parse_ubjson_internal(const bool get_char = true) - { - return get_ubjson_value(get_char ? get_ignore_noop() : current); - } - - /*! - @brief get next character from the input - - This function provides the interface to the used input adapter. It does - not throw in case the input reached EOF, but returns a -'ve valued - `std::char_traits::eof()` in that case. - - @return character read from the input - */ - int get() - { - ++chars_read; - return (current = ia->get_character()); - } - - /*! - @return character read from the input after ignoring all 'N' entries + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed */ - int get_ignore_noop() + bool get_cbor_array(const std::size_t len) { - do + if (JSON_UNLIKELY(not sax->start_array(len))) { - get(); + return false; } - while (current == 'N'); - return current; - } - - /* - @brief read a number from the input - - @tparam NumberType the type of the number - @param[out] result number of type @a NumberType - - @return whether conversion completed - - @note This function needs to respect the system's endianess, because - bytes in CBOR, MessagePack, and UBJSON are stored in network order - (big endian) and therefore need reordering on little endian systems. - */ - template - bool get_number(NumberType& result) - { - // step 1: read input into array with system's byte order - std::array vec; - for (std::size_t i = 0; i < sizeof(NumberType); ++i) + if (len != std::size_t(-1)) { - get(); - if (JSON_UNLIKELY(not unexpect_eof())) - { - return false; - } - - // reverse byte order prior to conversion if necessary - if (is_little_endian) + for (std::size_t i = 0; i < len; ++i) { - vec[sizeof(NumberType) - i - 1] = static_cast(current); + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } } - else + } + else + { + while (get() != 0xFF) { - vec[i] = static_cast(current); // LCOV_EXCL_LINE + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } } } - // step 2: convert array into number of type T and return - std::memcpy(&result, vec.data(), sizeof(NumberType)); - return true; + return sax->end_array(); } /*! - @brief create a string by reading characters from the input - - @tparam NumberType the type of the number - @param[in] len number of characters to read - @param[out] string created by reading @a len bytes - - @return whether string creation completed - - @note We can not reserve @a len bytes for the result, because @a len - may be too large. Usually, @ref unexpect_eof() detects the end of - the input before we run out of string memory. + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed */ - template - bool get_string(const NumberType len, string_t& result) + bool get_cbor_object(const std::size_t len) { - bool success = true; - std::generate_n(std::back_inserter(result), len, [this, &success]() + if (not JSON_UNLIKELY(sax->start_object(len))) { - get(); - if (JSON_UNLIKELY(not unexpect_eof())) + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) { - success = false; - } - return static_cast(current); - }); - return success; - } + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } - /*! - @brief reads a CBOR string + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } - @param[out] result created string + return sax->end_object(); + } - @return whether string creation completed + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser */ - bool get_cbor_string(string_t& result) + bool parse_msgpack_internal() { - if (JSON_UNLIKELY(not unexpect_eof())) + switch (get()) { - return false; - } + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); - switch (current) - { - // UTF-8 string (0x00..0x17 bytes follow) + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: case 0x60: case 0x61: case 0x62: @@ -973,138 +999,234 @@ class binary_reader case 0x75: case 0x76: case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast(current & 0x0F)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast(current & 0x0F)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: { - return get_string(current & 0x1F, result); + string_t s; + return get_msgpack_string(s) and sax->string(s); } - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xCA: // float 32 { - uint8_t len; - return get_number(len) and get_string(len, result); + float number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast(number), ""); } - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0xCB: // float 64 { - uint16_t len; - return get_number(len) and get_string(len, result); + double number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast(number), ""); } - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0xCC: // uint 8 { - uint32_t len; - return get_number(len) and get_string(len, result); + uint8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); } - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0xCD: // uint 16 { - uint64_t len; - return get_number(len) and get_string(len, result); + uint16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); } - case 0x7F: // UTF-8 string (indefinite length) + case 0xCE: // uint 32 { - while (get() != 0xFF) - { - string_t chunk; - if (not get_cbor_string(chunk)) - { - return false; - } - result.append(chunk); - } - return true; + uint32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); } - default: + case 0xCF: // uint 64 { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + last_token)); + uint64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); } - } - } - /*! - @param[in] len the length of the array or std::size_t(-1) for an - array of indefinite size - @return whether array creation completed - */ - bool get_cbor_array(const std::size_t len) - { - if (JSON_UNLIKELY(not sax->start_array(len))) - { - return false; - } + case 0xD0: // int 8 + { + int8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } - if (len != std::size_t(-1)) - for (std::size_t i = 0; i < len; ++i) + case 0xD1: // int 16 { - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } + int16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); } - else - { - while (get() != 0xFF) + + case 0xD2: // int 32 { - if (JSON_UNLIKELY(not parse_cbor_internal(false))) - { - return false; - } + int32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); } - } - return sax->end_array(); - } + case 0xD3: // int 64 + { + int64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } - /*! - @param[in] len the length of the object or std::size_t(-1) for an - object of indefinite size - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len) - { - if (not JSON_UNLIKELY(sax->start_object(len))) - { - return false; - } + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } - string_t key; - if (len != std::size_t(-1)) - { - for (std::size_t i = 0; i < len; ++i) + case 0xDC: // array 16 { - get(); - if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) - { - return false; - } + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast(len)); + } - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } - key.clear(); + case 0xDD: // array 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast(len)); } - } - else - { - while (get() != 0xFF) + + case 0xDE: // map 16 { - if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) - { - return false; - } + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast(len)); + } - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } - key.clear(); + case 0xDF: // map 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast(len)); } - } - return sax->end_object(); + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); + } + } } /*! @@ -1119,7 +1241,7 @@ class binary_reader */ bool get_msgpack_string(string_t& result) { - if (JSON_UNLIKELY(not unexpect_eof())) + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) { return false; } @@ -1160,31 +1282,31 @@ class binary_reader case 0xBE: case 0xBF: { - return get_string(current & 0x1F, result); + return get_string(input_format_t::msgpack, current & 0x1F, result); } case 0xD9: // str 8 { uint8_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); } case 0xDA: // str 16 { uint16_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); } case 0xDB: // str 32 { uint32_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); } default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "expected a MessagePack string; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); } } } @@ -1241,6 +1363,22 @@ class binary_reader return sax->end_object(); } + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + /*! @brief reads a UBJSON string @@ -1262,7 +1400,7 @@ class binary_reader get(); // TODO: may we ignore N here? } - if (JSON_UNLIKELY(not unexpect_eof())) + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) { return false; } @@ -1272,36 +1410,36 @@ class binary_reader case 'U': { uint8_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } case 'i': { int8_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } case 'I': { int16_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } case 'l': { int32_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } case 'L': { int64_t len; - return get_number(len) and get_string(len, result); + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } default: auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "expected a UBJSON string; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); } } @@ -1316,7 +1454,7 @@ class binary_reader case 'U': { uint8_t number; - if (JSON_UNLIKELY(not get_number(number))) + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { return false; } @@ -1327,7 +1465,7 @@ class binary_reader case 'i': { int8_t number; - if (JSON_UNLIKELY(not get_number(number))) + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { return false; } @@ -1338,7 +1476,7 @@ class binary_reader case 'I': { int16_t number; - if (JSON_UNLIKELY(not get_number(number))) + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { return false; } @@ -1349,7 +1487,7 @@ class binary_reader case 'l': { int32_t number; - if (JSON_UNLIKELY(not get_number(number))) + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { return false; } @@ -1360,7 +1498,7 @@ class binary_reader case 'L': { int64_t number; - if (JSON_UNLIKELY(not get_number(number))) + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { return false; } @@ -1371,7 +1509,7 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "byte after '#' must denote a number type; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); } } } @@ -1396,7 +1534,7 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (JSON_UNLIKELY(not unexpect_eof())) + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) { return false; } @@ -1404,12 +1542,12 @@ class binary_reader get_ignore_noop(); if (JSON_UNLIKELY(current != '#')) { - if (JSON_UNLIKELY(not unexpect_eof())) + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) { return false; } auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "expected '#' after UBJSON type information; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); } return get_ubjson_size_value(result.first); @@ -1430,7 +1568,7 @@ class binary_reader switch (prefix) { case std::char_traits::eof(): // EOF - return unexpect_eof(); + return unexpect_eof(input_format_t::ubjson, "value"); case 'T': // true return sax->boolean(true); @@ -1443,56 +1581,56 @@ class binary_reader case 'U': { uint8_t number; - return get_number(number) and sax->number_unsigned(number); + return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); } case 'i': { int8_t number; - return get_number(number) and sax->number_integer(number); + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); } case 'I': { int16_t number; - return get_number(number) and sax->number_integer(number); + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); } case 'l': { int32_t number; - return get_number(number) and sax->number_integer(number); + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); } case 'L': { int64_t number; - return get_number(number) and sax->number_integer(number); + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); } case 'd': { float number; - return get_number(number) and sax->number_float(static_cast(number), ""); + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast(number), ""); } case 'D': { double number; - return get_number(number) and sax->number_float(static_cast(number), ""); + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast(number), ""); } case 'C': // char { get(); - if (JSON_UNLIKELY(not unexpect_eof())) + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) { return false; } if (JSON_UNLIKELY(current > 127)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); } string_t s(1, static_cast(current)); return sax->string(s); @@ -1513,7 +1651,7 @@ class binary_reader default: // anything else { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "error reading UBJSON; last byte: 0x" + last_token)); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); } } } @@ -1655,14 +1793,124 @@ class binary_reader return sax->end_object(); } + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits::eof()` in that case. + + @return character read from the input + */ + int get() + { + ++chars_read; + return (current = ia->get_character()); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianess, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array vec; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian && !InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast(current); + } + else + { + vec[i] = static_cast(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success, &format]() + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) + { + success = false; + } + return static_cast(current); + }); + return success; + } + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) @return whether the last read character is not EOF */ - bool unexpect_eof() const + bool unexpect_eof(const input_format_t format, const char* context) const { if (JSON_UNLIKELY(current == std::char_traits::eof())) { - return sax->parse_error(chars_read, "", parse_error::create(110, chars_read, "unexpected end of input")); + return sax->parse_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); } return true; } @@ -1673,10 +1921,49 @@ class binary_reader std::string get_token_string() const { char cr[3]; - snprintf(cr, 3, "%.2hhX", static_cast(current)); + (std::snprintf)(cr, 3, "%.2hhX", static_cast(current)); return std::string{cr}; } + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further contect information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP + } + + return error_msg + " " + context + ": " + detail; + } + private: /// input adapter input_adapter_t ia = nullptr; @@ -1693,5 +1980,5 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/input/input_adapters.hpp b/include/lib/modernjson/detail/input/input_adapters.hpp index 5abaee3..dfb8caf 100644 --- a/include/lib/modernjson/detail/input/input_adapters.hpp +++ b/include/lib/modernjson/detail/input/input_adapters.hpp @@ -18,7 +18,7 @@ namespace nlohmann namespace detail { /// the supported input formats -enum class input_format_t { json, cbor, msgpack, ubjson }; +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; //////////////////// // input adapters // @@ -60,8 +60,8 @@ class input_stream_adapter : public input_adapter_protocol ~input_stream_adapter() override { // clear stream flags; we use underlying streambuf I/O, do not - // maintain ifstream flags - is.clear(); + // maintain ifstream flags, except eof + is.clear(is.rdstate() & std::ios::eofbit); } explicit input_stream_adapter(std::istream& i) @@ -71,13 +71,21 @@ class input_stream_adapter : public input_adapter_protocol // delete because of pointer members input_stream_adapter(const input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter(input_stream_adapter&&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; // std::istream/std::streambuf use std::char_traits::to_int_type, to // ensure that std::char_traits::eof() and the character 0xFF do not // end up as the same value, eg. 0xFFFFFFFF. std::char_traits::int_type get_character() override { - return sb.sbumpc(); + auto res = sb.sbumpc(); + // set eof manually, as we don't use the istream interface. + if (res == EOF) + { + is.clear(is.rdstate() | std::ios::eofbit); + } + return res; } private: @@ -90,13 +98,16 @@ class input_stream_adapter : public input_adapter_protocol class input_buffer_adapter : public input_adapter_protocol { public: - input_buffer_adapter(const char* b, const std::size_t l) + input_buffer_adapter(const char* b, const std::size_t l) noexcept : cursor(b), limit(b + l) {} // delete because of pointer members input_buffer_adapter(const input_buffer_adapter&) = delete; input_buffer_adapter& operator=(input_buffer_adapter&) = delete; + input_buffer_adapter(input_buffer_adapter&&) = delete; + input_buffer_adapter& operator=(input_buffer_adapter&&) = delete; + ~input_buffer_adapter() override = default; std::char_traits::int_type get_character() noexcept override { @@ -115,38 +126,66 @@ class input_buffer_adapter : public input_adapter_protocol const char* const limit; }; -template -class wide_string_input_adapter : public input_adapter_protocol +template +struct wide_string_input_helper { - public: - explicit wide_string_input_adapter(const WideStringType& w) : str(w) {} - - std::char_traits::int_type get_character() noexcept override + // UTF-32 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) { - // check if buffer needs to be filled - if (utf8_bytes_index == utf8_bytes_filled) + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else { - if (sizeof(typename WideStringType::value_type) == 2) + // get the current character + const auto wc = static_cast(str[current_wchar++]); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) { - fill_buffer_utf16(); + utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07); + utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 4; } else { - fill_buffer_utf32(); + // unknown character + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; } - - assert(utf8_bytes_filled > 0); - assert(utf8_bytes_index == 0); } - - // use buffer - assert(utf8_bytes_filled > 0); - assert(utf8_bytes_index < utf8_bytes_filled); - return utf8_bytes[utf8_bytes_index++]; } +}; - private: - void fill_buffer_utf16() +template +struct wide_string_input_helper +{ + // UTF-16 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) { utf8_bytes_index = 0; @@ -158,7 +197,7 @@ class wide_string_input_adapter : public input_adapter_protocol else { // get the current character - const int wc = static_cast(str[current_wchar++]); + const auto wc = static_cast(str[current_wchar++]); // UTF-16 to UTF-8 encoding if (wc < 0x80) @@ -183,7 +222,7 @@ class wide_string_input_adapter : public input_adapter_protocol { if (current_wchar < str.size()) { - const int wc2 = static_cast(str[current_wchar++]); + const auto wc2 = static_cast(str[current_wchar++]); const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); utf8_bytes[0] = 0xf0 | (charcode >> 18); utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); @@ -201,58 +240,40 @@ class wide_string_input_adapter : public input_adapter_protocol } } } +}; - void fill_buffer_utf32() - { - utf8_bytes_index = 0; +template +class wide_string_input_adapter : public input_adapter_protocol +{ + public: + explicit wide_string_input_adapter(const WideStringType& w) noexcept + : str(w) + {} - if (current_wchar == str.size()) - { - utf8_bytes[0] = std::char_traits::eof(); - utf8_bytes_filled = 1; - } - else + std::char_traits::int_type get_character() noexcept override + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) { - // get the current character - const int wc = static_cast(str[current_wchar++]); + fill_buffer(); - // UTF-32 to UTF-8 encoding - if (wc < 0x80) - { - utf8_bytes[0] = wc; - utf8_bytes_filled = 1; - } - else if (wc <= 0x7FF) - { - utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); - utf8_bytes[1] = 0x80 | (wc & 0x3F); - utf8_bytes_filled = 2; - } - else if (wc <= 0xFFFF) - { - utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); - utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[2] = 0x80 | (wc & 0x3F); - utf8_bytes_filled = 3; - } - else if (wc <= 0x10FFFF) - { - utf8_bytes[0] = 0xF0 | ((wc >> 18 ) & 0x07); - utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); - utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[3] = 0x80 | (wc & 0x3F); - utf8_bytes_filled = 4; - } - else - { - // unknown character - utf8_bytes[0] = wc; - utf8_bytes_filled = 1; - } + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index == 0); } + + // use buffer + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; } private: + template + void fill_buffer() + { + wide_string_input_helper::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + /// the wstring to process const WideStringType& str; @@ -373,5 +394,5 @@ class input_adapter /// the actual adapter input_adapter_t ia = nullptr; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/input/json_sax.hpp b/include/lib/modernjson/detail/input/json_sax.hpp index 968fcd0..e1d48a2 100644 --- a/include/lib/modernjson/detail/input/json_sax.hpp +++ b/include/lib/modernjson/detail/input/json_sax.hpp @@ -113,7 +113,7 @@ struct json_sax @brief a parse error occurred @param[in] position the position in the input where the error occurs @param[in] last_token the last read token - @param[in] error_msg a detailed error message + @param[in] ex an exception object describing the error @return whether parsing should proceed (must return false) */ virtual bool parse_error(std::size_t position, @@ -181,7 +181,7 @@ class json_sax_dom_parser return true; } - bool number_float(number_float_t val, const string_t&) + bool number_float(number_float_t val, const string_t& /*unused*/) { handle_value(val); return true; @@ -238,7 +238,7 @@ class json_sax_dom_parser return true; } - bool parse_error(std::size_t, const std::string&, + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& ex) { errored = true; @@ -286,20 +286,19 @@ class json_sax_dom_parser root = BasicJsonType(std::forward(v)); return &root; } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward(v)); + return &(ref_stack.back()->m_value.array->back()); + } else { - assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->emplace_back(std::forward(v)); - return &(ref_stack.back()->m_value.array->back()); - } - else - { - assert(object_element); - *object_element = BasicJsonType(std::forward(v)); - return object_element; - } + assert(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; } } @@ -358,7 +357,7 @@ class json_sax_dom_callback_parser return true; } - bool number_float(number_float_t val, const string_t&) + bool number_float(number_float_t val, const string_t& /*unused*/) { handle_value(val); return true; @@ -496,7 +495,7 @@ class json_sax_dom_callback_parser return true; } - bool parse_error(std::size_t, const std::string&, + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& ex) { errored = true; @@ -574,37 +573,37 @@ class json_sax_dom_callback_parser root = std::move(value); return {true, &root}; } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } else { - // skip this value if we already decided to skip the parent - // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) - if (not ref_stack.back()) - { - return {false, nullptr}; - } + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); - assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); - if (ref_stack.back()->is_array()) + if (not store_element) { - ref_stack.back()->m_value.array->push_back(std::move(value)); - return {true, &(ref_stack.back()->m_value.array->back())}; + return {false, nullptr}; } - else - { - // check if we should store an element for the current key - assert(not key_keep_stack.empty()); - const bool store_element = key_keep_stack.back(); - key_keep_stack.pop_back(); - - if (not store_element) - { - return {false, nullptr}; - } - assert(object_element); - *object_element = std::move(value); - return {true, object_element}; - } + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; } } @@ -642,37 +641,37 @@ class json_sax_acceptor return true; } - bool boolean(bool) + bool boolean(bool /*unused*/) { return true; } - bool number_integer(number_integer_t) + bool number_integer(number_integer_t /*unused*/) { return true; } - bool number_unsigned(number_unsigned_t) + bool number_unsigned(number_unsigned_t /*unused*/) { return true; } - bool number_float(number_float_t, const string_t&) + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) { return true; } - bool string(string_t&) + bool string(string_t& /*unused*/) { return true; } - bool start_object(std::size_t = std::size_t(-1)) + bool start_object(std::size_t /*unused*/ = std::size_t(-1)) { return true; } - bool key(string_t&) + bool key(string_t& /*unused*/) { return true; } @@ -682,7 +681,7 @@ class json_sax_acceptor return true; } - bool start_array(std::size_t = std::size_t(-1)) + bool start_array(std::size_t /*unused*/ = std::size_t(-1)) { return true; } @@ -692,11 +691,11 @@ class json_sax_acceptor return true; } - bool parse_error(std::size_t, const std::string&, const detail::exception&) + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) { return false; } }; -} +} // namespace detail -} +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/input/lexer.hpp b/include/lib/modernjson/detail/input/lexer.hpp index 9606211..b61e289 100644 --- a/include/lib/modernjson/detail/input/lexer.hpp +++ b/include/lib/modernjson/detail/input/lexer.hpp @@ -10,6 +10,7 @@ #include #include +#include namespace nlohmann { @@ -104,7 +105,10 @@ class lexer // delete because of pointer members lexer(const lexer&) = delete; + lexer(lexer&&) = delete; lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = delete; + ~lexer() = default; private: ///////////////////// @@ -393,39 +397,194 @@ class lexer // invalid control characters case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + case 0x1F: { - error_message = "invalid string: control character must be escaped"; + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; return token_type::parse_error; } @@ -709,7 +868,7 @@ class lexer locale's decimal point is used instead of `.` to work with the locale-dependent converters. */ - token_type scan_number() + token_type scan_number() // lgtm [cpp/use-of-goto] { // reset token_buffer to store the number's bytes reset(); @@ -1082,7 +1241,9 @@ scan_number_done: */ std::char_traits::int_type get() { - ++chars_read; + ++position.chars_read_total; + ++position.chars_read_current_line; + if (next_unget) { // just reset the next_unget variable and work with current @@ -1097,6 +1258,13 @@ scan_number_done: { token_string.push_back(std::char_traits::to_char_type(current)); } + + if (current == '\n') + { + ++position.lines_read; + ++position.chars_read_current_line = 0; + } + return current; } @@ -1104,14 +1272,29 @@ scan_number_done: @brief unget current character (read it again on next get) We implement unget by setting variable next_unget to true. The input is not - changed - we just simulate ungetting by modifying chars_read and - token_string. The next call to get() will behave as if the unget character - is read again. + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. */ void unget() { next_unget = true; - --chars_read; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + if (JSON_LIKELY(current != std::char_traits::eof())) { assert(token_string.size() != 0); @@ -1159,9 +1342,9 @@ scan_number_done: ///////////////////// /// return position of last read token - constexpr std::size_t get_position() const noexcept + constexpr position_t get_position() const noexcept { - return chars_read; + return position; } /// return the last read token (for errors only). Will never contain EOF @@ -1177,7 +1360,7 @@ scan_number_done: { // escape control characters char cs[9]; - snprintf(cs, 9, "", static_cast(c)); + (std::snprintf)(cs, 9, "", static_cast(c)); result += cs; } else @@ -1208,30 +1391,20 @@ scan_number_done: { if (get() == 0xEF) { - if (get() == 0xBB and get() == 0xBF) - { - // we completely parsed the BOM - return true; - } - else - { - // after reading 0xEF, an unexpected character followed - return false; - } - } - else - { - // the first character is not the beginning of the BOM; unget it to - // process is later - unget(); - return true; + // check if we completely parse the BOM + return get() == 0xBB and get() == 0xBF; } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; } token_type scan() { // initially, skip the BOM - if (chars_read == 0 and not skip_bom()) + if (position.chars_read_total == 0 and not skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -1309,8 +1482,8 @@ scan_number_done: /// whether the next get() call should just return current bool next_unget = false; - /// the number of characters read - std::size_t chars_read = 0; + /// the start position of the current token + position_t position; /// raw input token string (for error messages) std::vector token_string {}; @@ -1329,5 +1502,5 @@ scan_number_done: /// the decimal point const char decimal_point_char = '.'; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/input/parser.hpp b/include/lib/modernjson/detail/input/parser.hpp index a622077..d205bbb 100644 --- a/include/lib/modernjson/detail/input/parser.hpp +++ b/include/lib/modernjson/detail/input/parser.hpp @@ -91,7 +91,8 @@ class parser { sdp.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); } // in case of an error, return discarded value @@ -119,7 +120,8 @@ class parser { sdp.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); } // in case of an error, return discarded value @@ -154,7 +156,8 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); } return result; @@ -164,7 +167,7 @@ class parser template bool sax_parse_internal(SAX* sax) { - // stack to remember the hieararchy of structured values we are parsing + // stack to remember the hierarchy of structured values we are parsing // true = array; false = object std::vector states; // value to avoid a goto (see comment where set to true) @@ -199,14 +202,12 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); } - else + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) { - if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) - { - return false; - } + return false; } // parse separator (:) @@ -214,7 +215,8 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); } // remember we are now inside an object @@ -328,14 +330,16 @@ class parser // using "uninitialized" to avoid "expected" message return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::uninitialized, "value"))); } default: // the last token was unexpected { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::literal_or_value, "value"))); } } } @@ -347,7 +351,7 @@ class parser // we reached this line after we successfully parsed a value if (states.empty()) { - // empty stack: we reached the end of the hieararchy: done + // empty stack: we reached the end of the hierarchy: done return true; } else @@ -383,7 +387,8 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_array, "array"))); } } else // object @@ -396,7 +401,8 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); } else { @@ -411,7 +417,8 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); } // parse values @@ -440,7 +447,8 @@ class parser { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object))); + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_object, "object"))); } } } @@ -453,9 +461,17 @@ class parser return (last_token = m_lexer.scan()); } - std::string exception_message(const token_type expected) + std::string exception_message(const token_type expected, const std::string& context) { - std::string error_msg = "syntax error - "; + std::string error_msg = "syntax error "; + + if (not context.empty()) + { + error_msg += "while parsing " + context + " "; + } + + error_msg += "- "; + if (last_token == token_type::parse_error) { error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + @@ -484,5 +500,5 @@ class parser /// whether to throw exceptions in case of errors const bool allow_exceptions = true; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/input/position_t.hpp b/include/lib/modernjson/detail/input/position_t.hpp new file mode 100644 index 0000000..37f4ab1 --- /dev/null +++ b/include/lib/modernjson/detail/input/position_t.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} +} diff --git a/include/lib/modernjson/detail/iterators/internal_iterator.hpp b/include/lib/modernjson/detail/iterators/internal_iterator.hpp index 95f056d..e1c12a8 100644 --- a/include/lib/modernjson/detail/iterators/internal_iterator.hpp +++ b/include/lib/modernjson/detail/iterators/internal_iterator.hpp @@ -21,5 +21,5 @@ template struct internal_iterator /// generic iterator for all other types primitive_iterator_t primitive_iterator {}; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/iterators/iter_impl.hpp b/include/lib/modernjson/detail/iterators/iter_impl.hpp index 9e4795a..4ad94db 100644 --- a/include/lib/modernjson/detail/iterators/iter_impl.hpp +++ b/include/lib/modernjson/detail/iterators/iter_impl.hpp @@ -610,5 +610,5 @@ class iter_impl /// the actual iterator of the associated instance internal_iterator::type> m_it; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/iterators/iteration_proxy.hpp b/include/lib/modernjson/detail/iterators/iteration_proxy.hpp index 2da7154..31cc9f0 100644 --- a/include/lib/modernjson/detail/iterators/iteration_proxy.hpp +++ b/include/lib/modernjson/detail/iterators/iteration_proxy.hpp @@ -39,9 +39,6 @@ template class iteration_proxy public: explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {} - iteration_proxy_internal(const iteration_proxy_internal&) = default; - iteration_proxy_internal& operator=(const iteration_proxy_internal&) = default; - /// dereference operator (needed for range-based for) iteration_proxy_internal& operator*() { @@ -124,5 +121,5 @@ template class iteration_proxy return iteration_proxy_internal(container.end()); } }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/iterators/json_reverse_iterator.hpp b/include/lib/modernjson/detail/iterators/json_reverse_iterator.hpp index 2750de4..f3b5b5d 100644 --- a/include/lib/modernjson/detail/iterators/json_reverse_iterator.hpp +++ b/include/lib/modernjson/detail/iterators/json_reverse_iterator.hpp @@ -115,5 +115,5 @@ class json_reverse_iterator : public std::reverse_iterator return it.operator * (); } }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/iterators/primitive_iterator.hpp b/include/lib/modernjson/detail/iterators/primitive_iterator.hpp index db3f897..28d6f1a 100644 --- a/include/lib/modernjson/detail/iterators/primitive_iterator.hpp +++ b/include/lib/modernjson/detail/iterators/primitive_iterator.hpp @@ -116,5 +116,5 @@ class primitive_iterator_t return *this; } }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/json_pointer.hpp b/include/lib/modernjson/detail/json_pointer.hpp index 5689384..a280c9d 100644 --- a/include/lib/modernjson/detail/json_pointer.hpp +++ b/include/lib/modernjson/detail/json_pointer.hpp @@ -59,7 +59,7 @@ class json_pointer @since version 2.0.0 */ - std::string to_string() const noexcept + std::string to_string() const { return std::accumulate(reference_tokens.begin(), reference_tokens.end(), std::string{}, @@ -114,7 +114,7 @@ class json_pointer } /// return whether pointer points to the root document - bool is_root() const + bool is_root() const noexcept { return reference_tokens.empty(); } @@ -506,11 +506,11 @@ class json_pointer std::size_t slash = reference_string.find_first_of('/', 1), // set the beginning of the first reference token start = 1; - // we can stop if start == string::npos+1 = 0 + // we can stop if start == 0 (if slash == std::string::npos) start != 0; // set the beginning of the next reference token // (will eventually be 0 if slash == std::string::npos) - start = slash + 1, + start = (slash == std::string::npos) ? 0 : slash + 1, // find next slash slash = reference_string.find_first_of('/', start)) { @@ -566,7 +566,7 @@ class json_pointer {} } - /// escape "~"" to "~0" and "/" to "~1" + /// escape "~" to "~0" and "/" to "~1" static std::string escape(std::string s) { replace_substring(s, "~", "~0"); @@ -693,4 +693,4 @@ class json_pointer /// the reference tokens std::vector reference_tokens; }; -} +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/json_ref.hpp b/include/lib/modernjson/detail/json_ref.hpp index 8285be2..c44aa66 100644 --- a/include/lib/modernjson/detail/json_ref.hpp +++ b/include/lib/modernjson/detail/json_ref.hpp @@ -3,6 +3,8 @@ #include #include +#include + namespace nlohmann { namespace detail @@ -25,15 +27,19 @@ class json_ref : owned_value(init), value_ref(&owned_value), is_rvalue(true) {} - template - json_ref(Args&& ... args) - : owned_value(std::forward(args)...), value_ref(&owned_value), is_rvalue(true) - {} + template < + class... Args, + enable_if_t::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward(args)...), value_ref(&owned_value), + is_rvalue(true) {} // class should be movable only json_ref(json_ref&&) = default; json_ref(const json_ref&) = delete; json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; value_type moved_or_copied() const { @@ -59,5 +65,5 @@ class json_ref value_type* value_ref = nullptr; const bool is_rvalue; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/macro_scope.hpp b/include/lib/modernjson/detail/macro_scope.hpp index 80fbece..26db667 100644 --- a/include/lib/modernjson/detail/macro_scope.hpp +++ b/include/lib/modernjson/detail/macro_scope.hpp @@ -10,7 +10,7 @@ #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" #endif #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" #endif #endif @@ -87,6 +87,37 @@ #define JSON_HAS_CPP_14 #endif +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + // Ugly macros to avoid uglier copy-paste when specializing basic_json. They // may be removed in the future once the class is split. diff --git a/include/lib/modernjson/detail/meta/cpp_future.hpp b/include/lib/modernjson/detail/meta/cpp_future.hpp index fa7478b..948cd4f 100644 --- a/include/lib/modernjson/detail/meta/cpp_future.hpp +++ b/include/lib/modernjson/detail/meta/cpp_future.hpp @@ -59,5 +59,5 @@ struct static_const template constexpr T static_const::value; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/meta/detected.hpp b/include/lib/modernjson/detail/meta/detected.hpp index 78b25d5..8fb318a 100644 --- a/include/lib/modernjson/detail/meta/detected.hpp +++ b/include/lib/modernjson/detail/meta/detected.hpp @@ -52,5 +52,5 @@ using is_detected_exact = std::is_same>; template class Op, class... Args> using is_detected_convertible = std::is_convertible, To>; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/meta/is_sax.hpp b/include/lib/modernjson/detail/meta/is_sax.hpp index 63f9793..af63386 100644 --- a/include/lib/modernjson/detail/meta/is_sax.hpp +++ b/include/lib/modernjson/detail/meta/is_sax.hpp @@ -137,5 +137,5 @@ public: "Missing/invalid function: bool parse_error(std::size_t, const " "std::string&, const exception&)"); }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/meta/type_traits.hpp b/include/lib/modernjson/detail/meta/type_traits.hpp index e5b6986..2f0af28 100644 --- a/include/lib/modernjson/detail/meta/type_traits.hpp +++ b/include/lib/modernjson/detail/meta/type_traits.hpp @@ -26,6 +26,15 @@ namespace detail // helpers // ///////////// +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + template struct is_basic_json : std::false_type {}; NLOHMANN_BASIC_JSON_TPL_DECLARATION @@ -65,6 +74,55 @@ using to_json_function = decltype(T::to_json(std::declval()...)); template using from_json_function = decltype(T::from_json(std::declval()...)); +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +template +struct has_from_json::value>> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json::value>> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json::value>> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + + /////////////////// // is_ functions // /////////////////// @@ -120,6 +178,30 @@ template struct is_compatible_object_type : is_compatible_object_type_impl {}; +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t::value and + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (std::is_constructible::value and + std::is_same::value) or + (has_from_json::value or + has_non_default_from_json::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + template struct is_compatible_string_type_impl : std::false_type {}; @@ -134,9 +216,28 @@ struct is_compatible_string_type_impl < std::is_constructible::value; }; -template +template struct is_compatible_string_type - : is_compatible_string_type_impl {}; + : is_compatible_string_type_impl {}; + +template +struct is_constructible_string_type_impl : std::false_type {}; + +template +struct is_constructible_string_type_impl < + BasicJsonType, ConstructibleStringType, + enable_if_t::value >> +{ + static constexpr auto value = + std::is_constructible::value; +}; + +template +struct is_constructible_string_type + : is_constructible_string_type_impl {}; template struct is_compatible_array_type_impl : std::false_type {}; @@ -145,18 +246,61 @@ template struct is_compatible_array_type_impl < BasicJsonType, CompatibleArrayType, enable_if_t::value and - is_detected::value >> + is_detected::value and +// This is needed because json_reverse_iterator has a ::iterator type... +// Therefore it is detected as a CompatibleArrayType. +// The real fix would be to have an Iterable concept. + not is_iterator_traits< + std::iterator_traits>::value >> { - // This is needed because json_reverse_iterator has a ::iterator type... - // Therefore it is detected as a CompatibleArrayType. - // The real fix would be to have an Iterable concept. - static constexpr bool value = not is_iterator_traits>::value; + static constexpr bool value = + std::is_constructible::value; }; template struct is_compatible_array_type : is_compatible_array_type_impl {}; +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value and + is_detected::value and + is_detected::value and + is_complete_type< + detected_t>::value >> +{ + static constexpr bool value = + // This is needed because json_reverse_iterator has a ::iterator type, + // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`... + // Therefore it is detected as a ConstructibleArrayType. + // The real fix would be to have an Iterable concept. + not is_iterator_traits < + std::iterator_traits>::value and + + (std::is_same::value or + has_from_json::value or + has_non_default_from_json < + BasicJsonType, typename ConstructibleArrayType::value_type >::value); +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + template struct is_compatible_integer_type_impl : std::false_type {}; @@ -184,40 +328,6 @@ struct is_compatible_integer_type : is_compatible_integer_type_impl {}; -// trait checking if JSONSerializer::from_json(json const&, udt&) exists -template -struct has_from_json -{ - using serializer = typename BasicJsonType::template json_serializer; - - static constexpr bool value = - is_detected_exact::value; -}; - -// This trait checks if JSONSerializer::from_json(json const&) exists -// this overload is used for non-default-constructible user-defined-types -template -struct has_non_default_from_json -{ - using serializer = typename BasicJsonType::template json_serializer; - - static constexpr bool value = - is_detected_exact::value; -}; - -// This trait checks if BasicJsonType::json_serializer::to_json exists -template -struct has_to_json -{ - using serializer = typename BasicJsonType::template json_serializer; - - static constexpr bool value = - is_detected_exact::value; -}; - template struct is_compatible_type_impl: std::false_type {}; @@ -233,5 +343,5 @@ struct is_compatible_type_impl < template struct is_compatible_type : is_compatible_type_impl {}; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/meta/void_t.hpp b/include/lib/modernjson/detail/meta/void_t.hpp index 2528ea8..a256f85 100644 --- a/include/lib/modernjson/detail/meta/void_t.hpp +++ b/include/lib/modernjson/detail/meta/void_t.hpp @@ -9,5 +9,5 @@ template struct make_void using type = void; }; template using void_t = typename make_void::type; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/output/binary_writer.hpp b/include/lib/modernjson/detail/output/binary_writer.hpp index 1726e0c..1020e69 100644 --- a/include/lib/modernjson/detail/output/binary_writer.hpp +++ b/include/lib/modernjson/detail/output/binary_writer.hpp @@ -23,6 +23,8 @@ namespace detail template class binary_writer { + using string_t = typename BasicJsonType::string_t; + public: /*! @brief create a binary writer @@ -35,7 +37,28 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()))); + } + } + } + + /*! + @param[in] j JSON value to serialize */ void write_cbor(const BasicJsonType& j) { @@ -43,15 +66,15 @@ class binary_writer { case value_t::null: { - oa->write_character(static_cast(0xF6)); + oa->write_character(to_char_type(0xF6)); break; } case value_t::boolean: { oa->write_character(j.m_value.boolean - ? static_cast(0xF5) - : static_cast(0xF4)); + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -68,22 +91,22 @@ class binary_writer } else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x18)); + oa->write_character(to_char_type(0x18)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x19)); + oa->write_character(to_char_type(0x19)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x1A)); + oa->write_character(to_char_type(0x1A)); write_number(static_cast(j.m_value.number_integer)); } else { - oa->write_character(static_cast(0x1B)); + oa->write_character(to_char_type(0x1B)); write_number(static_cast(j.m_value.number_integer)); } } @@ -98,22 +121,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x38)); + oa->write_character(to_char_type(0x38)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x39)); + oa->write_character(to_char_type(0x39)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x3A)); + oa->write_character(to_char_type(0x3A)); write_number(static_cast(positive_number)); } else { - oa->write_character(static_cast(0x3B)); + oa->write_character(to_char_type(0x3B)); write_number(static_cast(positive_number)); } } @@ -128,22 +151,22 @@ class binary_writer } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x18)); + oa->write_character(to_char_type(0x18)); write_number(static_cast(j.m_value.number_unsigned)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x19)); + oa->write_character(to_char_type(0x19)); write_number(static_cast(j.m_value.number_unsigned)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x1A)); + oa->write_character(to_char_type(0x1A)); write_number(static_cast(j.m_value.number_unsigned)); } else { - oa->write_character(static_cast(0x1B)); + oa->write_character(to_char_type(0x1B)); write_number(static_cast(j.m_value.number_unsigned)); } break; @@ -166,23 +189,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x78)); + oa->write_character(to_char_type(0x78)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x79)); + oa->write_character(to_char_type(0x79)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x7A)); + oa->write_character(to_char_type(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x7B)); + oa->write_character(to_char_type(0x7B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -204,23 +227,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x98)); + oa->write_character(to_char_type(0x98)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x99)); + oa->write_character(to_char_type(0x99)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x9A)); + oa->write_character(to_char_type(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0x9B)); + oa->write_character(to_char_type(0x9B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -243,23 +266,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0xB8)); + oa->write_character(to_char_type(0xB8)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0xB9)); + oa->write_character(to_char_type(0xB9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0xBA)); + oa->write_character(to_char_type(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(static_cast(0xBB)); + oa->write_character(to_char_type(0xBB)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -279,7 +302,7 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize */ void write_msgpack(const BasicJsonType& j) { @@ -287,15 +310,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(static_cast(0xC0)); + oa->write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { oa->write_character(j.m_value.boolean - ? static_cast(0xC3) - : static_cast(0xC2)); + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -314,25 +337,25 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(static_cast(0xCC)); + oa->write_character(to_char_type(0xCC)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(static_cast(0xCD)); + oa->write_character(to_char_type(0xCD)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(static_cast(0xCE)); + oa->write_character(to_char_type(0xCE)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(static_cast(0xCF)); + oa->write_character(to_char_type(0xCF)); write_number(static_cast(j.m_value.number_integer)); } } @@ -347,28 +370,28 @@ class binary_writer j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 - oa->write_character(static_cast(0xD0)); + oa->write_character(to_char_type(0xD0)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 - oa->write_character(static_cast(0xD1)); + oa->write_character(to_char_type(0xD1)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 - oa->write_character(static_cast(0xD2)); + oa->write_character(to_char_type(0xD2)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 - oa->write_character(static_cast(0xD3)); + oa->write_character(to_char_type(0xD3)); write_number(static_cast(j.m_value.number_integer)); } } @@ -385,25 +408,25 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(static_cast(0xCC)); + oa->write_character(to_char_type(0xCC)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(static_cast(0xCD)); + oa->write_character(to_char_type(0xCD)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(static_cast(0xCE)); + oa->write_character(to_char_type(0xCE)); write_number(static_cast(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(static_cast(0xCF)); + oa->write_character(to_char_type(0xCF)); write_number(static_cast(j.m_value.number_integer)); } break; @@ -428,19 +451,19 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // str 8 - oa->write_character(static_cast(0xD9)); + oa->write_character(to_char_type(0xD9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 16 - oa->write_character(static_cast(0xDA)); + oa->write_character(to_char_type(0xDA)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 32 - oa->write_character(static_cast(0xDB)); + oa->write_character(to_char_type(0xDB)); write_number(static_cast(N)); } @@ -463,13 +486,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // array 16 - oa->write_character(static_cast(0xDC)); + oa->write_character(to_char_type(0xDC)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // array 32 - oa->write_character(static_cast(0xDD)); + oa->write_character(to_char_type(0xDD)); write_number(static_cast(N)); } @@ -493,13 +516,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // map 16 - oa->write_character(static_cast(0xDE)); + oa->write_character(to_char_type(0xDE)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // map 32 - oa->write_character(static_cast(0xDF)); + oa->write_character(to_char_type(0xDF)); write_number(static_cast(N)); } @@ -532,7 +555,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('Z')); + oa->write_character(to_char_type('Z')); } break; } @@ -540,9 +563,11 @@ class binary_writer case value_t::boolean: { if (add_prefix) + { oa->write_character(j.m_value.boolean - ? static_cast('T') - : static_cast('F')); + ? to_char_type('T') + : to_char_type('F')); + } break; } @@ -568,7 +593,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('S')); + oa->write_character(to_char_type('S')); } write_number_with_ubjson_prefix(j.m_value.string->size(), true); oa->write_characters( @@ -581,7 +606,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('[')); + oa->write_character(to_char_type('[')); } bool prefix_required = true; @@ -598,14 +623,14 @@ class binary_writer if (same_prefix) { prefix_required = false; - oa->write_character(static_cast('$')); + oa->write_character(to_char_type('$')); oa->write_character(first_prefix); } } if (use_count) { - oa->write_character(static_cast('#')); + oa->write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_value.array->size(), true); } @@ -616,7 +641,7 @@ class binary_writer if (not use_count) { - oa->write_character(static_cast(']')); + oa->write_character(to_char_type(']')); } break; @@ -626,7 +651,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('{')); + oa->write_character(to_char_type('{')); } bool prefix_required = true; @@ -643,14 +668,14 @@ class binary_writer if (same_prefix) { prefix_required = false; - oa->write_character(static_cast('$')); + oa->write_character(to_char_type('$')); oa->write_character(first_prefix); } } if (use_count) { - oa->write_character(static_cast('#')); + oa->write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_value.object->size(), true); } @@ -665,7 +690,7 @@ class binary_writer if (not use_count) { - oa->write_character(static_cast('}')); + oa->write_character(to_char_type('}')); } break; @@ -677,33 +702,349 @@ class binary_writer } private: - /* - @brief write a number to output input + ////////// + // BSON // + ////////// - @param[in] n number of type @a NumberType - @tparam NumberType the type of the number + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name) + { + const auto it = name.find(static_cast(0)); + if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, + "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")")); + } - @note This function needs to respect the system's endianess, because bytes - in CBOR, MessagePack, and UBJSON are stored in network order (big - endian) and therefore need reordering on little endian systems. + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter */ - template - void write_number(const NumberType n) + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) { - // step 1: write number to array of length NumberType - std::array vec; - std::memcpy(vec.data(), &n, sizeof(NumberType)); + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast(name.c_str()), + name.size() + 1u); + } - // step 2: write array to output (with possible reordering) - if (is_little_endian) + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number(static_cast(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + if ((std::numeric_limits::min)() <= value and value <= (std::numeric_limits::max)()) { - // reverse byte order prior to conversion if necessary - std::reverse(vec.begin(), vec.end()); + return sizeof(std::int32_t); } + else + { + return sizeof(std::int64_t); + } + } - oa->write_characters(vec.data(), sizeof(NumberType)); + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits::min)() <= value and value <= (std::numeric_limits::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number(static_cast(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number(static_cast(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast((std::numeric_limits::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const std::uint64_t value) + { + if (value <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number(static_cast(value)); + } + else if (value <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number(static_cast(value)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64")); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); + } + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number(static_cast(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + default: + assert(false); + return 0ul; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + @return The size of the BSON entry + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j.m_value.number_unsigned); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + default: + assert(false); + return; + // LCOV_EXCL_STOP + }; } + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul, + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number(static_cast(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + // UBJSON: write number (floating point) template::value, int>::type = 0> @@ -727,7 +1068,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('i')); // int8 + oa->write_character(to_char_type('i')); // int8 } write_number(static_cast(n)); } @@ -735,7 +1076,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('U')); // uint8 + oa->write_character(to_char_type('U')); // uint8 } write_number(static_cast(n)); } @@ -743,7 +1084,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('I')); // int16 + oa->write_character(to_char_type('I')); // int16 } write_number(static_cast(n)); } @@ -751,7 +1092,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('l')); // int32 + oa->write_character(to_char_type('l')); // int32 } write_number(static_cast(n)); } @@ -759,13 +1100,13 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('L')); // int64 + oa->write_character(to_char_type('L')); // int64 } write_number(static_cast(n)); } else { - JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); } } @@ -780,7 +1121,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('i')); // int8 + oa->write_character(to_char_type('i')); // int8 } write_number(static_cast(n)); } @@ -788,7 +1129,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('U')); // uint8 + oa->write_character(to_char_type('U')); // uint8 } write_number(static_cast(n)); } @@ -796,7 +1137,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('I')); // int16 + oa->write_character(to_char_type('I')); // int16 } write_number(static_cast(n)); } @@ -804,7 +1145,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('l')); // int32 + oa->write_character(to_char_type('l')); // int32 } write_number(static_cast(n)); } @@ -812,14 +1153,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('L')); // int64 + oa->write_character(to_char_type('L')); // int64 } write_number(static_cast(n)); } // LCOV_EXCL_START else { - JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); } // LCOV_EXCL_STOP } @@ -849,22 +1190,20 @@ class binary_writer { return 'i'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'U'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'I'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'l'; } - else // no check and assume int64_t (see note above) - { - return 'L'; - } + // no check and assume int64_t (see note above) + return 'L'; } case value_t::number_unsigned: @@ -873,22 +1212,20 @@ class binary_writer { return 'i'; } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'U'; } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'I'; } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'l'; } - else // no check and assume int64_t (see note above) - { - return 'L'; - } + // no check and assume int64_t (see note above) + return 'L'; } case value_t::number_float: @@ -908,34 +1245,87 @@ class binary_writer } } - static constexpr CharType get_cbor_float_prefix(float) + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) { - return static_cast(0xFA); // Single-Precision Float + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 } - static constexpr CharType get_cbor_float_prefix(double) + /////////////////////// + // Utility functions // + /////////////////////// + + /* + @brief write a number to output input + @param[in] n number of type @a NumberType + @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian + + @note This function needs to respect the system's endianess, because bytes + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. + */ + template + void write_number(const NumberType n) { - return static_cast(0xFB); // Double-Precision Float + // step 1: write number to array of length NumberType + std::array vec; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + + // step 2: write array to output (with possible reordering) + if (is_little_endian and not OutputIsLittleEndian) + { + // reverse byte order prior to conversion if necessary + std::reverse(vec.begin(), vec.end()); + } + + oa->write_characters(vec.data(), sizeof(NumberType)); } - static constexpr CharType get_msgpack_float_prefix(float) + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed::value and std::is_signed::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept { - return static_cast(0xCA); // float 32 + return *reinterpret_cast(&x); } - static constexpr CharType get_msgpack_float_prefix(double) + template < typename C = CharType, + enable_if_t < std::is_signed::value and std::is_unsigned::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept { - return static_cast(0xCB); // float 64 + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_pod::value, "CharType must be POD"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; } - static constexpr CharType get_ubjson_float_prefix(float) + template::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept { - return 'd'; // float 32 + return x; } - static constexpr CharType get_ubjson_float_prefix(double) + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed::value and + std::is_signed::value and + std::is_same::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept { - return 'D'; // float 64 + return x; } private: @@ -945,5 +1335,5 @@ class binary_writer /// the output output_adapter_t oa = nullptr; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/output/output_adapters.hpp b/include/lib/modernjson/detail/output/output_adapters.hpp index ff86a6e..699757d 100644 --- a/include/lib/modernjson/detail/output/output_adapters.hpp +++ b/include/lib/modernjson/detail/output/output_adapters.hpp @@ -30,7 +30,9 @@ template class output_vector_adapter : public output_adapter_protocol { public: - explicit output_vector_adapter(std::vector& vec) : v(vec) {} + explicit output_vector_adapter(std::vector& vec) noexcept + : v(vec) + {} void write_character(CharType c) override { @@ -51,7 +53,9 @@ template class output_stream_adapter : public output_adapter_protocol { public: - explicit output_stream_adapter(std::basic_ostream& s) : stream(s) {} + explicit output_stream_adapter(std::basic_ostream& s) noexcept + : stream(s) + {} void write_character(CharType c) override { @@ -72,7 +76,9 @@ template> class output_string_adapter : public output_adapter_protocol { public: - explicit output_string_adapter(StringType& s) : str(s) {} + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} void write_character(CharType c) override { @@ -109,5 +115,5 @@ class output_adapter private: output_adapter_t oa = nullptr; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/output/serializer.hpp b/include/lib/modernjson/detail/output/serializer.hpp index 7c6abb8..a34a452 100644 --- a/include/lib/modernjson/detail/output/serializer.hpp +++ b/include/lib/modernjson/detail/output/serializer.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,14 @@ namespace detail // serialization // /////////////////// +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + template class serializer { @@ -42,17 +51,25 @@ class serializer /*! @param[in] s output stream to serialize to @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors */ - serializer(output_adapter_t s, const char ichar) - : o(std::move(s)), loc(std::localeconv()), - thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)), - decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)), - indent_char(ichar), indent_string(512, indent_char) + serializer(output_adapter_t s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)) + , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) {} // delete because of pointer members serializer(const serializer&) = delete; serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; /*! @brief internal implementation of the serialization function @@ -284,6 +301,10 @@ class serializer uint8_t state = UTF8_ACCEPT; std::size_t bytes = 0; // number of bytes written to string_buffer + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; + for (std::size_t i = 0; i < s.size(); ++i) { const auto byte = static_cast(s[i]); @@ -351,15 +372,15 @@ class serializer { if (codepoint <= 0xFFFF) { - std::snprintf(string_buffer.data() + bytes, 7, "\\u%04x", - static_cast(codepoint)); + (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast(codepoint)); bytes += 6; } else { - std::snprintf(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", - static_cast(0xD7C0 + (codepoint >> 10)), - static_cast(0xDC00 + (codepoint & 0x3FF))); + (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast(0xD7C0 + (codepoint >> 10)), + static_cast(0xDC00 + (codepoint & 0x3FF))); bytes += 12; } } @@ -381,14 +402,69 @@ class serializer o->write_characters(string_buffer.data(), bytes); bytes = 0; } + + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; break; } case UTF8_REJECT: // decode found invalid UTF-8 byte { - std::string sn(3, '\0'); - snprintf(&sn[0], sn.size(), "%.2X", byte); - JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + } + + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } + + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); + } + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + } + break; } default: // decode found yet incomplete multi-byte code point @@ -398,11 +474,13 @@ class serializer // code point will not be escaped - copy byte to buffer string_buffer[bytes++] = s[i]; } + ++undumped_chars; break; } } } + // we finished processing the string if (JSON_LIKELY(state == UTF8_ACCEPT)) { // write buffer @@ -414,9 +492,38 @@ class serializer else { // we finish reading, but do not accept: string was incomplete - std::string sn(3, '\0'); - snprintf(&sn[0], sn.size(), "%.2X", static_cast(s.back())); - JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast(s.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); + } + + case error_handler_t::ignore: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + break; + } + + case error_handler_t::replace: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) + { + o->write_characters("\\ufffd", 6); + } + else + { + o->write_characters("\xEF\xBF\xBD", 3); + } + break; + } + } } } @@ -442,7 +549,7 @@ class serializer return; } - const bool is_negative = (x <= 0) and (x != 0); // see issue #755 + const bool is_negative = std::is_same::value and not (x >= 0); // see issue #755 std::size_t i = 0; while (x != 0) @@ -509,7 +616,7 @@ class serializer static constexpr auto d = std::numeric_limits::max_digits10; // the actual conversion - std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); // negative value indicates an error assert(len > 0); @@ -626,6 +733,9 @@ class serializer const char indent_char; /// the indentation string string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; }; -} -} +} // namespace detail +} // namespace nlohmann diff --git a/include/lib/modernjson/detail/value_t.hpp b/include/lib/modernjson/detail/value_t.hpp index 326367c..10d555b 100644 --- a/include/lib/modernjson/detail/value_t.hpp +++ b/include/lib/modernjson/detail/value_t.hpp @@ -72,5 +72,5 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept const auto r_index = static_cast(rhs); return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]; } -} -} +} // namespace detail +} // namespace nlohmann -- cgit v1.2.3