Unverified Commit cf9299d2 authored by Niels Lohmann's avatar Niels Lohmann

Merge branch 'feature/sax2' into develop #971

parents 1c6b332d 3cdc4d78
...@@ -9,6 +9,7 @@ SRCS = include/nlohmann/json.hpp \ ...@@ -9,6 +9,7 @@ SRCS = include/nlohmann/json.hpp \
include/nlohmann/detail/exceptions.hpp \ include/nlohmann/detail/exceptions.hpp \
include/nlohmann/detail/input/binary_reader.hpp \ include/nlohmann/detail/input/binary_reader.hpp \
include/nlohmann/detail/input/input_adapters.hpp \ include/nlohmann/detail/input/input_adapters.hpp \
include/nlohmann/detail/input/json_sax.hpp \
include/nlohmann/detail/input/lexer.hpp \ include/nlohmann/detail/input/lexer.hpp \
include/nlohmann/detail/input/parser.hpp \ include/nlohmann/detail/input/parser.hpp \
include/nlohmann/detail/iterators/internal_iterator.hpp \ include/nlohmann/detail/iterators/internal_iterator.hpp \
...@@ -84,7 +85,7 @@ coverage: ...@@ -84,7 +85,7 @@ coverage:
mkdir build_coverage mkdir build_coverage
cd build_coverage ; CXX=g++-5 cmake .. -GNinja -DJSON_Coverage=ON -DJSON_MultipleHeaders=ON cd build_coverage ; CXX=g++-5 cmake .. -GNinja -DJSON_Coverage=ON -DJSON_MultipleHeaders=ON
cd build_coverage ; ninja cd build_coverage ; ninja
cd build_coverage ; ctest -j10 cd build_coverage ; ctest -E '.*_default' -j10
cd build_coverage ; ninja lcov_html cd build_coverage ; ninja lcov_html
open build_coverage/test/html/index.html open build_coverage/test/html/index.html
......
#pragma once #pragma once
#include <algorithm> // min
#include <array> // array
#include <cassert> // assert #include <cassert> // assert
#include <cstddef> // size_t #include <cstddef> // size_t
#include <cstring> // strlen #include <cstring> // strlen
#include <ios> // streamsize, streamoff, streampos
#include <istream> // istream #include <istream> // istream
#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next #include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
#include <memory> // shared_ptr, make_shared, addressof #include <memory> // shared_ptr, make_shared, addressof
...@@ -20,6 +17,9 @@ namespace nlohmann ...@@ -20,6 +17,9 @@ namespace nlohmann
{ {
namespace detail namespace detail
{ {
/// the supported input formats
enum class input_format_t { json, cbor, msgpack, ubjson };
//////////////////// ////////////////////
// input adapters // // input adapters //
//////////////////// ////////////////////
...@@ -28,19 +28,17 @@ namespace detail ...@@ -28,19 +28,17 @@ namespace detail
@brief abstract input adapter interface @brief abstract input adapter interface
Produces a stream of std::char_traits<char>::int_type characters from a Produces a stream of std::char_traits<char>::int_type characters from a
std::istream, a buffer, or some other input type. Accepts the return of exactly std::istream, a buffer, or some other input type. Accepts the return of
one non-EOF character for future input. The int_type characters returned exactly one non-EOF character for future input. The int_type characters
consist of all valid char values as positive values (typically unsigned char), returned consist of all valid char values as positive values (typically
plus an EOF value outside that range, specified by the value of the function unsigned char), plus an EOF value outside that range, specified by the value
std::char_traits<char>::eof(). This value is typically -1, but could be any of the function std::char_traits<char>::eof(). This value is typically -1, but
arbitrary value which is not a valid char value. could be any arbitrary value which is not a valid char value.
*/ */
struct input_adapter_protocol struct input_adapter_protocol
{ {
/// get a character [0,255] or std::char_traits<char>::eof(). /// get a character [0,255] or std::char_traits<char>::eof().
virtual std::char_traits<char>::int_type get_character() = 0; virtual std::char_traits<char>::int_type get_character() = 0;
/// restore the last non-eof() character to input
virtual void unget_character() = 0;
virtual ~input_adapter_protocol() = default; virtual ~input_adapter_protocol() = default;
}; };
...@@ -68,34 +66,7 @@ class input_stream_adapter : public input_adapter_protocol ...@@ -68,34 +66,7 @@ class input_stream_adapter : public input_adapter_protocol
explicit input_stream_adapter(std::istream& i) explicit input_stream_adapter(std::istream& i)
: is(i), sb(*i.rdbuf()) : is(i), sb(*i.rdbuf())
{ {}
// skip byte order mark
std::char_traits<char>::int_type c;
if ((c = get_character()) == 0xEF)
{
if ((c = get_character()) == 0xBB)
{
if ((c = get_character()) == 0xBF)
{
return; // Ignore BOM
}
else if (c != std::char_traits<char>::eof())
{
is.unget();
}
is.putback('\xBB');
}
else if (c != std::char_traits<char>::eof())
{
is.unget();
}
is.putback('\xEF');
}
else if (c != std::char_traits<char>::eof())
{
is.unget(); // no byte order mark; process as usual
}
}
// delete because of pointer members // delete because of pointer members
input_stream_adapter(const input_stream_adapter&) = delete; input_stream_adapter(const input_stream_adapter&) = delete;
...@@ -109,11 +80,6 @@ class input_stream_adapter : public input_adapter_protocol ...@@ -109,11 +80,6 @@ class input_stream_adapter : public input_adapter_protocol
return sb.sbumpc(); return sb.sbumpc();
} }
void unget_character() override
{
sb.sungetc(); // is.unget() avoided for performance
}
private: private:
/// the associated input stream /// the associated input stream
std::istream& is; std::istream& is;
...@@ -125,14 +91,8 @@ class input_buffer_adapter : public input_adapter_protocol ...@@ -125,14 +91,8 @@ class input_buffer_adapter : public input_adapter_protocol
{ {
public: public:
input_buffer_adapter(const char* b, const std::size_t l) input_buffer_adapter(const char* b, const std::size_t l)
: cursor(b), limit(b + l), start(b) : cursor(b), limit(b + l)
{ {}
// skip byte order mark
if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
{
cursor += 3;
}
}
// delete because of pointer members // delete because of pointer members
input_buffer_adapter(const input_buffer_adapter&) = delete; input_buffer_adapter(const input_buffer_adapter&) = delete;
...@@ -148,21 +108,11 @@ class input_buffer_adapter : public input_adapter_protocol ...@@ -148,21 +108,11 @@ class input_buffer_adapter : public input_adapter_protocol
return std::char_traits<char>::eof(); return std::char_traits<char>::eof();
} }
void unget_character() noexcept override
{
if (JSON_LIKELY(cursor > start))
{
--cursor;
}
}
private: private:
/// pointer to the current character /// pointer to the current character
const char* cursor; const char* cursor;
/// pointer past the last character /// pointer past the last character
const char* limit; const char* const limit;
/// pointer to the first character
const char* start;
}; };
template<typename WideStringType> template<typename WideStringType>
...@@ -173,13 +123,6 @@ class wide_string_input_adapter : public input_adapter_protocol ...@@ -173,13 +123,6 @@ class wide_string_input_adapter : public input_adapter_protocol
std::char_traits<char>::int_type get_character() noexcept override std::char_traits<char>::int_type get_character() noexcept override
{ {
// unget_character() was called previously: return the last character
if (next_unget)
{
next_unget = false;
return last_char;
}
// check if buffer needs to be filled // check if buffer needs to be filled
if (utf8_bytes_index == utf8_bytes_filled) if (utf8_bytes_index == utf8_bytes_filled)
{ {
...@@ -199,12 +142,7 @@ class wide_string_input_adapter : public input_adapter_protocol ...@@ -199,12 +142,7 @@ class wide_string_input_adapter : public input_adapter_protocol
// use buffer // use buffer
assert(utf8_bytes_filled > 0); assert(utf8_bytes_filled > 0);
assert(utf8_bytes_index < utf8_bytes_filled); assert(utf8_bytes_index < utf8_bytes_filled);
return (last_char = utf8_bytes[utf8_bytes_index++]); return utf8_bytes[utf8_bytes_index++];
}
void unget_character() noexcept override
{
next_unget = true;
} }
private: private:
...@@ -328,11 +266,6 @@ class wide_string_input_adapter : public input_adapter_protocol ...@@ -328,11 +266,6 @@ class wide_string_input_adapter : public input_adapter_protocol
std::size_t utf8_bytes_index = 0; std::size_t utf8_bytes_index = 0;
/// number of valid bytes in the utf8_codes array /// number of valid bytes in the utf8_codes array
std::size_t utf8_bytes_filled = 0; std::size_t utf8_bytes_filled = 0;
/// the last character (returned after unget_character() is called)
std::char_traits<char>::int_type last_char = 0;
/// whether get_character() should return last_char
bool next_unget = false;
}; };
class input_adapter class input_adapter
......
This diff is collapsed.
...@@ -99,7 +99,7 @@ class lexer ...@@ -99,7 +99,7 @@ class lexer
} }
} }
explicit lexer(detail::input_adapter_t adapter) explicit lexer(detail::input_adapter_t&& adapter)
: ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {} : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
// delete because of pointer members // delete because of pointer members
...@@ -1081,7 +1081,16 @@ scan_number_done: ...@@ -1081,7 +1081,16 @@ scan_number_done:
std::char_traits<char>::int_type get() std::char_traits<char>::int_type get()
{ {
++chars_read; ++chars_read;
current = ia->get_character(); if (next_unget)
{
// just reset the next_unget variable and work with current
next_unget = false;
}
else
{
current = ia->get_character();
}
if (JSON_LIKELY(current != std::char_traits<char>::eof())) if (JSON_LIKELY(current != std::char_traits<char>::eof()))
{ {
token_string.push_back(std::char_traits<char>::to_char_type(current)); token_string.push_back(std::char_traits<char>::to_char_type(current));
...@@ -1089,13 +1098,20 @@ scan_number_done: ...@@ -1089,13 +1098,20 @@ scan_number_done:
return current; return current;
} }
/// unget current character (return it again on next get) /*!
@brief unget current character (read it again on next get)
We implement unget by setting variable next_unget to true. The input is not
changed - we just simulate ungetting by modifying chars_read and
token_string. The next call to get() will behave as if the unget character
is read again.
*/
void unget() void unget()
{ {
next_unget = true;
--chars_read; --chars_read;
if (JSON_LIKELY(current != std::char_traits<char>::eof())) if (JSON_LIKELY(current != std::char_traits<char>::eof()))
{ {
ia->unget_character();
assert(token_string.size() != 0); assert(token_string.size() != 0);
token_string.pop_back(); token_string.pop_back();
} }
...@@ -1131,9 +1147,9 @@ scan_number_done: ...@@ -1131,9 +1147,9 @@ scan_number_done:
} }
/// return current string value (implicitly resets the token; useful only once) /// return current string value (implicitly resets the token; useful only once)
string_t&& move_string() string_t& get_string()
{ {
return std::move(token_buffer); return token_buffer;
} }
///////////////////// /////////////////////
...@@ -1183,8 +1199,43 @@ scan_number_done: ...@@ -1183,8 +1199,43 @@ scan_number_done:
// actual scanner // actual scanner
///////////////////// /////////////////////
/*!
@brief skip the UTF-8 byte order mark
@return true iff there is no BOM or the correct BOM has been skipped
*/
bool skip_bom()
{
if (get() == 0xEF)
{
if (get() == 0xBB and get() == 0xBF)
{
// we completely parsed the BOM
return true;
}
else
{
// after reading 0xEF, an unexpected character followed
return false;
}
}
else
{
// the first character is not the beginning of the BOM; unget it to
// process is later
unget();
return true;
}
}
token_type scan() token_type scan()
{ {
// initially, skip the BOM
if (chars_read == 0 and not skip_bom())
{
error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
return token_type::parse_error;
}
// read next character and ignore whitespace // read next character and ignore whitespace
do do
{ {
...@@ -1254,6 +1305,9 @@ scan_number_done: ...@@ -1254,6 +1305,9 @@ scan_number_done:
/// the current character /// the current character
std::char_traits<char>::int_type current = std::char_traits<char>::eof(); std::char_traits<char>::int_type current = std::char_traits<char>::eof();
/// whether the next get() call should just return current
bool next_unget = false;
/// the number of characters read /// the number of characters read
std::size_t chars_read = 0; std::size_t chars_read = 0;
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -35,6 +35,170 @@ using nlohmann::json; ...@@ -35,6 +35,170 @@ using nlohmann::json;
#include <valarray> #include <valarray>
class SaxEventLogger : public nlohmann::json::json_sax_t
{
public:
bool null() override
{
events.push_back("null()");
return true;
}
bool boolean(bool val) override
{
events.push_back(val ? "boolean(true)" : "boolean(false)");
return true;
}
bool number_integer(json::number_integer_t val) override
{
events.push_back("number_integer(" + std::to_string(val) + ")");
return true;
}
bool number_unsigned(json::number_unsigned_t val) override
{
events.push_back("number_unsigned(" + std::to_string(val) + ")");
return true;
}
bool number_float(json::number_float_t, const std::string& s) override
{
events.push_back("number_float(" + s + ")");
return true;
}
bool string(std::string& val) override
{
events.push_back("string(" + val + ")");
return true;
}
bool start_object(std::size_t elements) override
{
if (elements == no_limit)
{
events.push_back("start_object()");
}
else
{
events.push_back("start_object(" + std::to_string(elements) + ")");
}
return true;
}
bool key(std::string& val) override
{
events.push_back("key(" + val + ")");
return true;
}
bool end_object() override
{
events.push_back("end_object()");
return true;
}
bool start_array(std::size_t elements) override
{
if (elements == no_limit)
{
events.push_back("start_array()");
}
else
{
events.push_back("start_array(" + std::to_string(elements) + ")");
}
return true;
}
bool end_array() override
{
events.push_back("end_array()");
return true;
}
bool parse_error(std::size_t position, const std::string&, const json::exception&) override
{
errored = true;
events.push_back("parse_error(" + std::to_string(position) + ")");
return false;
}
std::vector<std::string> events;
bool errored = false;
};
class SaxCountdown : public nlohmann::json::json_sax_t
{
public:
explicit SaxCountdown(const int count) : events_left(count)
{}
bool null() override
{
return events_left-- > 0;
}
bool boolean(bool) override
{
return events_left-- > 0;
}
bool number_integer(json::number_integer_t) override
{
return events_left-- > 0;
}
bool number_unsigned(json::number_unsigned_t) override
{
return events_left-- > 0;
}
bool number_float(json::number_float_t, const std::string&) override
{
return events_left-- > 0;
}
bool string(std::string&) override
{
return events_left-- > 0;
}
bool start_object(std::size_t) override
{
return events_left-- > 0;
}
bool key(std::string&) override
{
return events_left-- > 0;
}
bool end_object() override
{
return events_left-- > 0;
}
bool start_array(std::size_t) override
{
return events_left-- > 0;
}
bool end_array() override
{
return events_left-- > 0;
}
bool parse_error(std::size_t, const std::string&, const json::exception&) override
{
return false;
}
private:
int events_left = 0;
};
json parser_helper(const std::string& s); json parser_helper(const std::string& s);
bool accept_helper(const std::string& s); bool accept_helper(const std::string& s);
...@@ -49,11 +213,18 @@ json parser_helper(const std::string& s) ...@@ -49,11 +213,18 @@ json parser_helper(const std::string& s)
CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j_nothrow)); CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j_nothrow));
CHECK(j_nothrow == j); CHECK(j_nothrow == j);
json j_sax;
nlohmann::detail::json_sax_dom_parser<json> sdp(j_sax);
json::sax_parse(s, &sdp);
CHECK(j_sax == j);
return j; return j;
} }
bool accept_helper(const std::string& s) bool accept_helper(const std::string& s)
{ {
CAPTURE(s);
// 1. parse s without exceptions // 1. parse s without exceptions
json j; json j;
CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j)); CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j));
...@@ -65,7 +236,23 @@ bool accept_helper(const std::string& s) ...@@ -65,7 +236,23 @@ bool accept_helper(const std::string& s)
// 3. check if both approaches come to the same result // 3. check if both approaches come to the same result
CHECK(ok_noexcept == ok_accept); CHECK(ok_noexcept == ok_accept);
// 4. return result // 4. parse with SAX (compare with relaxed accept result)
SaxEventLogger el;
CHECK_NOTHROW(json::sax_parse(s, &el, json::input_format_t::json, false));
CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept(false) == not el.errored);
// 5. parse with simple callback
json::parser_callback_t cb = [](int, json::parse_event_t, json&)
{
return true;
};
json j_cb = json::parse(s, cb, false);
const bool ok_noexcept_cb = not j_cb.is_discarded();
// 6. check if this approach came to the same result
CHECK(ok_noexcept == ok_noexcept_cb);
// 7. return result
return ok_accept; return ok_accept;
} }
...@@ -1473,4 +1660,100 @@ TEST_CASE("parser class") ...@@ -1473,4 +1660,100 @@ TEST_CASE("parser class")
CHECK(j == json(true)); CHECK(j == json(true));
} }
} }
SECTION("improve test coverage")
{
SECTION("parser with callback")
{
json::parser_callback_t cb = [](int, json::parse_event_t, json&)
{
return true;
};
CHECK(json::parse("{\"foo\": true:", cb, false).is_discarded());
CHECK_THROWS_AS(json::parse("{\"foo\": true:", cb), json::parse_error&);
CHECK_THROWS_WITH(json::parse("{\"foo\": true:", cb),
"[json.exception.parse_error.101] parse error at 13: syntax error - unexpected ':'; expected '}'");
CHECK_THROWS_AS(json::parse("1.18973e+4932", cb), json::out_of_range&);
CHECK_THROWS_WITH(json::parse("1.18973e+4932", cb),
"[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'");
}
SECTION("SAX parser")
{
SECTION("} without value")
{
SaxCountdown s(1);
CHECK(json::sax_parse("{}", &s) == false);
}
SECTION("} with value")
{
SaxCountdown s(3);
CHECK(json::sax_parse("{\"k1\": true}", &s) == false);
}
SECTION("second key")
{
SaxCountdown s(3);
CHECK(json::sax_parse("{\"k1\": true, \"k2\": false}", &s) == false);
}
SECTION("] without value")
{
SaxCountdown s(1);
CHECK(json::sax_parse("[]", &s) == false);
}
SECTION("] with value")
{
SaxCountdown s(2);
CHECK(json::sax_parse("[1]", &s) == false);
}
SECTION("float")
{
SaxCountdown s(0);
CHECK(json::sax_parse("3.14", &s) == false);
}
SECTION("false")
{
SaxCountdown s(0);
CHECK(json::sax_parse("false", &s) == false);
}
SECTION("null")
{
SaxCountdown s(0);
CHECK(json::sax_parse("null", &s) == false);
}
SECTION("true")
{
SaxCountdown s(0);
CHECK(json::sax_parse("true", &s) == false);
}
SECTION("unsigned")
{
SaxCountdown s(0);
CHECK(json::sax_parse("12", &s) == false);
}
SECTION("integer")
{
SaxCountdown s(0);
CHECK(json::sax_parse("-12", &s) == false);
}
SECTION("string")
{
SaxCountdown s(0);
CHECK(json::sax_parse("\"foo\"", &s) == false);
}
}
}
} }
This diff is collapsed.
This diff is collapsed.
...@@ -1139,7 +1139,6 @@ TEST_CASE("nst's JSONTestSuite (2)") ...@@ -1139,7 +1139,6 @@ TEST_CASE("nst's JSONTestSuite (2)")
"test/data/nst_json_testsuite2/test_parsing/n_string_unescaped_tab.json", "test/data/nst_json_testsuite2/test_parsing/n_string_unescaped_tab.json",
"test/data/nst_json_testsuite2/test_parsing/n_string_unicode_CapitalU.json", "test/data/nst_json_testsuite2/test_parsing/n_string_unicode_CapitalU.json",
"test/data/nst_json_testsuite2/test_parsing/n_string_with_trailing_garbage.json", "test/data/nst_json_testsuite2/test_parsing/n_string_with_trailing_garbage.json",
//"test/data/nst_json_testsuite2/test_parsing/n_structure_100000_opening_arrays.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_U+2060_word_joined.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_U+2060_word_joined.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_UTF8_BOM_no_data.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_UTF8_BOM_no_data.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_angle_bracket_..json", "test/data/nst_json_testsuite2/test_parsing/n_structure_angle_bracket_..json",
...@@ -1165,7 +1164,6 @@ TEST_CASE("nst's JSONTestSuite (2)") ...@@ -1165,7 +1164,6 @@ TEST_CASE("nst's JSONTestSuite (2)")
"test/data/nst_json_testsuite2/test_parsing/n_structure_object_with_trailing_garbage.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_object_with_trailing_garbage.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_apostrophe.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_apostrophe.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_comma.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_comma.json",
//"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_object.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_open_object.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_open_object.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_open_string.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_open_string.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_string.json", "test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_string.json",
...@@ -1199,6 +1197,21 @@ TEST_CASE("nst's JSONTestSuite (2)") ...@@ -1199,6 +1197,21 @@ TEST_CASE("nst's JSONTestSuite (2)")
} }
} }
SECTION("n (previously overflowed)")
{
for (auto filename :
{
"test/data/nst_json_testsuite2/test_parsing/n_structure_100000_opening_arrays.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_object.json"
}
)
{
CAPTURE(filename);
std::ifstream f(filename);
CHECK(not json::accept(f));
}
}
SECTION("i -> y") SECTION("i -> y")
{ {
for (auto filename : for (auto filename :
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment