Unverified Commit cf9299d2 authored by Niels Lohmann's avatar Niels Lohmann

Merge branch 'feature/sax2' into develop #971

parents 1c6b332d 3cdc4d78
......@@ -9,6 +9,7 @@ SRCS = include/nlohmann/json.hpp \
include/nlohmann/detail/exceptions.hpp \
include/nlohmann/detail/input/binary_reader.hpp \
include/nlohmann/detail/input/input_adapters.hpp \
include/nlohmann/detail/input/json_sax.hpp \
include/nlohmann/detail/input/lexer.hpp \
include/nlohmann/detail/input/parser.hpp \
include/nlohmann/detail/iterators/internal_iterator.hpp \
......@@ -84,7 +85,7 @@ coverage:
mkdir build_coverage
cd build_coverage ; CXX=g++-5 cmake .. -GNinja -DJSON_Coverage=ON -DJSON_MultipleHeaders=ON
cd build_coverage ; ninja
cd build_coverage ; ctest -j10
cd build_coverage ; ctest -E '.*_default' -j10
cd build_coverage ; ninja lcov_html
open build_coverage/test/html/index.html
......
#pragma once
#include <algorithm> // min
#include <array> // array
#include <cassert> // assert
#include <cstddef> // size_t
#include <cstring> // strlen
#include <ios> // streamsize, streamoff, streampos
#include <istream> // istream
#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
#include <memory> // shared_ptr, make_shared, addressof
......@@ -20,6 +17,9 @@ namespace nlohmann
{
namespace detail
{
/// the supported input formats
enum class input_format_t { json, cbor, msgpack, ubjson };
////////////////////
// input adapters //
////////////////////
......@@ -28,19 +28,17 @@ namespace detail
@brief abstract input adapter interface
Produces a stream of std::char_traits<char>::int_type characters from a
std::istream, a buffer, or some other input type. Accepts the return of exactly
one non-EOF character for future input. The int_type characters returned
consist of all valid char values as positive values (typically unsigned char),
plus an EOF value outside that range, specified by the value of the function
std::char_traits<char>::eof(). This value is typically -1, but could be any
arbitrary value which is not a valid char value.
std::istream, a buffer, or some other input type. Accepts the return of
exactly one non-EOF character for future input. The int_type characters
returned consist of all valid char values as positive values (typically
unsigned char), plus an EOF value outside that range, specified by the value
of the function std::char_traits<char>::eof(). This value is typically -1, but
could be any arbitrary value which is not a valid char value.
*/
struct input_adapter_protocol
{
/// get a character [0,255] or std::char_traits<char>::eof().
virtual std::char_traits<char>::int_type get_character() = 0;
/// restore the last non-eof() character to input
virtual void unget_character() = 0;
virtual ~input_adapter_protocol() = default;
};
......@@ -68,34 +66,7 @@ class input_stream_adapter : public input_adapter_protocol
explicit input_stream_adapter(std::istream& i)
: is(i), sb(*i.rdbuf())
{
// skip byte order mark
std::char_traits<char>::int_type c;
if ((c = get_character()) == 0xEF)
{
if ((c = get_character()) == 0xBB)
{
if ((c = get_character()) == 0xBF)
{
return; // Ignore BOM
}
else if (c != std::char_traits<char>::eof())
{
is.unget();
}
is.putback('\xBB');
}
else if (c != std::char_traits<char>::eof())
{
is.unget();
}
is.putback('\xEF');
}
else if (c != std::char_traits<char>::eof())
{
is.unget(); // no byte order mark; process as usual
}
}
{}
// delete because of pointer members
input_stream_adapter(const input_stream_adapter&) = delete;
......@@ -109,11 +80,6 @@ class input_stream_adapter : public input_adapter_protocol
return sb.sbumpc();
}
void unget_character() override
{
sb.sungetc(); // is.unget() avoided for performance
}
private:
/// the associated input stream
std::istream& is;
......@@ -125,14 +91,8 @@ class input_buffer_adapter : public input_adapter_protocol
{
public:
input_buffer_adapter(const char* b, const std::size_t l)
: cursor(b), limit(b + l), start(b)
{
// skip byte order mark
if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
{
cursor += 3;
}
}
: cursor(b), limit(b + l)
{}
// delete because of pointer members
input_buffer_adapter(const input_buffer_adapter&) = delete;
......@@ -148,21 +108,11 @@ class input_buffer_adapter : public input_adapter_protocol
return std::char_traits<char>::eof();
}
void unget_character() noexcept override
{
if (JSON_LIKELY(cursor > start))
{
--cursor;
}
}
private:
/// pointer to the current character
const char* cursor;
/// pointer past the last character
const char* limit;
/// pointer to the first character
const char* start;
const char* const limit;
};
template<typename WideStringType>
......@@ -173,13 +123,6 @@ class wide_string_input_adapter : public input_adapter_protocol
std::char_traits<char>::int_type get_character() noexcept override
{
// unget_character() was called previously: return the last character
if (next_unget)
{
next_unget = false;
return last_char;
}
// check if buffer needs to be filled
if (utf8_bytes_index == utf8_bytes_filled)
{
......@@ -199,12 +142,7 @@ class wide_string_input_adapter : public input_adapter_protocol
// use buffer
assert(utf8_bytes_filled > 0);
assert(utf8_bytes_index < utf8_bytes_filled);
return (last_char = utf8_bytes[utf8_bytes_index++]);
}
void unget_character() noexcept override
{
next_unget = true;
return utf8_bytes[utf8_bytes_index++];
}
private:
......@@ -328,11 +266,6 @@ class wide_string_input_adapter : public input_adapter_protocol
std::size_t utf8_bytes_index = 0;
/// number of valid bytes in the utf8_codes array
std::size_t utf8_bytes_filled = 0;
/// the last character (returned after unget_character() is called)
std::char_traits<char>::int_type last_char = 0;
/// whether get_character() should return last_char
bool next_unget = false;
};
class input_adapter
......
This diff is collapsed.
......@@ -99,7 +99,7 @@ class lexer
}
}
explicit lexer(detail::input_adapter_t adapter)
explicit lexer(detail::input_adapter_t&& adapter)
: ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
// delete because of pointer members
......@@ -1081,7 +1081,16 @@ scan_number_done:
std::char_traits<char>::int_type get()
{
++chars_read;
current = ia->get_character();
if (next_unget)
{
// just reset the next_unget variable and work with current
next_unget = false;
}
else
{
current = ia->get_character();
}
if (JSON_LIKELY(current != std::char_traits<char>::eof()))
{
token_string.push_back(std::char_traits<char>::to_char_type(current));
......@@ -1089,13 +1098,20 @@ scan_number_done:
return current;
}
/// unget current character (return it again on next get)
/*!
@brief unget current character (read it again on next get)
We implement unget by setting variable next_unget to true. The input is not
changed - we just simulate ungetting by modifying chars_read and
token_string. The next call to get() will behave as if the unget character
is read again.
*/
void unget()
{
next_unget = true;
--chars_read;
if (JSON_LIKELY(current != std::char_traits<char>::eof()))
{
ia->unget_character();
assert(token_string.size() != 0);
token_string.pop_back();
}
......@@ -1131,9 +1147,9 @@ scan_number_done:
}
/// return current string value (implicitly resets the token; useful only once)
string_t&& move_string()
string_t& get_string()
{
return std::move(token_buffer);
return token_buffer;
}
/////////////////////
......@@ -1183,8 +1199,43 @@ scan_number_done:
// actual scanner
/////////////////////
/*!
@brief skip the UTF-8 byte order mark
@return true iff there is no BOM or the correct BOM has been skipped
*/
bool skip_bom()
{
if (get() == 0xEF)
{
if (get() == 0xBB and get() == 0xBF)
{
// we completely parsed the BOM
return true;
}
else
{
// after reading 0xEF, an unexpected character followed
return false;
}
}
else
{
// the first character is not the beginning of the BOM; unget it to
// process is later
unget();
return true;
}
}
token_type scan()
{
// initially, skip the BOM
if (chars_read == 0 and not skip_bom())
{
error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
return token_type::parse_error;
}
// read next character and ignore whitespace
do
{
......@@ -1254,6 +1305,9 @@ scan_number_done:
/// the current character
std::char_traits<char>::int_type current = std::char_traits<char>::eof();
/// whether the next get() call should just return current
bool next_unget = false;
/// the number of characters read
std::size_t chars_read = 0;
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -35,6 +35,170 @@ using nlohmann::json;
#include <valarray>
class SaxEventLogger : public nlohmann::json::json_sax_t
{
public:
bool null() override
{
events.push_back("null()");
return true;
}
bool boolean(bool val) override
{
events.push_back(val ? "boolean(true)" : "boolean(false)");
return true;
}
bool number_integer(json::number_integer_t val) override
{
events.push_back("number_integer(" + std::to_string(val) + ")");
return true;
}
bool number_unsigned(json::number_unsigned_t val) override
{
events.push_back("number_unsigned(" + std::to_string(val) + ")");
return true;
}
bool number_float(json::number_float_t, const std::string& s) override
{
events.push_back("number_float(" + s + ")");
return true;
}
bool string(std::string& val) override
{
events.push_back("string(" + val + ")");
return true;
}
bool start_object(std::size_t elements) override
{
if (elements == no_limit)
{
events.push_back("start_object()");
}
else
{
events.push_back("start_object(" + std::to_string(elements) + ")");
}
return true;
}
bool key(std::string& val) override
{
events.push_back("key(" + val + ")");
return true;
}
bool end_object() override
{
events.push_back("end_object()");
return true;
}
bool start_array(std::size_t elements) override
{
if (elements == no_limit)
{
events.push_back("start_array()");
}
else
{
events.push_back("start_array(" + std::to_string(elements) + ")");
}
return true;
}
bool end_array() override
{
events.push_back("end_array()");
return true;
}
bool parse_error(std::size_t position, const std::string&, const json::exception&) override
{
errored = true;
events.push_back("parse_error(" + std::to_string(position) + ")");
return false;
}
std::vector<std::string> events;
bool errored = false;
};
class SaxCountdown : public nlohmann::json::json_sax_t
{
public:
explicit SaxCountdown(const int count) : events_left(count)
{}
bool null() override
{
return events_left-- > 0;
}
bool boolean(bool) override
{
return events_left-- > 0;
}
bool number_integer(json::number_integer_t) override
{
return events_left-- > 0;
}
bool number_unsigned(json::number_unsigned_t) override
{
return events_left-- > 0;
}
bool number_float(json::number_float_t, const std::string&) override
{
return events_left-- > 0;
}
bool string(std::string&) override
{
return events_left-- > 0;
}
bool start_object(std::size_t) override
{
return events_left-- > 0;
}
bool key(std::string&) override
{
return events_left-- > 0;
}
bool end_object() override
{
return events_left-- > 0;
}
bool start_array(std::size_t) override
{
return events_left-- > 0;
}
bool end_array() override
{
return events_left-- > 0;
}
bool parse_error(std::size_t, const std::string&, const json::exception&) override
{
return false;
}
private:
int events_left = 0;
};
json parser_helper(const std::string& s);
bool accept_helper(const std::string& s);
......@@ -49,11 +213,18 @@ json parser_helper(const std::string& s)
CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j_nothrow));
CHECK(j_nothrow == j);
json j_sax;
nlohmann::detail::json_sax_dom_parser<json> sdp(j_sax);
json::sax_parse(s, &sdp);
CHECK(j_sax == j);
return j;
}
bool accept_helper(const std::string& s)
{
CAPTURE(s);
// 1. parse s without exceptions
json j;
CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j));
......@@ -65,7 +236,23 @@ bool accept_helper(const std::string& s)
// 3. check if both approaches come to the same result
CHECK(ok_noexcept == ok_accept);
// 4. return result
// 4. parse with SAX (compare with relaxed accept result)
SaxEventLogger el;
CHECK_NOTHROW(json::sax_parse(s, &el, json::input_format_t::json, false));
CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept(false) == not el.errored);
// 5. parse with simple callback
json::parser_callback_t cb = [](int, json::parse_event_t, json&)
{
return true;
};
json j_cb = json::parse(s, cb, false);
const bool ok_noexcept_cb = not j_cb.is_discarded();
// 6. check if this approach came to the same result
CHECK(ok_noexcept == ok_noexcept_cb);
// 7. return result
return ok_accept;
}
......@@ -1473,4 +1660,100 @@ TEST_CASE("parser class")
CHECK(j == json(true));
}
}
SECTION("improve test coverage")
{
SECTION("parser with callback")
{
json::parser_callback_t cb = [](int, json::parse_event_t, json&)
{
return true;
};
CHECK(json::parse("{\"foo\": true:", cb, false).is_discarded());
CHECK_THROWS_AS(json::parse("{\"foo\": true:", cb), json::parse_error&);
CHECK_THROWS_WITH(json::parse("{\"foo\": true:", cb),
"[json.exception.parse_error.101] parse error at 13: syntax error - unexpected ':'; expected '}'");
CHECK_THROWS_AS(json::parse("1.18973e+4932", cb), json::out_of_range&);
CHECK_THROWS_WITH(json::parse("1.18973e+4932", cb),
"[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'");
}
SECTION("SAX parser")
{
SECTION("} without value")
{
SaxCountdown s(1);
CHECK(json::sax_parse("{}", &s) == false);
}
SECTION("} with value")
{
SaxCountdown s(3);
CHECK(json::sax_parse("{\"k1\": true}", &s) == false);
}
SECTION("second key")
{
SaxCountdown s(3);
CHECK(json::sax_parse("{\"k1\": true, \"k2\": false}", &s) == false);
}
SECTION("] without value")
{
SaxCountdown s(1);
CHECK(json::sax_parse("[]", &s) == false);
}
SECTION("] with value")
{
SaxCountdown s(2);
CHECK(json::sax_parse("[1]", &s) == false);
}
SECTION("float")
{
SaxCountdown s(0);
CHECK(json::sax_parse("3.14", &s) == false);
}
SECTION("false")
{
SaxCountdown s(0);
CHECK(json::sax_parse("false", &s) == false);
}
SECTION("null")
{
SaxCountdown s(0);
CHECK(json::sax_parse("null", &s) == false);
}
SECTION("true")
{
SaxCountdown s(0);
CHECK(json::sax_parse("true", &s) == false);
}
SECTION("unsigned")
{
SaxCountdown s(0);
CHECK(json::sax_parse("12", &s) == false);
}
SECTION("integer")
{
SaxCountdown s(0);
CHECK(json::sax_parse("-12", &s) == false);
}
SECTION("string")
{
SaxCountdown s(0);
CHECK(json::sax_parse("\"foo\"", &s) == false);
}
}
}
}
This diff is collapsed.
This diff is collapsed.
......@@ -1139,7 +1139,6 @@ TEST_CASE("nst's JSONTestSuite (2)")
"test/data/nst_json_testsuite2/test_parsing/n_string_unescaped_tab.json",
"test/data/nst_json_testsuite2/test_parsing/n_string_unicode_CapitalU.json",
"test/data/nst_json_testsuite2/test_parsing/n_string_with_trailing_garbage.json",
//"test/data/nst_json_testsuite2/test_parsing/n_structure_100000_opening_arrays.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_U+2060_word_joined.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_UTF8_BOM_no_data.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_angle_bracket_..json",
......@@ -1165,7 +1164,6 @@ TEST_CASE("nst's JSONTestSuite (2)")
"test/data/nst_json_testsuite2/test_parsing/n_structure_object_with_trailing_garbage.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_apostrophe.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_comma.json",
//"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_object.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_open_object.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_open_string.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_string.json",
......@@ -1199,6 +1197,21 @@ TEST_CASE("nst's JSONTestSuite (2)")
}
}
SECTION("n (previously overflowed)")
{
for (auto filename :
{
"test/data/nst_json_testsuite2/test_parsing/n_structure_100000_opening_arrays.json",
"test/data/nst_json_testsuite2/test_parsing/n_structure_open_array_object.json"
}
)
{
CAPTURE(filename);
std::ifstream f(filename);
CHECK(not json::accept(f));
}
}
SECTION("i -> y")
{
for (auto filename :
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment