Unverified Commit 7b501de0 authored by Niels Lohmann's avatar Niels Lohmann Committed by GitHub

Merge pull request #1314 from nlohmann/feature/codec_errors

Allow to set error handler for decoding errors
parents 9294e25c 87ef3f25
...@@ -28,6 +28,14 @@ namespace detail ...@@ -28,6 +28,14 @@ namespace detail
// serialization // // serialization //
/////////////////// ///////////////////
/// how to treat decoding errors
enum class error_handler_t
{
strict, ///< throw a type_error exception in case of invalid UTF-8
replace, ///< replace invalid UTF-8 sequences with U+FFFD
ignore ///< ignore invalid UTF-8 sequences
};
template<typename BasicJsonType> template<typename BasicJsonType>
class serializer class serializer
{ {
...@@ -42,12 +50,17 @@ class serializer ...@@ -42,12 +50,17 @@ class serializer
/*! /*!
@param[in] s output stream to serialize to @param[in] s output stream to serialize to
@param[in] ichar indentation character to use @param[in] ichar indentation character to use
@param[in] error_handler_ how to react on decoding errors
*/ */
serializer(output_adapter_t<char> s, const char ichar) serializer(output_adapter_t<char> s, const char ichar,
: o(std::move(s)), loc(std::localeconv()), error_handler_t error_handler_ = error_handler_t::strict)
thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)), : o(std::move(s))
decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)), , loc(std::localeconv())
indent_char(ichar), indent_string(512, indent_char) , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep))
, decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point))
, indent_char(ichar)
, indent_string(512, indent_char)
, error_handler(error_handler_)
{} {}
// delete because of pointer members // delete because of pointer members
...@@ -287,6 +300,10 @@ class serializer ...@@ -287,6 +300,10 @@ class serializer
uint8_t state = UTF8_ACCEPT; uint8_t state = UTF8_ACCEPT;
std::size_t bytes = 0; // number of bytes written to string_buffer std::size_t bytes = 0; // number of bytes written to string_buffer
// number of bytes written at the point of the last valid byte
std::size_t bytes_after_last_accept = 0;
std::size_t undumped_chars = 0;
for (std::size_t i = 0; i < s.size(); ++i) for (std::size_t i = 0; i < s.size(); ++i)
{ {
const auto byte = static_cast<uint8_t>(s[i]); const auto byte = static_cast<uint8_t>(s[i]);
...@@ -384,14 +401,68 @@ class serializer ...@@ -384,14 +401,68 @@ class serializer
o->write_characters(string_buffer.data(), bytes); o->write_characters(string_buffer.data(), bytes);
bytes = 0; bytes = 0;
} }
// remember the byte position of this accept
bytes_after_last_accept = bytes;
undumped_chars = 0;
break; break;
} }
case UTF8_REJECT: // decode found invalid UTF-8 byte case UTF8_REJECT: // decode found invalid UTF-8 byte
{ {
std::string sn(3, '\0'); switch (error_handler)
snprintf(&sn[0], sn.size(), "%.2X", byte); {
JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); case error_handler_t::strict:
{
std::string sn(3, '\0');
snprintf(&sn[0], sn.size(), "%.2X", byte);
JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
}
case error_handler_t::ignore:
case error_handler_t::replace:
{
// in case we saw this character the first time, we
// would like to read it again, because the byte
// may be OK for itself, but just not OK for the
// previous sequence
if (undumped_chars > 0)
{
--i;
}
// reset length buffer to the last accepted index;
// thus removing/ignoring the invalid characters
bytes = bytes_after_last_accept;
if (error_handler == error_handler_t::replace)
{
// add a replacement character
if (ensure_ascii)
{
string_buffer[bytes++] = '\\';
string_buffer[bytes++] = 'u';
string_buffer[bytes++] = 'f';
string_buffer[bytes++] = 'f';
string_buffer[bytes++] = 'f';
string_buffer[bytes++] = 'd';
}
else
{
string_buffer[bytes++] = '\xEF';
string_buffer[bytes++] = '\xBF';
string_buffer[bytes++] = '\xBD';
}
bytes_after_last_accept = bytes;
}
undumped_chars = 0;
// continue processing the string
state = UTF8_ACCEPT;
continue;
}
}
} }
default: // decode found yet incomplete multi-byte code point default: // decode found yet incomplete multi-byte code point
...@@ -401,11 +472,13 @@ class serializer ...@@ -401,11 +472,13 @@ class serializer
// code point will not be escaped - copy byte to buffer // code point will not be escaped - copy byte to buffer
string_buffer[bytes++] = s[i]; string_buffer[bytes++] = s[i];
} }
++undumped_chars;
break; break;
} }
} }
} }
// we finished processing the string
if (JSON_LIKELY(state == UTF8_ACCEPT)) if (JSON_LIKELY(state == UTF8_ACCEPT))
{ {
// write buffer // write buffer
...@@ -417,9 +490,38 @@ class serializer ...@@ -417,9 +490,38 @@ class serializer
else else
{ {
// we finish reading, but do not accept: string was incomplete // we finish reading, but do not accept: string was incomplete
std::string sn(3, '\0'); switch (error_handler)
snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); {
JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); case error_handler_t::strict:
{
std::string sn(3, '\0');
snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
}
case error_handler_t::ignore:
{
// write all accepted bytes
o->write_characters(string_buffer.data(), bytes_after_last_accept);
break;
}
case error_handler_t::replace:
{
// write all accepted bytes
o->write_characters(string_buffer.data(), bytes_after_last_accept);
// add a replacement character
if (ensure_ascii)
{
o->write_characters("\\ufffd", 6);
}
else
{
o->write_characters("\xEF\xBF\xBD", 3);
}
break;
}
}
} }
} }
...@@ -629,6 +731,9 @@ class serializer ...@@ -629,6 +731,9 @@ class serializer
const char indent_char; const char indent_char;
/// the indentation string /// the indentation string
string_t indent_string; string_t indent_string;
/// error_handler how to react on decoding errors
const error_handler_t error_handler;
}; };
} // namespace detail } // namespace detail
} // namespace nlohmann } // namespace nlohmann
...@@ -208,6 +208,8 @@ class basic_json ...@@ -208,6 +208,8 @@ class basic_json
using json_pointer = ::nlohmann::json_pointer<basic_json>; using json_pointer = ::nlohmann::json_pointer<basic_json>;
template<typename T, typename SFINAE> template<typename T, typename SFINAE>
using json_serializer = JSONSerializer<T, SFINAE>; using json_serializer = JSONSerializer<T, SFINAE>;
/// how to treat decoding errors
using error_handler_t = detail::error_handler_t;
/// helper type for initializer lists of basic_json values /// helper type for initializer lists of basic_json values
using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
...@@ -1932,6 +1934,10 @@ class basic_json ...@@ -1932,6 +1934,10 @@ class basic_json
@param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
in the output are escaped with `\uXXXX` sequences, and the result consists in the output are escaped with `\uXXXX` sequences, and the result consists
of ASCII characters only. of ASCII characters only.
@param[in] error_handler how to react on decoding errors; there are three
possible values: `strict` (throws and exception in case a decoding error
occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD),
and `ignore` (ignore invalid UTF-8 sequences during serialization).
@return string containing the serialization of the JSON value @return string containing the serialization of the JSON value
...@@ -1950,13 +1956,16 @@ class basic_json ...@@ -1950,13 +1956,16 @@ class basic_json
@see https://docs.python.org/2/library/json.html#json.dump @see https://docs.python.org/2/library/json.html#json.dump
@since version 1.0.0; indentation character @a indent_char, option @since version 1.0.0; indentation character @a indent_char, option
@a ensure_ascii and exceptions added in version 3.0.0 @a ensure_ascii and exceptions added in version 3.0.0; error
handlers added in version 3.4.0.
*/ */
string_t dump(const int indent = -1, const char indent_char = ' ', string_t dump(const int indent = -1,
const bool ensure_ascii = false) const const char indent_char = ' ',
const bool ensure_ascii = false,
const error_handler_t error_handler = error_handler_t::strict) const
{ {
string_t result; string_t result;
serializer s(detail::output_adapter<char, string_t>(result), indent_char); serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
if (indent >= 0) if (indent >= 0)
{ {
......
...@@ -10122,6 +10122,14 @@ namespace detail ...@@ -10122,6 +10122,14 @@ namespace detail
// serialization // // serialization //
/////////////////// ///////////////////
/// how to treat decoding errors
enum class error_handler_t
{
strict, ///< throw a type_error exception in case of invalid UTF-8
replace, ///< replace invalid UTF-8 sequences with U+FFFD
ignore ///< ignore invalid UTF-8 sequences
};
template<typename BasicJsonType> template<typename BasicJsonType>
class serializer class serializer
{ {
...@@ -10136,12 +10144,17 @@ class serializer ...@@ -10136,12 +10144,17 @@ class serializer
/*! /*!
@param[in] s output stream to serialize to @param[in] s output stream to serialize to
@param[in] ichar indentation character to use @param[in] ichar indentation character to use
*/ @param[in] error_handler_ how to react on decoding errors
serializer(output_adapter_t<char> s, const char ichar) */
: o(std::move(s)), loc(std::localeconv()), serializer(output_adapter_t<char> s, const char ichar,
thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)), error_handler_t error_handler_ = error_handler_t::strict)
decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)), : o(std::move(s))
indent_char(ichar), indent_string(512, indent_char) , loc(std::localeconv())
, thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep))
, decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point))
, indent_char(ichar)
, indent_string(512, indent_char)
, error_handler(error_handler_)
{} {}
// delete because of pointer members // delete because of pointer members
...@@ -10381,6 +10394,10 @@ class serializer ...@@ -10381,6 +10394,10 @@ class serializer
uint8_t state = UTF8_ACCEPT; uint8_t state = UTF8_ACCEPT;
std::size_t bytes = 0; // number of bytes written to string_buffer std::size_t bytes = 0; // number of bytes written to string_buffer
// number of bytes written at the point of the last valid byte
std::size_t bytes_after_last_accept = 0;
std::size_t undumped_chars = 0;
for (std::size_t i = 0; i < s.size(); ++i) for (std::size_t i = 0; i < s.size(); ++i)
{ {
const auto byte = static_cast<uint8_t>(s[i]); const auto byte = static_cast<uint8_t>(s[i]);
...@@ -10478,14 +10495,68 @@ class serializer ...@@ -10478,14 +10495,68 @@ class serializer
o->write_characters(string_buffer.data(), bytes); o->write_characters(string_buffer.data(), bytes);
bytes = 0; bytes = 0;
} }
// remember the byte position of this accept
bytes_after_last_accept = bytes;
undumped_chars = 0;
break; break;
} }
case UTF8_REJECT: // decode found invalid UTF-8 byte case UTF8_REJECT: // decode found invalid UTF-8 byte
{ {
std::string sn(3, '\0'); switch (error_handler)
snprintf(&sn[0], sn.size(), "%.2X", byte); {
JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); case error_handler_t::strict:
{
std::string sn(3, '\0');
snprintf(&sn[0], sn.size(), "%.2X", byte);
JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
}
case error_handler_t::ignore:
case error_handler_t::replace:
{
// in case we saw this character the first time, we
// would like to read it again, because the byte
// may be OK for itself, but just not OK for the
// previous sequence
if (undumped_chars > 0)
{
--i;
}
// reset length buffer to the last accepted index;
// thus removing/ignoring the invalid characters
bytes = bytes_after_last_accept;
if (error_handler == error_handler_t::replace)
{
// add a replacement character
if (ensure_ascii)
{
string_buffer[bytes++] = '\\';
string_buffer[bytes++] = 'u';
string_buffer[bytes++] = 'f';
string_buffer[bytes++] = 'f';
string_buffer[bytes++] = 'f';
string_buffer[bytes++] = 'd';
}
else
{
string_buffer[bytes++] = '\xEF';
string_buffer[bytes++] = '\xBF';
string_buffer[bytes++] = '\xBD';
}
bytes_after_last_accept = bytes;
}
undumped_chars = 0;
// continue processing the string
state = UTF8_ACCEPT;
continue;
}
}
} }
default: // decode found yet incomplete multi-byte code point default: // decode found yet incomplete multi-byte code point
...@@ -10495,11 +10566,13 @@ class serializer ...@@ -10495,11 +10566,13 @@ class serializer
// code point will not be escaped - copy byte to buffer // code point will not be escaped - copy byte to buffer
string_buffer[bytes++] = s[i]; string_buffer[bytes++] = s[i];
} }
++undumped_chars;
break; break;
} }
} }
} }
// we finished processing the string
if (JSON_LIKELY(state == UTF8_ACCEPT)) if (JSON_LIKELY(state == UTF8_ACCEPT))
{ {
// write buffer // write buffer
...@@ -10511,9 +10584,38 @@ class serializer ...@@ -10511,9 +10584,38 @@ class serializer
else else
{ {
// we finish reading, but do not accept: string was incomplete // we finish reading, but do not accept: string was incomplete
std::string sn(3, '\0'); switch (error_handler)
snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); {
JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); case error_handler_t::strict:
{
std::string sn(3, '\0');
snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
}
case error_handler_t::ignore:
{
// write all accepted bytes
o->write_characters(string_buffer.data(), bytes_after_last_accept);
break;
}
case error_handler_t::replace:
{
// write all accepted bytes
o->write_characters(string_buffer.data(), bytes_after_last_accept);
// add a replacement character
if (ensure_ascii)
{
o->write_characters("\\ufffd", 6);
}
else
{
o->write_characters("\xEF\xBF\xBD", 3);
}
break;
}
}
} }
} }
...@@ -10723,6 +10825,9 @@ class serializer ...@@ -10723,6 +10825,9 @@ class serializer
const char indent_char; const char indent_char;
/// the indentation string /// the indentation string
string_t indent_string; string_t indent_string;
/// error_handler how to react on decoding errors
const error_handler_t error_handler;
}; };
} // namespace detail } // namespace detail
} // namespace nlohmann } // namespace nlohmann
...@@ -11692,6 +11797,8 @@ class basic_json ...@@ -11692,6 +11797,8 @@ class basic_json
using json_pointer = ::nlohmann::json_pointer<basic_json>; using json_pointer = ::nlohmann::json_pointer<basic_json>;
template<typename T, typename SFINAE> template<typename T, typename SFINAE>
using json_serializer = JSONSerializer<T, SFINAE>; using json_serializer = JSONSerializer<T, SFINAE>;
/// how to treat decoding errors
using error_handler_t = detail::error_handler_t;
/// helper type for initializer lists of basic_json values /// helper type for initializer lists of basic_json values
using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
...@@ -13416,6 +13523,10 @@ class basic_json ...@@ -13416,6 +13523,10 @@ class basic_json
@param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
in the output are escaped with `\uXXXX` sequences, and the result consists in the output are escaped with `\uXXXX` sequences, and the result consists
of ASCII characters only. of ASCII characters only.
@param[in] error_handler how to react on decoding errors; there are three
possible values: `strict` (throws and exception in case a decoding error
occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD),
and `ignore` (ignore invalid UTF-8 sequences during serialization).
@return string containing the serialization of the JSON value @return string containing the serialization of the JSON value
...@@ -13434,13 +13545,16 @@ class basic_json ...@@ -13434,13 +13545,16 @@ class basic_json
@see https://docs.python.org/2/library/json.html#json.dump @see https://docs.python.org/2/library/json.html#json.dump
@since version 1.0.0; indentation character @a indent_char, option @since version 1.0.0; indentation character @a indent_char, option
@a ensure_ascii and exceptions added in version 3.0.0 @a ensure_ascii and exceptions added in version 3.0.0; error
handlers added in version 3.4.0.
*/ */
string_t dump(const int indent = -1, const char indent_char = ' ', string_t dump(const int indent = -1,
const bool ensure_ascii = false) const const char indent_char = ' ',
const bool ensure_ascii = false,
const error_handler_t error_handler = error_handler_t::strict) const
{ {
string_t result; string_t result;
serializer s(detail::output_adapter<char, string_t>(result), indent_char); serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
if (indent >= 0) if (indent >= 0)
{ {
......
...@@ -94,4 +94,80 @@ TEST_CASE("serialization") ...@@ -94,4 +94,80 @@ TEST_CASE("serialization")
"[\n\t\"foo\",\n\t1,\n\t2,\n\t3,\n\tfalse,\n\t{\n\t\t\"one\": 1\n\t}\n]"); "[\n\t\"foo\",\n\t1,\n\t2,\n\t3,\n\tfalse,\n\t{\n\t\t\"one\": 1\n\t}\n]");
} }
} }
SECTION("dump")
{
SECTION("invalid character")
{
json j = \xA9ü";
CHECK_THROWS_AS(j.dump(), json::type_error&);
CHECK_THROWS_WITH(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9");
CHECK_THROWS_AS(j.dump(1, ' ', false, json::error_handler_t::strict), json::type_error&);
CHECK_THROWS_WITH(j.dump(1, ' ', false, json::error_handler_t::strict), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"äü\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"ä\xEF\xBF\xBDü\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\"");
}
SECTION("ending with incomplete character")
{
json j = "123\xC2";
CHECK_THROWS_AS(j.dump(), json::type_error&);
CHECK_THROWS_WITH(j.dump(), "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0xC2");
CHECK_THROWS_AS(j.dump(1, ' ', false, json::error_handler_t::strict), json::type_error&);
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd\"");
}
SECTION("unexpected character")
{
json j = "123\xF1\xB0\x34\x35\x36";
CHECK_THROWS_AS(j.dump(), json::type_error&);
CHECK_THROWS_WITH(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0x34");
CHECK_THROWS_AS(j.dump(1, ' ', false, json::error_handler_t::strict), json::type_error&);
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123456\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\x34\x35\x36\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd456\"");
}
SECTION("U+FFFD Substitution of Maximal Subparts")
{
// Some tests (mostly) from
// https://www.unicode.org/versions/Unicode11.0.0/ch03.pdf
// Section 3.9 -- U+FFFD Substitution of Maximal Subparts
auto test = [&](std::string const & input, std::string const & expected)
{
json j = input;
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"" + expected + "\"");
};
test("\xC2", "\\ufffd");
test("\xC2\x41\x42", "\\ufffd" "\x41" "\x42");
test("\xC2\xF4", "\\ufffd" "\\ufffd");
test("\xF0\x80\x80\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
test("\xF1\x80\x80\x41", "\\ufffd" "\x41");
test("\xF2\x80\x80\x41", "\\ufffd" "\x41");
test("\xF3\x80\x80\x41", "\\ufffd" "\x41");
test("\xF4\x80\x80\x41", "\\ufffd" "\x41");
test("\xF5\x80\x80\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
test("\xF0\x90\x80\x41", "\\ufffd" "\x41");
test("\xF1\x90\x80\x41", "\\ufffd" "\x41");
test("\xF2\x90\x80\x41", "\\ufffd" "\x41");
test("\xF3\x90\x80\x41", "\\ufffd" "\x41");
test("\xF4\x90\x80\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
test("\xF5\x90\x80\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
test("\xC0\xAF\xE0\x80\xBF\xF0\x81\x82\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
test("\xED\xA0\x80\xED\xBF\xBF\xED\xAF\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
test("\xF4\x91\x92\x93\xFF\x41\x80\xBF\x42", "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\x41" "\\ufffd""\\ufffd" "\x42");
test("\xE1\x80\xE2\xF0\x91\x92\xF1\xBF\x41", "\\ufffd" "\\ufffd" "\\ufffd" "\\ufffd" "\x41");
}
}
} }
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment