Commit 8d88a1da authored by Niels Lohmann's avatar Niels Lohmann

🔀 merged #379 and fixed conflicts

parents d284c86d c236b596
...@@ -10838,188 +10838,275 @@ basic_json_parser_66: ...@@ -10838,188 +10838,275 @@ basic_json_parser_66:
return result; return result;
} }
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate /*!
standard floating point number parsing function based on the type @brief parse string into a built-in arithmetic type as if
supplied via the first parameter. Set this to @a the current locale is POSIX.
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after note: in floating-point case strtod may parse
the number past the token's end - this is not an error.
@return the floating point number any leading blanks are not handled.
*/ */
long double str_to_float_t(long double* /* type */, char** endptr) const struct strtonum
{ {
return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); public:
} strtonum(const char* start, const char* end)
: m_start(start), m_end(end)
/*! {}
@brief parse floating point number
This function (and its overloads) serves to select the most approprate /// return true iff parsed successfully as
standard floating point number parsing function based on the type /// number of type T.
supplied via the first parameter. Set this to @a ///
static_cast<number_float_t*>(nullptr). /// @val shall contain parsed value, or
/// undefined value if could not parse.
template<typename T,
typename = typename std::enable_if<
std::is_arithmetic<T>::value>::type >
bool to(T& val) const
{
return parse(val, std::is_integral<T>());
}
@param[in,out] endptr recieves a pointer to the first character after /// return true iff token matches ^[+-]\d+$
the number ///
/// this is a helper to determine whether to
/// parse the token into floating-point or
/// integral type. We wouldn't need it if
/// we had separate token types for integral
/// and floating-point cases.
bool is_integral() const
{
const char* p = m_start;
@return the floating point number if (!p)
*/
double str_to_float_t(double* /* type */, char** endptr) const
{ {
return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); return false; // LCOV_EXCL_LINE
} }
/*! if (*p == '-' or * p == '+')
@brief parse floating point number {
++p;
This function (and its overloads) serves to select the most approprate }
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after if (p == m_end)
the number {
return false; // LCOV_EXCL_LINE
}
@return the floating point number while (p < m_end and* p >= '0'
*/ and * p <= '9')
float str_to_float_t(float* /* type */, char** endptr) const
{ {
return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); ++p;
} }
/*! return p == m_end;
@brief return number value for number tokens }
This function translates the last token into the most appropriate private:
number type (either integer, unsigned integer or floating point), const char* const m_start = nullptr;
which is passed back to the caller via the result parameter. const char* const m_end = nullptr;
This function parses the integer component up to the radix point or // overloaded wrappers for strtod/strtof/strtold
exponent while collecting information about the 'floating point // that will be called from parse<floating_point_t>
representation', which it stores in the result parameter. If there is
no radix point or exponent, and the number can fit into a @ref
number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly.
If the number is a floating point number the number is then parsed static void strtof(float& f,
using @a std:strtod (or @a std:strtof or @a std::strtold). const char* str,
char** endptr)
{
f = std::strtof(str, endptr);
}
@param[out] result @ref basic_json object to receive the number, or static void strtof(double& f,
NAN if the conversion read past the current token. The latter case const char* str,
needs to be treated by the caller function. char** endptr)
*/
void get_number(basic_json& result) const
{ {
assert(m_start != nullptr); f = std::strtod(str, endptr);
}
const lexer::lexer_char_t* curptr = m_start; static void strtof(long double& f,
const char* str,
char** endptr)
{
f = std::strtold(str, endptr);
}
// accumulate the integer conversion result (unsigned for now) template<typename T>
number_unsigned_t value = 0; bool parse(T& value, /*is_integral=*/std::false_type) const
{
// replace decimal separator with locale-specific
// version, when necessary; data will point to
// either the original string, or buf, or tempstr
// containing the fixed string.
std::string tempstr;
std::array<char, 64> buf;
const size_t len = static_cast<size_t>(m_end - m_start);
// maximum absolute value of the relevant integer type // Since dealing with strtod family of functions,
number_unsigned_t max; // we're getting the decimal point char from the
// C locale facilities instead of C++'s numpunct
// facet of the current std::locale;
const auto loc = localeconv();
assert(loc != nullptr);
const char decimal_point_char = !loc->decimal_point ? '.'
: loc->decimal_point[0];
// temporarily store the type to avoid unecessary bitfield access const char* data = m_start;
value_t type;
// look for sign if (decimal_point_char != '.')
if (*curptr == '-') {
const size_t ds_pos = static_cast<size_t>(
std::find(m_start, m_end, '.') - m_start );
if (ds_pos != len)
{
// copy the data into the local buffer or
// tempstr, if buffer is too small;
// replace decimal separator, and update
// data to point to the modified bytes
if (len + 1 < buf.size())
{ {
type = value_t::number_integer; std::copy(m_start, m_end, buf.data());
max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1; buf[len] = 0;
curptr++; buf[ds_pos] = decimal_point_char;
data = buf.data();
} }
else else
{ {
type = value_t::number_unsigned; tempstr.assign(m_start, m_end);
max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)()); tempstr[ds_pos] = decimal_point_char;
data = tempstr.c_str();
}
}
} }
// count the significant figures char* endptr = nullptr;
for (; curptr < m_cursor; curptr++) value = 0;
{ // this calls appropriate overload depending on T
// quickly skip tests if a digit strtof(value, data, &endptr);
if (*curptr < '0' or* curptr > '9')
{ // note that reading past the end is OK, the data may be,
if (*curptr == '.') // for example, "123.", where the parsed token only
// contains "123", but strtod will read the dot as well.
const bool ok = endptr >= data + len
and len > 0;
if (ok and value == 0.0 and * data == '-')
{ {
// don't count '.' but change to float // some implementations forget to negate the zero
type = value_t::number_float; value = -0.0;
continue;
} }
// assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details return ok;
type = value_t::number_float;
break;
} }
// skip if definitely not an integer signed long long parse_integral(
if (type != value_t::number_float) char** endptr,
/*is_signed*/std::true_type) const
{ {
auto digit = static_cast<number_unsigned_t>(*curptr - '0'); return std::strtoll(m_start, endptr, 10);
}
// overflow if value * 10 + digit > max, move terms around unsigned long long parse_integral(
// to avoid overflow in intermediate values char** endptr,
if (value > (max - digit) / 10) /*is_signed*/std::false_type) const
{ {
// overflow return std::strtoull(m_start, endptr, 10);
type = value_t::number_float;
} }
else
template<typename T>
bool parse(T& value, /*is_integral=*/std::true_type) const
{ {
// no overflow char* endptr = nullptr;
value = value * 10 + digit; errno = 0; // these are thread-local
} const auto x = parse_integral(&endptr, std::is_signed<T>());
}
static_assert(std::is_signed<T>() // called right overload?
== std::is_signed<decltype(x)>(), "");
value = static_cast<T>(x);
return x == static_cast<decltype(x)>(value) // x fits into destination T
and (x < 0) == (value < 0) // preserved sign
and (x != 0 or is_integral()) // strto[u]ll did nto fail
and errno == 0 // strto[u]ll did not overflow
and m_start < m_end // token was not empty
and endptr == m_end; // parsed entire token exactly
} }
};
/*!
@brief return number value for number tokens
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
// save the value (if not a float) integral numbers that don't fit into the the range of the respective
if (type == value_t::number_unsigned) type are parsed as number_float_t
floating-point values do not satisfy std::isfinite predicate
are converted to value_t::null
throws if the entire string [m_start .. m_cursor) cannot be
interpreted as a number
@param[out] result @ref basic_json object to receive the number.
*/
void get_number(basic_json& result) const
{
assert(m_start != nullptr);
assert(m_start < m_cursor);
strtonum num(reinterpret_cast<const char*>(m_start),
reinterpret_cast<const char*>(m_cursor));
const bool is_negative = *m_start == '-';
result.m_type = value_t::discarded;
if (not num.is_integral())
{ {
result.m_value.number_unsigned = value; ; // will parse as float below
} }
else if (type == value_t::number_integer) else if (is_negative)
{ {
// invariant: if we parsed a '-', the absolute value is between number_integer_t val{0};
// 0 (we allow -0) and max == -INT64_MIN if (num.to(val))
assert(value >= 0);
assert(value <= max);
if (value == max)
{ {
// we cannot simply negate value (== max == -INT64_MIN), result.m_type = value_t::number_integer;
// see https://github.com/nlohmann/json/issues/389 result.m_value = val;
result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN); }
} }
else else
{ {
// all other values can be negated safely number_unsigned_t val{0};
result.m_value.number_integer = -static_cast<number_integer_t>(value); if (num.to(val))
{
result.m_type = value_t::number_unsigned;
result.m_value = val;
} }
} }
else
number_float_t val{0};
if (result.m_type != value_t::discarded
or !num.to(val))
{ {
// parse with strtod return; // already have a value from above
result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), nullptr); // or couldn't parse as float_t
}
result.m_type = value_t::number_float;
result.m_value = val;
// replace infinity and NAN by null // replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float)) if (not std::isfinite(result.m_value.number_float))
{ {
type = value_t::null; result.m_type = value_t::null;
result.m_value = basic_json::json_value(); result.m_value = basic_json::json_value();
} }
} }
// save the type
result.m_type = type;
}
private: private:
/// optional input stream /// optional input stream
std::istream* m_stream = nullptr; std::istream* m_stream = nullptr;
......
...@@ -9988,188 +9988,275 @@ class basic_json ...@@ -9988,188 +9988,275 @@ class basic_json
return result; return result;
} }
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate /*!
standard floating point number parsing function based on the type @brief parse string into a built-in arithmetic type as if
supplied via the first parameter. Set this to @a the current locale is POSIX.
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after note: in floating-point case strtod may parse
the number past the token's end - this is not an error.
@return the floating point number any leading blanks are not handled.
*/ */
long double str_to_float_t(long double* /* type */, char** endptr) const struct strtonum
{ {
return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); public:
} strtonum(const char* start, const char* end)
: m_start(start), m_end(end)
/*! {}
@brief parse floating point number
This function (and its overloads) serves to select the most approprate /// return true iff parsed successfully as
standard floating point number parsing function based on the type /// number of type T.
supplied via the first parameter. Set this to @a ///
static_cast<number_float_t*>(nullptr). /// @val shall contain parsed value, or
/// undefined value if could not parse.
template<typename T,
typename = typename std::enable_if<
std::is_arithmetic<T>::value>::type >
bool to(T& val) const
{
return parse(val, std::is_integral<T>());
}
@param[in,out] endptr recieves a pointer to the first character after /// return true iff token matches ^[+-]\d+$
the number ///
/// this is a helper to determine whether to
/// parse the token into floating-point or
/// integral type. We wouldn't need it if
/// we had separate token types for integral
/// and floating-point cases.
bool is_integral() const
{
const char* p = m_start;
@return the floating point number if (!p)
*/
double str_to_float_t(double* /* type */, char** endptr) const
{ {
return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); return false; // LCOV_EXCL_LINE
} }
/*! if (*p == '-' or * p == '+')
@brief parse floating point number {
++p;
This function (and its overloads) serves to select the most approprate }
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after if (p == m_end)
the number {
return false; // LCOV_EXCL_LINE
}
@return the floating point number while (p < m_end and* p >= '0'
*/ and * p <= '9')
float str_to_float_t(float* /* type */, char** endptr) const
{ {
return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); ++p;
} }
/*! return p == m_end;
@brief return number value for number tokens }
This function translates the last token into the most appropriate private:
number type (either integer, unsigned integer or floating point), const char* const m_start = nullptr;
which is passed back to the caller via the result parameter. const char* const m_end = nullptr;
This function parses the integer component up to the radix point or // overloaded wrappers for strtod/strtof/strtold
exponent while collecting information about the 'floating point // that will be called from parse<floating_point_t>
representation', which it stores in the result parameter. If there is
no radix point or exponent, and the number can fit into a @ref
number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly.
If the number is a floating point number the number is then parsed static void strtof(float& f,
using @a std:strtod (or @a std:strtof or @a std::strtold). const char* str,
char** endptr)
{
f = std::strtof(str, endptr);
}
@param[out] result @ref basic_json object to receive the number, or static void strtof(double& f,
NAN if the conversion read past the current token. The latter case const char* str,
needs to be treated by the caller function. char** endptr)
*/
void get_number(basic_json& result) const
{ {
assert(m_start != nullptr); f = std::strtod(str, endptr);
}
const lexer::lexer_char_t* curptr = m_start; static void strtof(long double& f,
const char* str,
char** endptr)
{
f = std::strtold(str, endptr);
}
// accumulate the integer conversion result (unsigned for now) template<typename T>
number_unsigned_t value = 0; bool parse(T& value, /*is_integral=*/std::false_type) const
{
// replace decimal separator with locale-specific
// version, when necessary; data will point to
// either the original string, or buf, or tempstr
// containing the fixed string.
std::string tempstr;
std::array<char, 64> buf;
const size_t len = static_cast<size_t>(m_end - m_start);
// maximum absolute value of the relevant integer type // Since dealing with strtod family of functions,
number_unsigned_t max; // we're getting the decimal point char from the
// C locale facilities instead of C++'s numpunct
// facet of the current std::locale;
const auto loc = localeconv();
assert(loc != nullptr);
const char decimal_point_char = !loc->decimal_point ? '.'
: loc->decimal_point[0];
// temporarily store the type to avoid unecessary bitfield access const char* data = m_start;
value_t type;
if (decimal_point_char != '.')
{
const size_t ds_pos = static_cast<size_t>(
std::find(m_start, m_end, '.') - m_start );
// look for sign if (ds_pos != len)
if (*curptr == '-')
{ {
type = value_t::number_integer; // copy the data into the local buffer or
max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1; // tempstr, if buffer is too small;
curptr++; // replace decimal separator, and update
// data to point to the modified bytes
if (len + 1 < buf.size())
{
std::copy(m_start, m_end, buf.data());
buf[len] = 0;
buf[ds_pos] = decimal_point_char;
data = buf.data();
} }
else else
{ {
type = value_t::number_unsigned; tempstr.assign(m_start, m_end);
max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)()); tempstr[ds_pos] = decimal_point_char;
data = tempstr.c_str();
}
}
} }
// count the significant figures char* endptr = nullptr;
for (; curptr < m_cursor; curptr++) value = 0;
{ // this calls appropriate overload depending on T
// quickly skip tests if a digit strtof(value, data, &endptr);
if (*curptr < '0' or* curptr > '9')
{ // note that reading past the end is OK, the data may be,
if (*curptr == '.') // for example, "123.", where the parsed token only
// contains "123", but strtod will read the dot as well.
const bool ok = endptr >= data + len
and len > 0;
if (ok and value == 0.0 and * data == '-')
{ {
// don't count '.' but change to float // some implementations forget to negate the zero
type = value_t::number_float; value = -0.0;
continue;
} }
// assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details return ok;
type = value_t::number_float;
break;
} }
// skip if definitely not an integer signed long long parse_integral(
if (type != value_t::number_float) char** endptr,
/*is_signed*/std::true_type) const
{ {
auto digit = static_cast<number_unsigned_t>(*curptr - '0'); return std::strtoll(m_start, endptr, 10);
}
// overflow if value * 10 + digit > max, move terms around unsigned long long parse_integral(
// to avoid overflow in intermediate values char** endptr,
if (value > (max - digit) / 10) /*is_signed*/std::false_type) const
{ {
// overflow return std::strtoull(m_start, endptr, 10);
type = value_t::number_float;
} }
else
template<typename T>
bool parse(T& value, /*is_integral=*/std::true_type) const
{ {
// no overflow char* endptr = nullptr;
value = value * 10 + digit; errno = 0; // these are thread-local
} const auto x = parse_integral(&endptr, std::is_signed<T>());
}
static_assert(std::is_signed<T>() // called right overload?
== std::is_signed<decltype(x)>(), "");
value = static_cast<T>(x);
return x == static_cast<decltype(x)>(value) // x fits into destination T
and (x < 0) == (value < 0) // preserved sign
and (x != 0 or is_integral()) // strto[u]ll did nto fail
and errno == 0 // strto[u]ll did not overflow
and m_start < m_end // token was not empty
and endptr == m_end; // parsed entire token exactly
} }
};
/*!
@brief return number value for number tokens
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
// save the value (if not a float) integral numbers that don't fit into the the range of the respective
if (type == value_t::number_unsigned) type are parsed as number_float_t
floating-point values do not satisfy std::isfinite predicate
are converted to value_t::null
throws if the entire string [m_start .. m_cursor) cannot be
interpreted as a number
@param[out] result @ref basic_json object to receive the number.
*/
void get_number(basic_json& result) const
{
assert(m_start != nullptr);
assert(m_start < m_cursor);
strtonum num(reinterpret_cast<const char*>(m_start),
reinterpret_cast<const char*>(m_cursor));
const bool is_negative = *m_start == '-';
result.m_type = value_t::discarded;
if (not num.is_integral())
{ {
result.m_value.number_unsigned = value; ; // will parse as float below
} }
else if (type == value_t::number_integer) else if (is_negative)
{ {
// invariant: if we parsed a '-', the absolute value is between number_integer_t val{0};
// 0 (we allow -0) and max == -INT64_MIN if (num.to(val))
assert(value >= 0);
assert(value <= max);
if (value == max)
{ {
// we cannot simply negate value (== max == -INT64_MIN), result.m_type = value_t::number_integer;
// see https://github.com/nlohmann/json/issues/389 result.m_value = val;
result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN); }
} }
else else
{ {
// all other values can be negated safely number_unsigned_t val{0};
result.m_value.number_integer = -static_cast<number_integer_t>(value); if (num.to(val))
{
result.m_type = value_t::number_unsigned;
result.m_value = val;
} }
} }
else
number_float_t val{0};
if (result.m_type != value_t::discarded
or !num.to(val))
{ {
// parse with strtod return; // already have a value from above
result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), nullptr); // or couldn't parse as float_t
}
result.m_type = value_t::number_float;
result.m_value = val;
// replace infinity and NAN by null // replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float)) if (not std::isfinite(result.m_value.number_float))
{ {
type = value_t::null; result.m_type = value_t::null;
result.m_value = basic_json::json_value(); result.m_value = basic_json::json_value();
} }
} }
// save the type
result.m_type = type;
}
private: private:
/// optional input stream /// optional input stream
std::istream* m_stream = nullptr; std::istream* m_stream = nullptr;
......
...@@ -383,7 +383,7 @@ TEST_CASE("regression tests") ...@@ -383,7 +383,7 @@ TEST_CASE("regression tests")
}; };
// change locale to mess with decimal points // change locale to mess with decimal points
std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator)); auto orig_locale = std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator));
CHECK(j1a.dump() == "23.42"); CHECK(j1a.dump() == "23.42");
CHECK(j1b.dump() == "23.42"); CHECK(j1b.dump() == "23.42");
...@@ -407,8 +407,34 @@ TEST_CASE("regression tests") ...@@ -407,8 +407,34 @@ TEST_CASE("regression tests")
CHECK(j3c.dump() == "10000"); CHECK(j3c.dump() == "10000");
//CHECK(j3b.dump() == "1E04"); // roundtrip error //CHECK(j3b.dump() == "1E04"); // roundtrip error
//CHECK(j3c.dump() == "1e04"); // roundtrip error //CHECK(j3c.dump() == "1e04"); // roundtrip error
std::locale::global(orig_locale);
} }
SECTION("issue #379 - locale-independent str-to-num")
{
setlocale(LC_NUMERIC, "de_DE.UTF-8");
// disabled, because locale-specific beharivor is not
// triggered in AppVeyor for some reason
#if 0
{
// verify that strtod now uses commas as decimal-separator
CHECK(std::strtod("3,14", nullptr) == 3.14);
// verify that strtod does not understand dots as decimal separator
CHECK(std::strtod("3.14", nullptr) == 3);
}
#endif
// verify that parsed correctly despite using strtod internally
CHECK(json::parse("3.14").get<double>() == 3.14);
// check a different code path
CHECK(json::parse("1.000000000000000000000000000000000000000000000000000000000000000000000000").get<double>() == 1.0);
}
SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator") SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator")
{ {
json source = {"a", "b", "c"}; json source = {"a", "b", "c"};
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment