Commit 8d88a1da authored by Niels Lohmann's avatar Niels Lohmann

🔀 merged #379 and fixed conflicts

parents d284c86d c236b596
...@@ -10838,186 +10838,273 @@ basic_json_parser_66: ...@@ -10838,186 +10838,273 @@ basic_json_parser_66:
return result; return result;
} }
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after
the number
@return the floating point number
*/
long double str_to_float_t(long double* /* type */, char** endptr) const
{
return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
}
/*! /*!
@brief parse floating point number @brief parse string into a built-in arithmetic type as if
the current locale is POSIX.
This function (and its overloads) serves to select the most approprate note: in floating-point case strtod may parse
standard floating point number parsing function based on the type past the token's end - this is not an error.
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after any leading blanks are not handled.
the number
@return the floating point number
*/ */
double str_to_float_t(double* /* type */, char** endptr) const struct strtonum
{ {
return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); public:
} strtonum(const char* start, const char* end)
: m_start(start), m_end(end)
/*! {}
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after
the number
@return the floating point number /// return true iff parsed successfully as
*/ /// number of type T.
float str_to_float_t(float* /* type */, char** endptr) const ///
{ /// @val shall contain parsed value, or
return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); /// undefined value if could not parse.
} template<typename T,
typename = typename std::enable_if<
std::is_arithmetic<T>::value>::type >
bool to(T& val) const
{
return parse(val, std::is_integral<T>());
}
/*! /// return true iff token matches ^[+-]\d+$
@brief return number value for number tokens ///
/// this is a helper to determine whether to
/// parse the token into floating-point or
/// integral type. We wouldn't need it if
/// we had separate token types for integral
/// and floating-point cases.
bool is_integral() const
{
const char* p = m_start;
This function translates the last token into the most appropriate if (!p)
number type (either integer, unsigned integer or floating point), {
which is passed back to the caller via the result parameter. return false; // LCOV_EXCL_LINE
}
This function parses the integer component up to the radix point or if (*p == '-' or * p == '+')
exponent while collecting information about the 'floating point {
representation', which it stores in the result parameter. If there is ++p;
no radix point or exponent, and the number can fit into a @ref }
number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly.
If the number is a floating point number the number is then parsed if (p == m_end)
using @a std:strtod (or @a std:strtof or @a std::strtold). {
return false; // LCOV_EXCL_LINE
}
@param[out] result @ref basic_json object to receive the number, or while (p < m_end and* p >= '0'
NAN if the conversion read past the current token. The latter case and * p <= '9')
needs to be treated by the caller function. {
*/ ++p;
void get_number(basic_json& result) const }
{
assert(m_start != nullptr);
const lexer::lexer_char_t* curptr = m_start; return p == m_end;
}
// accumulate the integer conversion result (unsigned for now) private:
number_unsigned_t value = 0; const char* const m_start = nullptr;
const char* const m_end = nullptr;
// maximum absolute value of the relevant integer type // overloaded wrappers for strtod/strtof/strtold
number_unsigned_t max; // that will be called from parse<floating_point_t>
// temporarily store the type to avoid unecessary bitfield access static void strtof(float& f,
value_t type; const char* str,
char** endptr)
{
f = std::strtof(str, endptr);
}
// look for sign static void strtof(double& f,
if (*curptr == '-') const char* str,
char** endptr)
{ {
type = value_t::number_integer; f = std::strtod(str, endptr);
max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1;
curptr++;
} }
else
static void strtof(long double& f,
const char* str,
char** endptr)
{ {
type = value_t::number_unsigned; f = std::strtold(str, endptr);
max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)());
} }
// count the significant figures template<typename T>
for (; curptr < m_cursor; curptr++) bool parse(T& value, /*is_integral=*/std::false_type) const
{ {
// quickly skip tests if a digit // replace decimal separator with locale-specific
if (*curptr < '0' or* curptr > '9') // version, when necessary; data will point to
// either the original string, or buf, or tempstr
// containing the fixed string.
std::string tempstr;
std::array<char, 64> buf;
const size_t len = static_cast<size_t>(m_end - m_start);
// Since dealing with strtod family of functions,
// we're getting the decimal point char from the
// C locale facilities instead of C++'s numpunct
// facet of the current std::locale;
const auto loc = localeconv();
assert(loc != nullptr);
const char decimal_point_char = !loc->decimal_point ? '.'
: loc->decimal_point[0];
const char* data = m_start;
if (decimal_point_char != '.')
{ {
if (*curptr == '.') const size_t ds_pos = static_cast<size_t>(
std::find(m_start, m_end, '.') - m_start );
if (ds_pos != len)
{ {
// don't count '.' but change to float // copy the data into the local buffer or
type = value_t::number_float; // tempstr, if buffer is too small;
continue; // replace decimal separator, and update
// data to point to the modified bytes
if (len + 1 < buf.size())
{
std::copy(m_start, m_end, buf.data());
buf[len] = 0;
buf[ds_pos] = decimal_point_char;
data = buf.data();
}
else
{
tempstr.assign(m_start, m_end);
tempstr[ds_pos] = decimal_point_char;
data = tempstr.c_str();
}
} }
// assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details
type = value_t::number_float;
break;
} }
// skip if definitely not an integer char* endptr = nullptr;
if (type != value_t::number_float) value = 0;
{ // this calls appropriate overload depending on T
auto digit = static_cast<number_unsigned_t>(*curptr - '0'); strtof(value, data, &endptr);
// overflow if value * 10 + digit > max, move terms around // note that reading past the end is OK, the data may be,
// to avoid overflow in intermediate values // for example, "123.", where the parsed token only
if (value > (max - digit) / 10) // contains "123", but strtod will read the dot as well.
{ const bool ok = endptr >= data + len
// overflow and len > 0;
type = value_t::number_float;
} if (ok and value == 0.0 and * data == '-')
else {
{ // some implementations forget to negate the zero
// no overflow value = -0.0;
value = value * 10 + digit;
}
} }
return ok;
}
signed long long parse_integral(
char** endptr,
/*is_signed*/std::true_type) const
{
return std::strtoll(m_start, endptr, 10);
} }
// save the value (if not a float) unsigned long long parse_integral(
if (type == value_t::number_unsigned) char** endptr,
/*is_signed*/std::false_type) const
{ {
result.m_value.number_unsigned = value; return std::strtoull(m_start, endptr, 10);
} }
else if (type == value_t::number_integer)
template<typename T>
bool parse(T& value, /*is_integral=*/std::true_type) const
{ {
// invariant: if we parsed a '-', the absolute value is between char* endptr = nullptr;
// 0 (we allow -0) and max == -INT64_MIN errno = 0; // these are thread-local
assert(value >= 0); const auto x = parse_integral(&endptr, std::is_signed<T>());
assert(value <= max);
if (value == max) static_assert(std::is_signed<T>() // called right overload?
{ == std::is_signed<decltype(x)>(), "");
// we cannot simply negate value (== max == -INT64_MIN),
// see https://github.com/nlohmann/json/issues/389 value = static_cast<T>(x);
result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN);
} return x == static_cast<decltype(x)>(value) // x fits into destination T
else and (x < 0) == (value < 0) // preserved sign
and (x != 0 or is_integral()) // strto[u]ll did nto fail
and errno == 0 // strto[u]ll did not overflow
and m_start < m_end // token was not empty
and endptr == m_end; // parsed entire token exactly
}
};
/*!
@brief return number value for number tokens
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
integral numbers that don't fit into the the range of the respective
type are parsed as number_float_t
floating-point values do not satisfy std::isfinite predicate
are converted to value_t::null
throws if the entire string [m_start .. m_cursor) cannot be
interpreted as a number
@param[out] result @ref basic_json object to receive the number.
*/
void get_number(basic_json& result) const
{
assert(m_start != nullptr);
assert(m_start < m_cursor);
strtonum num(reinterpret_cast<const char*>(m_start),
reinterpret_cast<const char*>(m_cursor));
const bool is_negative = *m_start == '-';
result.m_type = value_t::discarded;
if (not num.is_integral())
{
; // will parse as float below
}
else if (is_negative)
{
number_integer_t val{0};
if (num.to(val))
{ {
// all other values can be negated safely result.m_type = value_t::number_integer;
result.m_value.number_integer = -static_cast<number_integer_t>(value); result.m_value = val;
} }
} }
else else
{ {
// parse with strtod number_unsigned_t val{0};
result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), nullptr); if (num.to(val))
// replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float))
{ {
type = value_t::null; result.m_type = value_t::number_unsigned;
result.m_value = basic_json::json_value(); result.m_value = val;
} }
} }
// save the type number_float_t val{0};
result.m_type = type; if (result.m_type != value_t::discarded
or !num.to(val))
{
return; // already have a value from above
// or couldn't parse as float_t
}
result.m_type = value_t::number_float;
result.m_value = val;
// replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float))
{
result.m_type = value_t::null;
result.m_value = basic_json::json_value();
}
} }
private: private:
......
...@@ -9988,186 +9988,273 @@ class basic_json ...@@ -9988,186 +9988,273 @@ class basic_json
return result; return result;
} }
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after
the number
@return the floating point number
*/
long double str_to_float_t(long double* /* type */, char** endptr) const
{
return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
}
/*! /*!
@brief parse floating point number @brief parse string into a built-in arithmetic type as if
the current locale is POSIX.
This function (and its overloads) serves to select the most approprate note: in floating-point case strtod may parse
standard floating point number parsing function based on the type past the token's end - this is not an error.
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after any leading blanks are not handled.
the number
@return the floating point number
*/ */
double str_to_float_t(double* /* type */, char** endptr) const struct strtonum
{ {
return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); public:
} strtonum(const char* start, const char* end)
: m_start(start), m_end(end)
/*! {}
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
@param[in,out] endptr recieves a pointer to the first character after
the number
@return the floating point number /// return true iff parsed successfully as
*/ /// number of type T.
float str_to_float_t(float* /* type */, char** endptr) const ///
{ /// @val shall contain parsed value, or
return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); /// undefined value if could not parse.
} template<typename T,
typename = typename std::enable_if<
std::is_arithmetic<T>::value>::type >
bool to(T& val) const
{
return parse(val, std::is_integral<T>());
}
/*! /// return true iff token matches ^[+-]\d+$
@brief return number value for number tokens ///
/// this is a helper to determine whether to
/// parse the token into floating-point or
/// integral type. We wouldn't need it if
/// we had separate token types for integral
/// and floating-point cases.
bool is_integral() const
{
const char* p = m_start;
This function translates the last token into the most appropriate if (!p)
number type (either integer, unsigned integer or floating point), {
which is passed back to the caller via the result parameter. return false; // LCOV_EXCL_LINE
}
This function parses the integer component up to the radix point or if (*p == '-' or * p == '+')
exponent while collecting information about the 'floating point {
representation', which it stores in the result parameter. If there is ++p;
no radix point or exponent, and the number can fit into a @ref }
number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly.
If the number is a floating point number the number is then parsed if (p == m_end)
using @a std:strtod (or @a std:strtof or @a std::strtold). {
return false; // LCOV_EXCL_LINE
}
@param[out] result @ref basic_json object to receive the number, or while (p < m_end and* p >= '0'
NAN if the conversion read past the current token. The latter case and * p <= '9')
needs to be treated by the caller function. {
*/ ++p;
void get_number(basic_json& result) const }
{
assert(m_start != nullptr);
const lexer::lexer_char_t* curptr = m_start; return p == m_end;
}
// accumulate the integer conversion result (unsigned for now) private:
number_unsigned_t value = 0; const char* const m_start = nullptr;
const char* const m_end = nullptr;
// maximum absolute value of the relevant integer type // overloaded wrappers for strtod/strtof/strtold
number_unsigned_t max; // that will be called from parse<floating_point_t>
// temporarily store the type to avoid unecessary bitfield access static void strtof(float& f,
value_t type; const char* str,
char** endptr)
{
f = std::strtof(str, endptr);
}
// look for sign static void strtof(double& f,
if (*curptr == '-') const char* str,
char** endptr)
{ {
type = value_t::number_integer; f = std::strtod(str, endptr);
max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1;
curptr++;
} }
else
static void strtof(long double& f,
const char* str,
char** endptr)
{ {
type = value_t::number_unsigned; f = std::strtold(str, endptr);
max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)());
} }
// count the significant figures template<typename T>
for (; curptr < m_cursor; curptr++) bool parse(T& value, /*is_integral=*/std::false_type) const
{ {
// quickly skip tests if a digit // replace decimal separator with locale-specific
if (*curptr < '0' or* curptr > '9') // version, when necessary; data will point to
// either the original string, or buf, or tempstr
// containing the fixed string.
std::string tempstr;
std::array<char, 64> buf;
const size_t len = static_cast<size_t>(m_end - m_start);
// Since dealing with strtod family of functions,
// we're getting the decimal point char from the
// C locale facilities instead of C++'s numpunct
// facet of the current std::locale;
const auto loc = localeconv();
assert(loc != nullptr);
const char decimal_point_char = !loc->decimal_point ? '.'
: loc->decimal_point[0];
const char* data = m_start;
if (decimal_point_char != '.')
{ {
if (*curptr == '.') const size_t ds_pos = static_cast<size_t>(
std::find(m_start, m_end, '.') - m_start );
if (ds_pos != len)
{ {
// don't count '.' but change to float // copy the data into the local buffer or
type = value_t::number_float; // tempstr, if buffer is too small;
continue; // replace decimal separator, and update
// data to point to the modified bytes
if (len + 1 < buf.size())
{
std::copy(m_start, m_end, buf.data());
buf[len] = 0;
buf[ds_pos] = decimal_point_char;
data = buf.data();
}
else
{
tempstr.assign(m_start, m_end);
tempstr[ds_pos] = decimal_point_char;
data = tempstr.c_str();
}
} }
// assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details
type = value_t::number_float;
break;
} }
// skip if definitely not an integer char* endptr = nullptr;
if (type != value_t::number_float) value = 0;
{ // this calls appropriate overload depending on T
auto digit = static_cast<number_unsigned_t>(*curptr - '0'); strtof(value, data, &endptr);
// overflow if value * 10 + digit > max, move terms around // note that reading past the end is OK, the data may be,
// to avoid overflow in intermediate values // for example, "123.", where the parsed token only
if (value > (max - digit) / 10) // contains "123", but strtod will read the dot as well.
{ const bool ok = endptr >= data + len
// overflow and len > 0;
type = value_t::number_float;
} if (ok and value == 0.0 and * data == '-')
else {
{ // some implementations forget to negate the zero
// no overflow value = -0.0;
value = value * 10 + digit;
}
} }
return ok;
}
signed long long parse_integral(
char** endptr,
/*is_signed*/std::true_type) const
{
return std::strtoll(m_start, endptr, 10);
} }
// save the value (if not a float) unsigned long long parse_integral(
if (type == value_t::number_unsigned) char** endptr,
/*is_signed*/std::false_type) const
{ {
result.m_value.number_unsigned = value; return std::strtoull(m_start, endptr, 10);
} }
else if (type == value_t::number_integer)
template<typename T>
bool parse(T& value, /*is_integral=*/std::true_type) const
{ {
// invariant: if we parsed a '-', the absolute value is between char* endptr = nullptr;
// 0 (we allow -0) and max == -INT64_MIN errno = 0; // these are thread-local
assert(value >= 0); const auto x = parse_integral(&endptr, std::is_signed<T>());
assert(value <= max);
if (value == max) static_assert(std::is_signed<T>() // called right overload?
{ == std::is_signed<decltype(x)>(), "");
// we cannot simply negate value (== max == -INT64_MIN),
// see https://github.com/nlohmann/json/issues/389 value = static_cast<T>(x);
result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN);
} return x == static_cast<decltype(x)>(value) // x fits into destination T
else and (x < 0) == (value < 0) // preserved sign
and (x != 0 or is_integral()) // strto[u]ll did nto fail
and errno == 0 // strto[u]ll did not overflow
and m_start < m_end // token was not empty
and endptr == m_end; // parsed entire token exactly
}
};
/*!
@brief return number value for number tokens
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
integral numbers that don't fit into the the range of the respective
type are parsed as number_float_t
floating-point values do not satisfy std::isfinite predicate
are converted to value_t::null
throws if the entire string [m_start .. m_cursor) cannot be
interpreted as a number
@param[out] result @ref basic_json object to receive the number.
*/
void get_number(basic_json& result) const
{
assert(m_start != nullptr);
assert(m_start < m_cursor);
strtonum num(reinterpret_cast<const char*>(m_start),
reinterpret_cast<const char*>(m_cursor));
const bool is_negative = *m_start == '-';
result.m_type = value_t::discarded;
if (not num.is_integral())
{
; // will parse as float below
}
else if (is_negative)
{
number_integer_t val{0};
if (num.to(val))
{ {
// all other values can be negated safely result.m_type = value_t::number_integer;
result.m_value.number_integer = -static_cast<number_integer_t>(value); result.m_value = val;
} }
} }
else else
{ {
// parse with strtod number_unsigned_t val{0};
result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), nullptr); if (num.to(val))
// replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float))
{ {
type = value_t::null; result.m_type = value_t::number_unsigned;
result.m_value = basic_json::json_value(); result.m_value = val;
} }
} }
// save the type number_float_t val{0};
result.m_type = type; if (result.m_type != value_t::discarded
or !num.to(val))
{
return; // already have a value from above
// or couldn't parse as float_t
}
result.m_type = value_t::number_float;
result.m_value = val;
// replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float))
{
result.m_type = value_t::null;
result.m_value = basic_json::json_value();
}
} }
private: private:
......
...@@ -383,7 +383,7 @@ TEST_CASE("regression tests") ...@@ -383,7 +383,7 @@ TEST_CASE("regression tests")
}; };
// change locale to mess with decimal points // change locale to mess with decimal points
std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator)); auto orig_locale = std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator));
CHECK(j1a.dump() == "23.42"); CHECK(j1a.dump() == "23.42");
CHECK(j1b.dump() == "23.42"); CHECK(j1b.dump() == "23.42");
...@@ -407,8 +407,34 @@ TEST_CASE("regression tests") ...@@ -407,8 +407,34 @@ TEST_CASE("regression tests")
CHECK(j3c.dump() == "10000"); CHECK(j3c.dump() == "10000");
//CHECK(j3b.dump() == "1E04"); // roundtrip error //CHECK(j3b.dump() == "1E04"); // roundtrip error
//CHECK(j3c.dump() == "1e04"); // roundtrip error //CHECK(j3c.dump() == "1e04"); // roundtrip error
std::locale::global(orig_locale);
} }
SECTION("issue #379 - locale-independent str-to-num")
{
setlocale(LC_NUMERIC, "de_DE.UTF-8");
// disabled, because locale-specific beharivor is not
// triggered in AppVeyor for some reason
#if 0
{
// verify that strtod now uses commas as decimal-separator
CHECK(std::strtod("3,14", nullptr) == 3.14);
// verify that strtod does not understand dots as decimal separator
CHECK(std::strtod("3.14", nullptr) == 3);
}
#endif
// verify that parsed correctly despite using strtod internally
CHECK(json::parse("3.14").get<double>() == 3.14);
// check a different code path
CHECK(json::parse("1.000000000000000000000000000000000000000000000000000000000000000000000000").get<double>() == 1.0);
}
SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator") SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator")
{ {
json source = {"a", "b", "c"}; json source = {"a", "b", "c"};
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment