Commit 8d88a1da authored by Niels Lohmann's avatar Niels Lohmann

🔀 merged #379 and fixed conflicts

parents d284c86d c236b596
......@@ -10838,188 +10838,275 @@ basic_json_parser_66:
return result;
}
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
/*!
@brief parse string into a built-in arithmetic type as if
the current locale is POSIX.
@param[in,out] endptr recieves a pointer to the first character after
the number
note: in floating-point case strtod may parse
past the token's end - this is not an error.
@return the floating point number
any leading blanks are not handled.
*/
long double str_to_float_t(long double* /* type */, char** endptr) const
struct strtonum
{
return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
}
/*!
@brief parse floating point number
public:
strtonum(const char* start, const char* end)
: m_start(start), m_end(end)
{}
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
/// return true iff parsed successfully as
/// number of type T.
///
/// @val shall contain parsed value, or
/// undefined value if could not parse.
template<typename T,
typename = typename std::enable_if<
std::is_arithmetic<T>::value>::type >
bool to(T& val) const
{
return parse(val, std::is_integral<T>());
}
@param[in,out] endptr recieves a pointer to the first character after
the number
/// return true iff token matches ^[+-]\d+$
///
/// this is a helper to determine whether to
/// parse the token into floating-point or
/// integral type. We wouldn't need it if
/// we had separate token types for integral
/// and floating-point cases.
bool is_integral() const
{
const char* p = m_start;
@return the floating point number
*/
double str_to_float_t(double* /* type */, char** endptr) const
if (!p)
{
return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
return false; // LCOV_EXCL_LINE
}
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
if (*p == '-' or * p == '+')
{
++p;
}
@param[in,out] endptr recieves a pointer to the first character after
the number
if (p == m_end)
{
return false; // LCOV_EXCL_LINE
}
@return the floating point number
*/
float str_to_float_t(float* /* type */, char** endptr) const
while (p < m_end and* p >= '0'
and * p <= '9')
{
return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
++p;
}
/*!
@brief return number value for number tokens
return p == m_end;
}
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
private:
const char* const m_start = nullptr;
const char* const m_end = nullptr;
This function parses the integer component up to the radix point or
exponent while collecting information about the 'floating point
representation', which it stores in the result parameter. If there is
no radix point or exponent, and the number can fit into a @ref
number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly.
// overloaded wrappers for strtod/strtof/strtold
// that will be called from parse<floating_point_t>
If the number is a floating point number the number is then parsed
using @a std:strtod (or @a std:strtof or @a std::strtold).
static void strtof(float& f,
const char* str,
char** endptr)
{
f = std::strtof(str, endptr);
}
@param[out] result @ref basic_json object to receive the number, or
NAN if the conversion read past the current token. The latter case
needs to be treated by the caller function.
*/
void get_number(basic_json& result) const
static void strtof(double& f,
const char* str,
char** endptr)
{
assert(m_start != nullptr);
f = std::strtod(str, endptr);
}
const lexer::lexer_char_t* curptr = m_start;
static void strtof(long double& f,
const char* str,
char** endptr)
{
f = std::strtold(str, endptr);
}
// accumulate the integer conversion result (unsigned for now)
number_unsigned_t value = 0;
template<typename T>
bool parse(T& value, /*is_integral=*/std::false_type) const
{
// replace decimal separator with locale-specific
// version, when necessary; data will point to
// either the original string, or buf, or tempstr
// containing the fixed string.
std::string tempstr;
std::array<char, 64> buf;
const size_t len = static_cast<size_t>(m_end - m_start);
// maximum absolute value of the relevant integer type
number_unsigned_t max;
// Since dealing with strtod family of functions,
// we're getting the decimal point char from the
// C locale facilities instead of C++'s numpunct
// facet of the current std::locale;
const auto loc = localeconv();
assert(loc != nullptr);
const char decimal_point_char = !loc->decimal_point ? '.'
: loc->decimal_point[0];
// temporarily store the type to avoid unecessary bitfield access
value_t type;
const char* data = m_start;
// look for sign
if (*curptr == '-')
if (decimal_point_char != '.')
{
const size_t ds_pos = static_cast<size_t>(
std::find(m_start, m_end, '.') - m_start );
if (ds_pos != len)
{
// copy the data into the local buffer or
// tempstr, if buffer is too small;
// replace decimal separator, and update
// data to point to the modified bytes
if (len + 1 < buf.size())
{
type = value_t::number_integer;
max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1;
curptr++;
std::copy(m_start, m_end, buf.data());
buf[len] = 0;
buf[ds_pos] = decimal_point_char;
data = buf.data();
}
else
{
type = value_t::number_unsigned;
max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)());
tempstr.assign(m_start, m_end);
tempstr[ds_pos] = decimal_point_char;
data = tempstr.c_str();
}
}
}
// count the significant figures
for (; curptr < m_cursor; curptr++)
{
// quickly skip tests if a digit
if (*curptr < '0' or* curptr > '9')
{
if (*curptr == '.')
char* endptr = nullptr;
value = 0;
// this calls appropriate overload depending on T
strtof(value, data, &endptr);
// note that reading past the end is OK, the data may be,
// for example, "123.", where the parsed token only
// contains "123", but strtod will read the dot as well.
const bool ok = endptr >= data + len
and len > 0;
if (ok and value == 0.0 and * data == '-')
{
// don't count '.' but change to float
type = value_t::number_float;
continue;
// some implementations forget to negate the zero
value = -0.0;
}
// assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details
type = value_t::number_float;
break;
return ok;
}
// skip if definitely not an integer
if (type != value_t::number_float)
signed long long parse_integral(
char** endptr,
/*is_signed*/std::true_type) const
{
auto digit = static_cast<number_unsigned_t>(*curptr - '0');
return std::strtoll(m_start, endptr, 10);
}
// overflow if value * 10 + digit > max, move terms around
// to avoid overflow in intermediate values
if (value > (max - digit) / 10)
unsigned long long parse_integral(
char** endptr,
/*is_signed*/std::false_type) const
{
// overflow
type = value_t::number_float;
return std::strtoull(m_start, endptr, 10);
}
else
template<typename T>
bool parse(T& value, /*is_integral=*/std::true_type) const
{
// no overflow
value = value * 10 + digit;
}
}
char* endptr = nullptr;
errno = 0; // these are thread-local
const auto x = parse_integral(&endptr, std::is_signed<T>());
static_assert(std::is_signed<T>() // called right overload?
== std::is_signed<decltype(x)>(), "");
value = static_cast<T>(x);
return x == static_cast<decltype(x)>(value) // x fits into destination T
and (x < 0) == (value < 0) // preserved sign
and (x != 0 or is_integral()) // strto[u]ll did nto fail
and errno == 0 // strto[u]ll did not overflow
and m_start < m_end // token was not empty
and endptr == m_end; // parsed entire token exactly
}
};
/*!
@brief return number value for number tokens
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
// save the value (if not a float)
if (type == value_t::number_unsigned)
integral numbers that don't fit into the the range of the respective
type are parsed as number_float_t
floating-point values do not satisfy std::isfinite predicate
are converted to value_t::null
throws if the entire string [m_start .. m_cursor) cannot be
interpreted as a number
@param[out] result @ref basic_json object to receive the number.
*/
void get_number(basic_json& result) const
{
assert(m_start != nullptr);
assert(m_start < m_cursor);
strtonum num(reinterpret_cast<const char*>(m_start),
reinterpret_cast<const char*>(m_cursor));
const bool is_negative = *m_start == '-';
result.m_type = value_t::discarded;
if (not num.is_integral())
{
result.m_value.number_unsigned = value;
; // will parse as float below
}
else if (type == value_t::number_integer)
else if (is_negative)
{
// invariant: if we parsed a '-', the absolute value is between
// 0 (we allow -0) and max == -INT64_MIN
assert(value >= 0);
assert(value <= max);
if (value == max)
number_integer_t val{0};
if (num.to(val))
{
// we cannot simply negate value (== max == -INT64_MIN),
// see https://github.com/nlohmann/json/issues/389
result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN);
result.m_type = value_t::number_integer;
result.m_value = val;
}
}
else
{
// all other values can be negated safely
result.m_value.number_integer = -static_cast<number_integer_t>(value);
number_unsigned_t val{0};
if (num.to(val))
{
result.m_type = value_t::number_unsigned;
result.m_value = val;
}
}
else
number_float_t val{0};
if (result.m_type != value_t::discarded
or !num.to(val))
{
// parse with strtod
result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), nullptr);
return; // already have a value from above
// or couldn't parse as float_t
}
result.m_type = value_t::number_float;
result.m_value = val;
// replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float))
{
type = value_t::null;
result.m_type = value_t::null;
result.m_value = basic_json::json_value();
}
}
// save the type
result.m_type = type;
}
private:
/// optional input stream
std::istream* m_stream = nullptr;
......
......@@ -9988,188 +9988,275 @@ class basic_json
return result;
}
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
/*!
@brief parse string into a built-in arithmetic type as if
the current locale is POSIX.
@param[in,out] endptr recieves a pointer to the first character after
the number
note: in floating-point case strtod may parse
past the token's end - this is not an error.
@return the floating point number
any leading blanks are not handled.
*/
long double str_to_float_t(long double* /* type */, char** endptr) const
struct strtonum
{
return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
}
/*!
@brief parse floating point number
public:
strtonum(const char* start, const char* end)
: m_start(start), m_end(end)
{}
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
/// return true iff parsed successfully as
/// number of type T.
///
/// @val shall contain parsed value, or
/// undefined value if could not parse.
template<typename T,
typename = typename std::enable_if<
std::is_arithmetic<T>::value>::type >
bool to(T& val) const
{
return parse(val, std::is_integral<T>());
}
@param[in,out] endptr recieves a pointer to the first character after
the number
/// return true iff token matches ^[+-]\d+$
///
/// this is a helper to determine whether to
/// parse the token into floating-point or
/// integral type. We wouldn't need it if
/// we had separate token types for integral
/// and floating-point cases.
bool is_integral() const
{
const char* p = m_start;
@return the floating point number
*/
double str_to_float_t(double* /* type */, char** endptr) const
if (!p)
{
return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
return false; // LCOV_EXCL_LINE
}
/*!
@brief parse floating point number
This function (and its overloads) serves to select the most approprate
standard floating point number parsing function based on the type
supplied via the first parameter. Set this to @a
static_cast<number_float_t*>(nullptr).
if (*p == '-' or * p == '+')
{
++p;
}
@param[in,out] endptr recieves a pointer to the first character after
the number
if (p == m_end)
{
return false; // LCOV_EXCL_LINE
}
@return the floating point number
*/
float str_to_float_t(float* /* type */, char** endptr) const
while (p < m_end and* p >= '0'
and * p <= '9')
{
return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
++p;
}
/*!
@brief return number value for number tokens
return p == m_end;
}
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
private:
const char* const m_start = nullptr;
const char* const m_end = nullptr;
This function parses the integer component up to the radix point or
exponent while collecting information about the 'floating point
representation', which it stores in the result parameter. If there is
no radix point or exponent, and the number can fit into a @ref
number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly.
// overloaded wrappers for strtod/strtof/strtold
// that will be called from parse<floating_point_t>
If the number is a floating point number the number is then parsed
using @a std:strtod (or @a std:strtof or @a std::strtold).
static void strtof(float& f,
const char* str,
char** endptr)
{
f = std::strtof(str, endptr);
}
@param[out] result @ref basic_json object to receive the number, or
NAN if the conversion read past the current token. The latter case
needs to be treated by the caller function.
*/
void get_number(basic_json& result) const
static void strtof(double& f,
const char* str,
char** endptr)
{
assert(m_start != nullptr);
f = std::strtod(str, endptr);
}
const lexer::lexer_char_t* curptr = m_start;
static void strtof(long double& f,
const char* str,
char** endptr)
{
f = std::strtold(str, endptr);
}
// accumulate the integer conversion result (unsigned for now)
number_unsigned_t value = 0;
template<typename T>
bool parse(T& value, /*is_integral=*/std::false_type) const
{
// replace decimal separator with locale-specific
// version, when necessary; data will point to
// either the original string, or buf, or tempstr
// containing the fixed string.
std::string tempstr;
std::array<char, 64> buf;
const size_t len = static_cast<size_t>(m_end - m_start);
// maximum absolute value of the relevant integer type
number_unsigned_t max;
// Since dealing with strtod family of functions,
// we're getting the decimal point char from the
// C locale facilities instead of C++'s numpunct
// facet of the current std::locale;
const auto loc = localeconv();
assert(loc != nullptr);
const char decimal_point_char = !loc->decimal_point ? '.'
: loc->decimal_point[0];
// temporarily store the type to avoid unecessary bitfield access
value_t type;
const char* data = m_start;
if (decimal_point_char != '.')
{
const size_t ds_pos = static_cast<size_t>(
std::find(m_start, m_end, '.') - m_start );
// look for sign
if (*curptr == '-')
if (ds_pos != len)
{
type = value_t::number_integer;
max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1;
curptr++;
// copy the data into the local buffer or
// tempstr, if buffer is too small;
// replace decimal separator, and update
// data to point to the modified bytes
if (len + 1 < buf.size())
{
std::copy(m_start, m_end, buf.data());
buf[len] = 0;
buf[ds_pos] = decimal_point_char;
data = buf.data();
}
else
{
type = value_t::number_unsigned;
max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)());
tempstr.assign(m_start, m_end);
tempstr[ds_pos] = decimal_point_char;
data = tempstr.c_str();
}
}
}
// count the significant figures
for (; curptr < m_cursor; curptr++)
{
// quickly skip tests if a digit
if (*curptr < '0' or* curptr > '9')
{
if (*curptr == '.')
char* endptr = nullptr;
value = 0;
// this calls appropriate overload depending on T
strtof(value, data, &endptr);
// note that reading past the end is OK, the data may be,
// for example, "123.", where the parsed token only
// contains "123", but strtod will read the dot as well.
const bool ok = endptr >= data + len
and len > 0;
if (ok and value == 0.0 and * data == '-')
{
// don't count '.' but change to float
type = value_t::number_float;
continue;
// some implementations forget to negate the zero
value = -0.0;
}
// assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details
type = value_t::number_float;
break;
return ok;
}
// skip if definitely not an integer
if (type != value_t::number_float)
signed long long parse_integral(
char** endptr,
/*is_signed*/std::true_type) const
{
auto digit = static_cast<number_unsigned_t>(*curptr - '0');
return std::strtoll(m_start, endptr, 10);
}
// overflow if value * 10 + digit > max, move terms around
// to avoid overflow in intermediate values
if (value > (max - digit) / 10)
unsigned long long parse_integral(
char** endptr,
/*is_signed*/std::false_type) const
{
// overflow
type = value_t::number_float;
return std::strtoull(m_start, endptr, 10);
}
else
template<typename T>
bool parse(T& value, /*is_integral=*/std::true_type) const
{
// no overflow
value = value * 10 + digit;
}
}
char* endptr = nullptr;
errno = 0; // these are thread-local
const auto x = parse_integral(&endptr, std::is_signed<T>());
static_assert(std::is_signed<T>() // called right overload?
== std::is_signed<decltype(x)>(), "");
value = static_cast<T>(x);
return x == static_cast<decltype(x)>(value) // x fits into destination T
and (x < 0) == (value < 0) // preserved sign
and (x != 0 or is_integral()) // strto[u]ll did nto fail
and errno == 0 // strto[u]ll did not overflow
and m_start < m_end // token was not empty
and endptr == m_end; // parsed entire token exactly
}
};
/*!
@brief return number value for number tokens
This function translates the last token into the most appropriate
number type (either integer, unsigned integer or floating point),
which is passed back to the caller via the result parameter.
// save the value (if not a float)
if (type == value_t::number_unsigned)
integral numbers that don't fit into the the range of the respective
type are parsed as number_float_t
floating-point values do not satisfy std::isfinite predicate
are converted to value_t::null
throws if the entire string [m_start .. m_cursor) cannot be
interpreted as a number
@param[out] result @ref basic_json object to receive the number.
*/
void get_number(basic_json& result) const
{
assert(m_start != nullptr);
assert(m_start < m_cursor);
strtonum num(reinterpret_cast<const char*>(m_start),
reinterpret_cast<const char*>(m_cursor));
const bool is_negative = *m_start == '-';
result.m_type = value_t::discarded;
if (not num.is_integral())
{
result.m_value.number_unsigned = value;
; // will parse as float below
}
else if (type == value_t::number_integer)
else if (is_negative)
{
// invariant: if we parsed a '-', the absolute value is between
// 0 (we allow -0) and max == -INT64_MIN
assert(value >= 0);
assert(value <= max);
if (value == max)
number_integer_t val{0};
if (num.to(val))
{
// we cannot simply negate value (== max == -INT64_MIN),
// see https://github.com/nlohmann/json/issues/389
result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN);
result.m_type = value_t::number_integer;
result.m_value = val;
}
}
else
{
// all other values can be negated safely
result.m_value.number_integer = -static_cast<number_integer_t>(value);
number_unsigned_t val{0};
if (num.to(val))
{
result.m_type = value_t::number_unsigned;
result.m_value = val;
}
}
else
number_float_t val{0};
if (result.m_type != value_t::discarded
or !num.to(val))
{
// parse with strtod
result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), nullptr);
return; // already have a value from above
// or couldn't parse as float_t
}
result.m_type = value_t::number_float;
result.m_value = val;
// replace infinity and NAN by null
if (not std::isfinite(result.m_value.number_float))
{
type = value_t::null;
result.m_type = value_t::null;
result.m_value = basic_json::json_value();
}
}
// save the type
result.m_type = type;
}
private:
/// optional input stream
std::istream* m_stream = nullptr;
......
......@@ -383,7 +383,7 @@ TEST_CASE("regression tests")
};
// change locale to mess with decimal points
std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator));
auto orig_locale = std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator));
CHECK(j1a.dump() == "23.42");
CHECK(j1b.dump() == "23.42");
......@@ -407,8 +407,34 @@ TEST_CASE("regression tests")
CHECK(j3c.dump() == "10000");
//CHECK(j3b.dump() == "1E04"); // roundtrip error
//CHECK(j3c.dump() == "1e04"); // roundtrip error
std::locale::global(orig_locale);
}
SECTION("issue #379 - locale-independent str-to-num")
{
setlocale(LC_NUMERIC, "de_DE.UTF-8");
// disabled, because locale-specific beharivor is not
// triggered in AppVeyor for some reason
#if 0
{
// verify that strtod now uses commas as decimal-separator
CHECK(std::strtod("3,14", nullptr) == 3.14);
// verify that strtod does not understand dots as decimal separator
CHECK(std::strtod("3.14", nullptr) == 3);
}
#endif
// verify that parsed correctly despite using strtod internally
CHECK(json::parse("3.14").get<double>() == 3.14);
// check a different code path
CHECK(json::parse("1.000000000000000000000000000000000000000000000000000000000000000000000000").get<double>() == 1.0);
}
SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator")
{
json source = {"a", "b", "c"};
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment