Merge pull request #450 from nlohmann/TurpentineDistillery-feature/locale_independent_str_to_num

TurpentineDistillery feature/locale independent str to num

Makefile

@@ -94,7 +94,7 @@ cppcheck:
 
 # run clang sanitize (we are overrding the CXXFLAGS provided by travis in order to use gcc's libstdc++)
 clang_sanitize: clean
-	CXX=clang++ CXXFLAGS="-g -O2 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer" $(MAKE)
+	CXX=clang++ CXXFLAGS="-g -O2 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer" $(MAKE) check
 
 
 ##########################################################################

test/src/unit-class_lexer.cpp

@@ -65,25 +65,37 @@ TEST_CASE("lexer class")
         SECTION("numbers")
         {
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("0"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("1"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("2"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("3"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("4"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("5"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("6"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("7"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("8"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
             CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("9"),
-                               1).scan() == json::lexer::token_type::value_number));
+                               1).scan() == json::lexer::token_type::value_unsigned));
+
+            CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("-0"),
+                               2).scan() == json::lexer::token_type::value_integer));
+            CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("-1"),
+                               2).scan() == json::lexer::token_type::value_integer));
+
+            CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("1.1"),
+                               3).scan() == json::lexer::token_type::value_float));
+            CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("-1.1"),
+                               4).scan() == json::lexer::token_type::value_float));
+            CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("1E10"),
+                               4).scan() == json::lexer::token_type::value_float));
         }
 
         SECTION("whitespace")
@@ -109,7 +121,9 @@ TEST_CASE("lexer class")
         CHECK((json::lexer::token_type_name(json::lexer::token_type::literal_false) == "false literal"));
         CHECK((json::lexer::token_type_name(json::lexer::token_type::literal_null) == "null literal"));
         CHECK((json::lexer::token_type_name(json::lexer::token_type::value_string) == "string literal"));
-        CHECK((json::lexer::token_type_name(json::lexer::token_type::value_number) == "number literal"));
+        CHECK((json::lexer::token_type_name(json::lexer::token_type::value_unsigned) == "number literal"));
+        CHECK((json::lexer::token_type_name(json::lexer::token_type::value_integer) == "number literal"));
+        CHECK((json::lexer::token_type_name(json::lexer::token_type::value_float) == "number literal"));
         CHECK((json::lexer::token_type_name(json::lexer::token_type::begin_array) == "'['"));
         CHECK((json::lexer::token_type_name(json::lexer::token_type::begin_object) == "'{'"));
         CHECK((json::lexer::token_type_name(json::lexer::token_type::end_array) == "']'"));

test/src/unit-class_parser.cpp

@@ -101,6 +101,7 @@ TEST_CASE("parser class")
                 CHECK_THROWS_WITH(json::parser("\"\b\"").parse(), "parse error - unexpected '\"'");
                 // improve code coverage
                 CHECK_THROWS_AS(json::parser("\uFF01").parse(), std::invalid_argument);
+                CHECK_THROWS_AS(json::parser("[-4:1,]").parse(), std::invalid_argument);
                 // unescaped control characters
                 CHECK_THROWS_AS(json::parser("\"\x00\"").parse(), std::invalid_argument);
                 CHECK_THROWS_AS(json::parser("\"\x01\"").parse(), std::invalid_argument);
@@ -269,6 +270,11 @@ TEST_CASE("parser class")
                 }
             }
 
+            SECTION("overflow")
+            {
+                CHECK(json::parser("1.18973e+4932").parse() == json());
+            }
+
             SECTION("invalid numbers")
             {
                 CHECK_THROWS_AS(json::parser("01").parse(), std::invalid_argument);
@@ -293,7 +299,7 @@ TEST_CASE("parser class")
                 CHECK_THROWS_AS(json::parser("+0").parse(), std::invalid_argument);
 
                 CHECK_THROWS_WITH(json::parser("01").parse(),
-                                  "parse error - unexpected number literal; expected end of input");
+                                  "parse error - unexpected number literal");
                 CHECK_THROWS_WITH(json::parser("--1").parse(), "parse error - unexpected '-'");
                 CHECK_THROWS_WITH(json::parser("1.").parse(),
                                   "parse error - unexpected '.'; expected end of input");

test/src/unit-regression.cpp

@@ -383,7 +383,7 @@ TEST_CASE("regression tests")
         };
 
         // change locale to mess with decimal points
-        std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator));
+        auto orig_locale = std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator));
 
         CHECK(j1a.dump() == "23.42");
         CHECK(j1b.dump() == "23.42");
@@ -407,8 +407,34 @@ TEST_CASE("regression tests")
         CHECK(j3c.dump() == "10000");
         //CHECK(j3b.dump() == "1E04"); // roundtrip error
         //CHECK(j3c.dump() == "1e04"); // roundtrip error
+
+        std::locale::global(orig_locale);
     }
 
+    SECTION("issue #379 - locale-independent str-to-num")
+    {
+        setlocale(LC_NUMERIC, "de_DE.UTF-8");
+
+        // disabled, because locale-specific beharivor is not
+        // triggered in AppVeyor for some reason
+#ifndef _MSC_VER
+        {
+            // verify that strtod now uses commas as decimal-separator
+            CHECK(std::strtod("3,14", nullptr) == 3.14);
+
+            // verify that strtod does not understand dots as decimal separator
+            CHECK(std::strtod("3.14", nullptr) == 3);
+        }
+#endif
+
+        // verify that parsed correctly despite using strtod internally
+        CHECK(json::parse("3.14").get<double>() == 3.14);
+
+        // check a different code path
+        CHECK(json::parse("1.000000000000000000000000000000000000000000000000000000000000000000000000").get<double>() == 1.0);
+    }
+
+
     SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator")
     {
         json source = {"a", "b", "c"};