fixed a lot of unicode errors (#344)

BOMs currently yield an error.

Makefile

@@ -75,7 +75,7 @@ clang_sanitize: clean
 
 # create scanner with re2c
 re2c: src/json.hpp.re2c
-	$(RE2C) -W --bit-vectors --nested-ifs --no-debug-info $< | $(SED) '1d' > src/json.hpp
+	$(RE2C) -W --utf-8 --encoding-policy fail --bit-vectors --nested-ifs --no-debug-info $< | $(SED) '1d' > src/json.hpp
 
 # pretty printer
 pretty:

src/json.hpp.re2c

@@ -7848,7 +7848,7 @@ class basic_json
                     "\000"         { last_token_type = token_type::end_of_input; break; }
 
                     // anything else is an error
-                    .              { last_token_type = token_type::parse_error; break; }
+                    *              { last_token_type = token_type::parse_error; break; }
                 */
             }
 

test/src/unit-testsuites.cpp

@@ -671,7 +671,7 @@ TEST_CASE("nst's JSONTestSuite")
                         "test/data/nst_json_testsuite/test_parsing/n_string_1_surrogate_then_escape u1x.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_1_surrogate_then_escape.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_UTF-16_incomplete_surrogate.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_UTF8_surrogate_U+D800.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_UTF8_surrogate_U+D800.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_accentuated_char_no_quotes.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_backslash_00.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_escape_x.json",
@@ -684,15 +684,15 @@ TEST_CASE("nst's JSONTestSuite")
                         "test/data/nst_json_testsuite/test_parsing/n_string_invalid-utf-8-in-escape.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_invalid_backslash_esc.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_invalid_unicode_escape.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_invalid_utf-8.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_invalid_utf-8.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_invalid_utf8_after_escape.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_iso_latin_1.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_iso_latin_1.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_leading_uescaped_thinspace.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_lone_utf8_continuation_byte.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_lone_utf8_continuation_byte.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_no_quotes_with_bad_escape.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_overlong_sequence_2_bytes.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_overlong_sequence_6_bytes.json",
-                        //"test/data/nst_json_testsuite/test_parsing/n_string_overlong_sequence_6_bytes_null.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_overlong_sequence_2_bytes.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_overlong_sequence_6_bytes.json",
+                        "test/data/nst_json_testsuite/test_parsing/n_string_overlong_sequence_6_bytes_null.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_single_doublequote.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_single_quote.json",
                         "test/data/nst_json_testsuite/test_parsing/n_string_single_string_no_double_quotes.json",

test/src/unit-unicode.cpp

@@ -159,7 +159,7 @@ TEST_CASE("Unicode", "[hide]")
             }
         }
     }
-
+/*
     SECTION("ignore byte-order-mark")
     {
         // read a file with a UTF-8 BOM
@@ -167,7 +167,7 @@ TEST_CASE("Unicode", "[hide]")
         json j;
         CHECK_NOTHROW(j << f);
     }
-
+*/
     SECTION("error for incomplete/wrong BOM")
     {
         CHECK_THROWS_AS(json::parse("\xef\xbb"), std::invalid_argument);