Commit 34307dc2 authored by Jez Ng's avatar Jez Ng Committed by Sara Golemon

Factor out JSON stripComments().

Summary:
Comments are a useful extension to JSON, especially for configuration files.

Facebook: twagent would previously barf on JSON files that contained '//' in their strings, and this commit allows twagent to strip comments properly. Fixes T4686066.

Test Plan: fbconfig common/json mcrouter/lib/config tupperware/agent && fbmake runtests

Reviewed By: aravindn@fb.com

Subscribers: anarayanan, pavlo, stepan, dipanshu, alikhtarov

FB internal diff: D1493963

Tasks: 4686066
parent 6e46d468
...@@ -684,6 +684,62 @@ void escapeString(StringPiece input, ...@@ -684,6 +684,62 @@ void escapeString(StringPiece input,
out.push_back('\"'); out.push_back('\"');
} }
fbstring stripComments(StringPiece jsonC) {
fbstring result;
enum class State {
None,
InString,
InlineComment,
LineComment
} state = State::None;
for (size_t i = 0; i < jsonC.size(); ++i) {
auto s = jsonC.subpiece(i);
switch (state) {
case State::None:
if (s.startsWith("/*")) {
state = State::InlineComment;
++i;
continue;
} else if (s.startsWith("//")) {
state = State::LineComment;
++i;
continue;
} else if (s.startsWith("\"")) {
state = State::InString;
}
result.push_back(s[0]);
break;
case State::InString:
if (s.startsWith("\\\"")) {
result.push_back(s[0]);
result.push_back(s[1]);
++i;
continue;
} else if (s.startsWith("\"")) {
state = State::None;
}
result.push_back(s[0]);
break;
case State::InlineComment:
if (s.startsWith("*/")) {
state = State::None;
++i;
}
break;
case State::LineComment:
if (s.startsWith("\n")) {
// skip the line break. It doesn't matter.
state = State::None;
}
break;
default:
throw std::logic_error("Unknown comment state");
}
}
return result;
}
} }
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
......
...@@ -142,6 +142,11 @@ namespace json { ...@@ -142,6 +142,11 @@ namespace json {
void escapeString(StringPiece input, void escapeString(StringPiece input,
fbstring& out, fbstring& out,
const serialization_opts& opts); const serialization_opts& opts);
/*
* Strip all C99-like comments (i.e. // and / * ... * /)
*/
fbstring stripComments(StringPiece jsonC);
} }
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
#include <folly/FileUtil.h>
#include <folly/json.h> #include <folly/json.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <gflags/gflags.h> #include <gflags/gflags.h>
...@@ -379,6 +380,23 @@ TEST(Json, SortKeys) { ...@@ -379,6 +380,23 @@ TEST(Json, SortKeys) {
EXPECT_EQ(sorted_keys, folly::json::serialize(value, opts_on)); EXPECT_EQ(sorted_keys, folly::json::serialize(value, opts_on));
} }
TEST(Json, StripComments) {
const std::string kTestFile =
"folly/test/json_test_data/commented.json";
const std::string kTestExpected =
"folly/test/json_test_data/commented.json.exp";
std::string testStr;
std::string expectedStr;
if (!folly::readFile(kTestFile.data(), testStr)) {
FAIL() << "can not read test file " << kTestFile;
}
if (!folly::readFile(kTestExpected.data(), expectedStr)) {
FAIL() << "can not read test file " << kTestExpected;
}
EXPECT_EQ(expectedStr, folly::json::stripComments(testStr));
}
BENCHMARK(jsonSerialize, iters) { BENCHMARK(jsonSerialize, iters) {
folly::json::serialization_opts opts; folly::json::serialization_opts opts;
for (int i = 0; i < iters; ++i) { for (int i = 0; i < iters; ++i) {
......
{
// comment
"test": "foo", // comment
"test2": "foo // bar", // more comments
/*
"test3": "baz"
*/
"test4": "foo /* bar", /* comment */
"te//": "foo",
"te/*": "bar"
}
{
"test": "foo", "test2": "foo // bar",
"test4": "foo /* bar",
"te//": "foo",
"te/*": "bar"
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment