Commit 92320e04 authored by Steve O'Brien's avatar Steve O'Brien Committed by Facebook Github Bot

folly/String: split out `UTF8StringPiece`

Summary:
This has functionality not overlapping with `FBString` or `folly::fbstring` or `folly::StringPiece`; and a pretty expensive dependency on `boost/regex`.

Much of the thrift-gen `_types.h` code comes from this inclusion chain:

* `_types.h` uses strings (for `str` fields in thrift structs); these are represented by `folly::fbstring`, from...
* `folly/String.h` includes `FBString.h` and other things to support string-related stuff; simple enough.  But it has a class `UTF8StringPiece`, which uses an iterator, from Boost called `boost::u8_to_u32_iterator<char const *>`.  So then we have...
* `boost/regex/pending/unicode_iterator.hpp` includes tons of things in the usual boost header avalanche: iterator facades, lots of `mpl` code.

Reviewed By: meyering

Differential Revision: D7373577

fbshipit-source-id: efdb920b6fe8c7b7c57d3becf2bf0710975fcf73
parent d22d402e
...@@ -290,7 +290,7 @@ void internalSplit(DelimT delim, StringPiece sp, OutputIterator out, ...@@ -290,7 +290,7 @@ void internalSplit(DelimT delim, StringPiece sp, OutputIterator out,
} }
return; return;
} }
if (boost::is_same<DelimT,StringPiece>::value && dSize == 1) { if (std::is_same<DelimT, StringPiece>::value && dSize == 1) {
// Call the char version because it is significantly faster. // Call the char version because it is significantly faster.
return internalSplit<OutStringT>(delimFront(delim), sp, out, return internalSplit<OutStringT>(delimFront(delim), sp, out,
ignoreEmpty); ignoreEmpty);
......
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
#include <cstdarg> #include <cstdarg>
#include <cstring> #include <cstring>
#include <iterator> #include <iterator>
#include <sstream>
#include <stdexcept> #include <stdexcept>
#include <glog/logging.h> #include <glog/logging.h>
......
...@@ -24,8 +24,6 @@ ...@@ -24,8 +24,6 @@
#include <unordered_set> #include <unordered_set>
#include <vector> #include <vector>
#include <boost/regex/pending/unicode_iterator.hpp>
#include <folly/Conv.h> #include <folly/Conv.h>
#include <folly/ExceptionString.h> #include <folly/ExceptionString.h>
#include <folly/FBString.h> #include <folly/FBString.h>
...@@ -35,6 +33,11 @@ ...@@ -35,6 +33,11 @@
#include <folly/ScopeGuard.h> #include <folly/ScopeGuard.h>
#include <folly/Traits.h> #include <folly/Traits.h>
// Temporary compatilibility shim; UTF8StringPiece is no longer provided by
// this file, but we'll allow includes of this file to continue to "provide"
// its definition during a brief period while include-sites are updated.
#include <folly/UTF8String.h>
// Compatibility function, to make sure toStdString(s) can be called // Compatibility function, to make sure toStdString(s) can be called
// to convert a std::string or fbstring variable s into type std::string // to convert a std::string or fbstring variable s into type std::string
// with very little overhead if s was already std::string // with very little overhead if s was already std::string
...@@ -601,22 +604,6 @@ inline void toLowerAscii(std::string& str) { ...@@ -601,22 +604,6 @@ inline void toLowerAscii(std::string& str) {
toLowerAscii(&str[0], str.size()); toLowerAscii(&str[0], str.size());
} }
template <
class Iterator = const char*,
class Base = folly::Range<boost::u8_to_u32_iterator<Iterator>>>
class UTF8Range : public Base {
public:
/* implicit */ UTF8Range(const folly::Range<Iterator> baseRange)
: Base(boost::u8_to_u32_iterator<Iterator>(
baseRange.begin(), baseRange.begin(), baseRange.end()),
boost::u8_to_u32_iterator<Iterator>(
baseRange.end(), baseRange.begin(), baseRange.end())) {}
/* implicit */ UTF8Range(const std::string& baseString)
: Base(folly::Range<Iterator>(baseString)) {}
};
using UTF8StringPiece = UTF8Range<const char*>;
} // namespace folly } // namespace folly
#include <folly/String-inl.h> #include <folly/String-inl.h>
/*
* Copyright 2011-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <string>
#include <boost/regex/pending/unicode_iterator.hpp>
#include <folly/Range.h>
namespace folly {
template <
class Iterator = const char*,
class Base = folly::Range<boost::u8_to_u32_iterator<Iterator>>>
class UTF8Range : public Base {
public:
/* implicit */ UTF8Range(const folly::Range<Iterator> baseRange)
: Base(
boost::u8_to_u32_iterator<Iterator>(
baseRange.begin(),
baseRange.begin(),
baseRange.end()),
boost::u8_to_u32_iterator<Iterator>(
baseRange.end(),
baseRange.begin(),
baseRange.end())) {}
/* implicit */ UTF8Range(const std::string& baseString)
: Base(folly::Range<Iterator>(baseString)) {}
};
using UTF8StringPiece = UTF8Range<const char*>;
} // namespace folly
...@@ -24,6 +24,8 @@ ...@@ -24,6 +24,8 @@
*/ */
#include <folly/experimental/DynamicParser.h> #include <folly/experimental/DynamicParser.h>
#include <sstream>
#include <folly/Optional.h> #include <folly/Optional.h>
namespace folly { namespace folly {
......
...@@ -1377,34 +1377,3 @@ TEST(String, stripLeftMargin_no_post_whitespace) { ...@@ -1377,34 +1377,3 @@ TEST(String, stripLeftMargin_no_post_whitespace) {
auto expected = "hi there bob!\n \nso long! "; auto expected = "hi there bob!\n \nso long! ";
EXPECT_EQ(expected, stripLeftMargin(input)); EXPECT_EQ(expected, stripLeftMargin(input));
} }
const folly::StringPiece kTestUTF8 = u8"This is \U0001F602 stuff!";
TEST(UTF8StringPiece, valid_utf8) {
folly::StringPiece sp = kTestUTF8;
UTF8StringPiece utf8 = sp;
// utf8.size() not available since it's not a random-access range
EXPECT_EQ(16, utf8.walk_size());
}
TEST(UTF8StringPiece, valid_suffix) {
UTF8StringPiece utf8 = kTestUTF8.subpiece(8);
EXPECT_EQ(8, utf8.walk_size());
}
TEST(UTF8StringPiece, empty_mid_codepoint) {
UTF8StringPiece utf8 = kTestUTF8.subpiece(9, 0); // okay since it's empty
EXPECT_EQ(0, utf8.walk_size());
}
TEST(UTF8StringPiece, invalid_mid_codepoint) {
EXPECT_THROW(UTF8StringPiece(kTestUTF8.subpiece(9, 1)), std::out_of_range);
}
TEST(UTF8StringPiece, valid_implicit_conversion) {
std::string input = u8"\U0001F602\U0001F602\U0001F602";
auto checkImplicitCtor = [](UTF8StringPiece implicitCtor) {
return implicitCtor.walk_size();
};
EXPECT_EQ(3, checkImplicitCtor(input));
}
/*
* Copyright 2012-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/UTF8String.h>
#include <folly/Range.h>
#include <folly/portability/GTest.h>
#include <folly/test/TestUtils.h>
using namespace folly;
using namespace std;
const folly::StringPiece kTestUTF8 = u8"This is \U0001F602 stuff!";
TEST(UTF8StringPiece, valid_utf8) {
folly::StringPiece sp = kTestUTF8;
UTF8StringPiece utf8 = sp;
// utf8.size() not available since it's not a random-access range
EXPECT_EQ(16, utf8.walk_size());
}
TEST(UTF8StringPiece, valid_suffix) {
UTF8StringPiece utf8 = kTestUTF8.subpiece(8);
EXPECT_EQ(8, utf8.walk_size());
}
TEST(UTF8StringPiece, empty_mid_codepoint) {
UTF8StringPiece utf8 = kTestUTF8.subpiece(9, 0); // okay since it's empty
EXPECT_EQ(0, utf8.walk_size());
}
TEST(UTF8StringPiece, invalid_mid_codepoint) {
EXPECT_THROW(UTF8StringPiece(kTestUTF8.subpiece(9, 1)), std::out_of_range);
}
TEST(UTF8StringPiece, valid_implicit_conversion) {
std::string input = u8"\U0001F602\U0001F602\U0001F602";
auto checkImplicitCtor = [](UTF8StringPiece implicitCtor) {
return implicitCtor.walk_size();
};
EXPECT_EQ(3, checkImplicitCtor(input));
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment