Commit db66e2f4 authored by Brian Pane's avatar Brian Pane Committed by Tudor Bosman

Move the branchless, parallel ASCII toLower into folly

Summary:
* Moved the fast toLower code into folly
* Updated the name to emphasize that it's ASCII-only

Test Plan: * Unit tests and benchmarks included.

Reviewed By: tudorb@fb.com

Subscribers: ruibalp, bmatheny

FB internal diff: D1421056
parent a0d5122f
...@@ -348,6 +348,146 @@ loop: ...@@ -348,6 +348,146 @@ loop:
return sp; return sp;
} }
namespace {
void toLowerAscii8(char& c) {
// Branchless tolower, based on the input-rotating trick described
// at http://www.azillionmonkeys.com/qed/asmexample.html
//
// This algorithm depends on an observation: each uppercase
// ASCII character can be converted to its lowercase equivalent
// by adding 0x20.
// Step 1: Clear the high order bit. We'll deal with it in Step 5.
unsigned char rotated = c & 0x7f;
// Currently, the value of rotated, as a function of the original c is:
// below 'A': 0- 64
// 'A'-'Z': 65- 90
// above 'Z': 91-127
// Step 2: Add 0x25 (37)
rotated += 0x25;
// Now the value of rotated, as a function of the original c is:
// below 'A': 37-101
// 'A'-'Z': 102-127
// above 'Z': 128-164
// Step 3: clear the high order bit
rotated &= 0x7f;
// below 'A': 37-101
// 'A'-'Z': 102-127
// above 'Z': 0- 36
// Step 4: Add 0x1a (26)
rotated += 0x1a;
// below 'A': 63-127
// 'A'-'Z': 128-153
// above 'Z': 25- 62
// At this point, note that only the uppercase letters have been
// transformed into values with the high order bit set (128 and above).
// Step 5: Shift the high order bit 2 spaces to the right: the spot
// where the only 1 bit in 0x20 is. But first, how we ignored the
// high order bit of the original c in step 1? If that bit was set,
// we may have just gotten a false match on a value in the range
// 128+'A' to 128+'Z'. To correct this, need to clear the high order
// bit of rotated if the high order bit of c is set. Since we don't
// care about the other bits in rotated, the easiest thing to do
// is invert all the bits in c and bitwise-and them with rotated.
rotated &= ~c;
rotated >>= 2;
// Step 6: Apply a mask to clear everything except the 0x20 bit
// in rotated.
rotated &= 0x20;
// At this point, rotated is 0x20 if c is 'A'-'Z' and 0x00 otherwise
// Step 7: Add rotated to c
c += rotated;
}
void toLowerAscii32(uint32_t& c) {
// Besides being branchless, the algorithm in toLowerAscii8() has another
// interesting property: None of the addition operations will cause
// an overflow in the 8-bit value. So we can pack four 8-bit values
// into a uint32_t and run each operation on all four values in parallel
// without having to use any CPU-specific SIMD instructions.
uint32_t rotated = c & uint32_t(0x7f7f7f7fL);
rotated += uint32_t(0x25252525L);
rotated &= uint32_t(0x7f7f7f7fL);
rotated += uint32_t(0x1a1a1a1aL);
// Step 5 involves a shift, so some bits will spill over from each
// 8-bit value into the next. But that's okay, because they're bits
// that will be cleared by the mask in step 6 anyway.
rotated &= ~c;
rotated >>= 2;
rotated &= uint32_t(0x20202020L);
c += rotated;
}
void toLowerAscii64(uint64_t& c) {
// 64-bit version of toLower32
uint64_t rotated = c & uint64_t(0x7f7f7f7f7f7f7f7fL);
rotated += uint64_t(0x2525252525252525L);
rotated &= uint64_t(0x7f7f7f7f7f7f7f7fL);
rotated += uint64_t(0x1a1a1a1a1a1a1a1aL);
rotated &= ~c;
rotated >>= 2;
rotated &= uint64_t(0x2020202020202020L);
c += rotated;
}
} // anon namespace
void toLowerAscii(char* str, size_t length) {
static const size_t kAlignMask64 = 7;
static const size_t kAlignMask32 = 3;
// Convert a character at a time until we reach an address that
// is at least 32-bit aligned
size_t n = (size_t)str;
n &= kAlignMask32;
n = std::min(n, length);
size_t offset = 0;
if (n != 0) {
do {
toLowerAscii8(str[offset]);
offset++;
} while (offset < n);
}
n = (size_t)(str + offset);
n &= kAlignMask64;
if ((n != 0) && (offset + 4 < length)) {
// The next address is 32-bit aligned but not 64-bit aligned.
// Convert the next 4 bytes in order to get to the 64-bit aligned
// part of the input.
toLowerAscii32(*(uint32_t*)(str + offset));
offset += 4;
}
// Convert 8 characters at a time
while (offset + 8 < length) {
toLowerAscii64(*(uint64_t*)(str + offset));
offset += 8;
}
// Convert 4 characters at a time
while (offset + 4 < length) {
toLowerAscii32(*(uint32_t*)(str + offset));
offset += 4;
}
// Convert any characters remaining after the last 4-byte aligned group
while (offset < length) {
toLowerAscii8(str[offset]);
offset++;
}
}
namespace detail { namespace detail {
size_t hexDumpLine(const void* ptr, size_t offset, size_t size, size_t hexDumpLine(const void* ptr, size_t offset, size_t size,
......
...@@ -509,6 +509,19 @@ std::string join(const Delim& delimiter, ...@@ -509,6 +509,19 @@ std::string join(const Delim& delimiter,
*/ */
StringPiece skipWhitespace(StringPiece sp); StringPiece skipWhitespace(StringPiece sp);
/**
* Fast, in-place lowercasing of ASCII alphabetic characters in strings.
* Leaves all other characters unchanged, including those with the 0x80
* bit set.
* @param str String to convert
* @param len Length of str, in bytes
*/
void toLowerAscii(char* str, size_t length);
inline void toLowerAscii(MutableStringPiece str) {
toLowerAscii(str.begin(), str.size());
}
} // namespace folly } // namespace folly
// Hash functions to make std::string usable with e.g. hash_map // Hash functions to make std::string usable with e.g. hash_map
......
/*
* Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/Benchmark.h>
#include <folly/String.h>
BENCHMARK(libc_tolower, iters) {
static const size_t kSize = 256;
// This array is static to keep the compiler from optimizing the
// entire function down to a no-op if it has an inlined impl of
// tolower and thus is able to tell that there are no side-effects.
// No side-effects + no writes to anything other than local variables
// + no return value = no need to run any of the code in the function.
// gcc, for example, makes that optimization with -O2.
static char input[kSize];
for (size_t i = 0; i < kSize; i++) {
input[i] = (char)(i & 0xff);
}
for (auto i = iters; i > 0; i--) {
for (size_t offset = 0; offset < kSize; offset++) {
input[offset] = tolower(input[offset]);
}
}
}
BENCHMARK(folly_toLowerAscii, iters) {
static const size_t kSize = 256;
static char input[kSize];
for (size_t i = 0; i < kSize; i++) {
input[i] = (char)(i & 0xff);
}
for (auto i = iters; i > 0; i--) {
folly::toLowerAscii(input, kSize);
}
}
int main(int argc, char** argv) {
google::ParseCommandLineFlags(&argc, &argv, true);
folly::runBenchmarks();
if (FLAGS_benchmark) {
folly::runBenchmarks();
}
return 0;
}
/*
Results on x86_64:
============================================================================
folly/test/StringBenchmark.cpp relative time/iter iters/s
============================================================================
libc_tolower 1.30us 767.50K
folly_toLowerAscii 115.21ns 8.68M
============================================================================
*/
...@@ -1034,6 +1034,49 @@ TEST(String, humanify) { ...@@ -1034,6 +1034,49 @@ TEST(String, humanify) {
EXPECT_EQ("0x61ffffffffff", humanify(string("a\xff\xff\xff\xff\xff"))); EXPECT_EQ("0x61ffffffffff", humanify(string("a\xff\xff\xff\xff\xff")));
} }
namespace {
void testToLowerAscii(Range<const char*> src) {
char control[src.size()];
memcpy(control, src.begin(), src.size());
char test[src.size()];
memcpy(test, src.begin(), src.size());
for (size_t i = 0; i < src.size(); i++) {
control[i] = tolower(control[i]);
}
toLowerAscii(test, src.size());
for (size_t i = 0; i < src.size(); i++) {
EXPECT_EQ(control[i], test[i]);
}
}
} // anon namespace
TEST(String, toLowerAsciiAligned) {
static const size_t kSize = 256;
char input[kSize];
for (size_t i = 0; i < kSize; i++) {
input[i] = (char)(i & 0xff);
}
testToLowerAscii(Range<const char*>(input, kSize));
}
TEST(String, toLowerAsciiUnaligned) {
static const size_t kSize = 256;
char input[kSize];
for (size_t i = 0; i < kSize; i++) {
input[i] = (char)(i & 0xff);
}
// Test input buffers of several lengths to exercise all the
// cases: buffer at the start/middle/end of an aligned block, plus
// buffers that span multiple aligned blocks.
for (size_t length = 1; length < 11; length++) {
for (size_t offset = 0; offset + length <= kSize; offset++) {
testToLowerAscii(Range<const char*>(input + offset, length));
}
}
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
BENCHMARK(splitOnSingleChar, iters) { BENCHMARK(splitOnSingleChar, iters) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment