Commit d9acfc9e authored by Dave Watson's avatar Dave Watson Committed by Facebook Github Bot

Add hardware crc impl

Summary:
A faster crc32 impl for folly.  Similar to crc32c, except
intel doesn't provide crc32 directly in hardware - instead, pclmul can be used,
which is ~2x slower than crc32c, but still ~5-10x faster than software implementation.

Reviewed By: Orvid, yfeldblum

Differential Revision: D4994761

fbshipit-source-id: ad8ba856649eea6dc7b541d561329ff7d7fe2d60
parent 932973bf
...@@ -15,10 +15,11 @@ ...@@ -15,10 +15,11 @@
*/ */
#include <folly/Checksum.h> #include <folly/Checksum.h>
#include <algorithm>
#include <stdexcept>
#include <boost/crc.hpp> #include <boost/crc.hpp>
#include <folly/CpuId.h> #include <folly/CpuId.h>
#include <folly/detail/ChecksumDetail.h>
#include <algorithm>
#include <stdexcept>
#if FOLLY_X64 && (__SSE4_2__ || defined(__clang__) || __GNUC_PREREQ(4, 9)) #if FOLLY_X64 && (__SSE4_2__ || defined(__clang__) || __GNUC_PREREQ(4, 9))
#include <nmmintrin.h> #include <nmmintrin.h>
...@@ -28,6 +29,8 @@ namespace folly { ...@@ -28,6 +29,8 @@ namespace folly {
namespace detail { namespace detail {
uint32_t
crc32c_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum);
#if FOLLY_X64 && (__SSE4_2__ || defined(__clang__) || __GNUC_PREREQ(4, 9)) #if FOLLY_X64 && (__SSE4_2__ || defined(__clang__) || __GNUC_PREREQ(4, 9))
// Fast SIMD implementation of CRC-32C for x86 with SSE 4.2 // Fast SIMD implementation of CRC-32C for x86 with SSE 4.2
...@@ -64,11 +67,43 @@ uint32_t crc32c_hw(const uint8_t *data, size_t nbytes, ...@@ -64,11 +67,43 @@ uint32_t crc32c_hw(const uint8_t *data, size_t nbytes,
return sum; return sum;
} }
uint32_t
crc32_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum);
// Fast SIMD implementation of CRC-32 for x86 with pclmul
uint32_t
crc32_hw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
uint32_t sum = startingChecksum;
size_t offset = 0;
// Process unaligned bytes
if ((uintptr_t)data & 15) {
size_t limit = std::min(nbytes, -(uintptr_t)data & 15);
sum = crc32_sw(data, limit, sum);
offset += limit;
nbytes -= limit;
}
if (nbytes >= 16) {
sum = crc32_hw_aligned(sum, (const __m128i*)(data + offset), nbytes / 16);
offset += nbytes & ~15;
nbytes &= 15;
}
// Remaining unaligned bytes
return crc32_sw(data + offset, nbytes, sum);
}
bool crc32c_hw_supported() { bool crc32c_hw_supported() {
static folly::CpuId id; static folly::CpuId id;
return id.sse42(); return id.sse42();
} }
bool crc32_hw_supported() {
static folly::CpuId id;
return id.sse42();
}
#else #else
uint32_t crc32c_hw(const uint8_t *data, size_t nbytes, uint32_t crc32c_hw(const uint8_t *data, size_t nbytes,
...@@ -80,11 +115,13 @@ bool crc32c_hw_supported() { ...@@ -80,11 +115,13 @@ bool crc32c_hw_supported() {
return false; return false;
} }
bool crc32_hw_supported() {
return false;
}
#endif #endif
uint32_t crc32c_sw(const uint8_t *data, size_t nbytes, template <uint32_t CRC_POLYNOMIAL>
uint32_t startingChecksum) { uint32_t crc_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
// Reverse the bits in the starting checksum so they'll be in the // Reverse the bits in the starting checksum so they'll be in the
// right internal format for Boost's CRC engine. // right internal format for Boost's CRC engine.
// O(1)-time, branchless bit reversal algorithm from // O(1)-time, branchless bit reversal algorithm from
...@@ -100,13 +137,24 @@ uint32_t crc32c_sw(const uint8_t *data, size_t nbytes, ...@@ -100,13 +137,24 @@ uint32_t crc32c_sw(const uint8_t *data, size_t nbytes,
startingChecksum = (startingChecksum >> 16) | startingChecksum = (startingChecksum >> 16) |
(startingChecksum << 16); (startingChecksum << 16);
static const uint32_t CRC32C_POLYNOMIAL = 0x1EDC6F41; boost::crc_optimal<32, CRC_POLYNOMIAL, ~0U, 0, true, true> sum(
boost::crc_optimal<32, CRC32C_POLYNOMIAL, ~0U, 0, true, true> sum(
startingChecksum); startingChecksum);
sum.process_bytes(data, nbytes); sum.process_bytes(data, nbytes);
return sum.checksum(); return sum.checksum();
} }
uint32_t
crc32c_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
constexpr uint32_t CRC32C_POLYNOMIAL = 0x1EDC6F41;
return crc_sw<CRC32C_POLYNOMIAL>(data, nbytes, startingChecksum);
}
uint32_t
crc32_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
constexpr uint32_t CRC32_POLYNOMIAL = 0x04C11DB7;
return crc_sw<CRC32_POLYNOMIAL>(data, nbytes, startingChecksum);
}
} // folly::detail } // folly::detail
uint32_t crc32c(const uint8_t *data, size_t nbytes, uint32_t crc32c(const uint8_t *data, size_t nbytes,
...@@ -118,4 +166,12 @@ uint32_t crc32c(const uint8_t *data, size_t nbytes, ...@@ -118,4 +166,12 @@ uint32_t crc32c(const uint8_t *data, size_t nbytes,
} }
} }
uint32_t crc32(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
if (detail::crc32_hw_supported()) {
return detail::crc32_hw(data, nbytes, startingChecksum);
} else {
return detail::crc32_sw(data, nbytes, startingChecksum);
}
}
} // folly } // folly
...@@ -37,4 +37,12 @@ namespace folly { ...@@ -37,4 +37,12 @@ namespace folly {
uint32_t crc32c(const uint8_t* data, size_t nbytes, uint32_t crc32c(const uint8_t* data, size_t nbytes,
uint32_t startingChecksum = ~0U); uint32_t startingChecksum = ~0U);
/**
* Compute the CRC-32 checksum of a buffer, using a hardware-accelerated
* implementation if available or a portable software implementation as
* a default.
*/
uint32_t
crc32(const uint8_t* data, size_t nbytes, uint32_t startingChecksum = ~0U);
} // folly } // folly
...@@ -418,6 +418,7 @@ GroupVarintTables.cpp: build/generate_varint_tables.py ...@@ -418,6 +418,7 @@ GroupVarintTables.cpp: build/generate_varint_tables.py
CLEANFILES += GroupVarintTables.cpp CLEANFILES += GroupVarintTables.cpp
libfollybasesse42_la_SOURCES = \ libfollybasesse42_la_SOURCES = \
detail/ChecksumDetail.cpp \
detail/RangeSse42.cpp detail/RangeSse42.cpp
libfollybase_la_SOURCES = \ libfollybase_la_SOURCES = \
...@@ -613,7 +614,7 @@ libfolly_la_SOURCES += \ ...@@ -613,7 +614,7 @@ libfolly_la_SOURCES += \
endif endif
libfollybasesse42_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(LT_VERSION) libfollybasesse42_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(LT_VERSION)
libfollybasesse42_la_CXXFLAGS = -msse4.2 libfollybasesse42_la_CXXFLAGS = -msse4.2 -mpclmul
libfollybase_la_LIBADD = libfollybasesse42.la libfollybase_la_LIBADD = libfollybasesse42.la
libfollybase_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(LT_VERSION) libfollybase_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(LT_VERSION)
......
This diff is collapsed.
...@@ -16,6 +16,10 @@ ...@@ -16,6 +16,10 @@
#pragma once #pragma once
#include <immintrin.h>
#include <stdint.h>
#include <cstddef>
namespace folly { namespace detail { namespace folly { namespace detail {
/** /**
...@@ -51,5 +55,39 @@ bool crc32c_hw_supported(); ...@@ -51,5 +55,39 @@ bool crc32c_hw_supported();
uint32_t crc32c_sw(const uint8_t* data, size_t nbytes, uint32_t crc32c_sw(const uint8_t* data, size_t nbytes,
uint32_t startingChecksum = ~0U); uint32_t startingChecksum = ~0U);
/**
* Compute a CRC-32 checksum of a buffer using a hardware-accelerated
* implementation.
*
* @note This function is exposed to support special cases where the
* calling code is absolutely certain it ought to invoke a hardware-
* accelerated CRC-32 implementation - unit tests, for example. For
* all other scenarios, please call crc32() and let it pick an
* implementation based on the capabilities of the underlying CPU.
*/
uint32_t
crc32_hw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum = ~0U);
uint32_t
crc32_hw_aligned(uint32_t remainder, const __m128i* p, size_t vec_count);
/**
* Check whether a hardware-accelerated CRC-32 implementation is
* supported on the current CPU.
*/
bool crc32_hw_supported();
/**
* Compute a CRC-32 checksum of a buffer using a portable,
* software-only implementation.
*
* @note This function is exposed to support special cases where the
* calling code is absolutely certain it wants to use the software
* implementation instead of the hardware-accelerated code - unit
* tests, for example. For all other scenarios, please call crc32()
* and let it pick an implementation based on the capabilities of
* the underlying CPU.
*/
uint32_t
crc32_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum = ~0U);
}} // folly::detail }} // folly::detail
...@@ -126,6 +126,49 @@ TEST(Checksum, crc32c_continuation_autodetect) { ...@@ -126,6 +126,49 @@ TEST(Checksum, crc32c_continuation_autodetect) {
testCRC32CContinuation(folly::crc32c); testCRC32CContinuation(folly::crc32c);
} }
TEST(Checksum, crc32) {
if (folly::detail::crc32c_hw_supported()) {
// Just check that sw and hw match
for (auto expected : expectedResults) {
uint32_t sw_res =
folly::detail::crc32_sw(buffer + expected.offset, expected.length, 0);
uint32_t hw_res =
folly::detail::crc32_hw(buffer + expected.offset, expected.length, 0);
EXPECT_EQ(sw_res, hw_res);
}
} else {
LOG(WARNING) << "skipping hardware-accelerated CRC-32 tests"
<< " (not supported on this CPU)";
}
}
TEST(Checksum, crc32_continuation) {
if (folly::detail::crc32c_hw_supported()) {
// Just check that sw and hw match
for (auto expected : expectedResults) {
auto halflen = expected.length / 2;
uint32_t sw_res =
folly::detail::crc32_sw(buffer + expected.offset, halflen, 0);
sw_res = folly::detail::crc32_sw(
buffer + expected.offset + halflen, halflen, sw_res);
uint32_t hw_res =
folly::detail::crc32_hw(buffer + expected.offset, halflen, 0);
hw_res = folly::detail::crc32_hw(
buffer + expected.offset + halflen, halflen, hw_res);
EXPECT_EQ(sw_res, hw_res);
uint32_t sw_res2 =
folly::detail::crc32_sw(buffer + expected.offset, halflen * 2, 0);
EXPECT_EQ(sw_res, sw_res2);
uint32_t hw_res2 =
folly::detail::crc32_hw(buffer + expected.offset, halflen * 2, 0);
EXPECT_EQ(hw_res, hw_res2);
}
} else {
LOG(WARNING) << "skipping hardware-accelerated CRC-32 tests"
<< " (not supported on this CPU)";
}
}
void benchmarkHardwareCRC32C(unsigned long iters, size_t blockSize) { void benchmarkHardwareCRC32C(unsigned long iters, size_t blockSize) {
if (folly::detail::crc32c_hw_supported()) { if (folly::detail::crc32c_hw_supported()) {
uint32_t checksum; uint32_t checksum;
...@@ -147,6 +190,27 @@ void benchmarkSoftwareCRC32C(unsigned long iters, size_t blockSize) { ...@@ -147,6 +190,27 @@ void benchmarkSoftwareCRC32C(unsigned long iters, size_t blockSize) {
} }
} }
void benchmarkHardwareCRC32(unsigned long iters, size_t blockSize) {
if (folly::detail::crc32_hw_supported()) {
uint32_t checksum;
for (unsigned long i = 0; i < iters; i++) {
checksum = folly::detail::crc32_hw(buffer, blockSize);
folly::doNotOptimizeAway(checksum);
}
} else {
LOG(WARNING) << "skipping hardware-accelerated CRC-32 benchmarks"
<< " (not supported on this CPU)";
}
}
void benchmarkSoftwareCRC32(unsigned long iters, size_t blockSize) {
uint32_t checksum;
for (unsigned long i = 0; i < iters; i++) {
checksum = folly::detail::crc32_sw(buffer, blockSize);
folly::doNotOptimizeAway(checksum);
}
}
// This test fits easily in the L1 cache on modern server processors, // This test fits easily in the L1 cache on modern server processors,
// and thus it mainly measures the speed of the checksum computation. // and thus it mainly measures the speed of the checksum computation.
BENCHMARK(crc32c_hardware_1KB_block, iters) { BENCHMARK(crc32c_hardware_1KB_block, iters) {
...@@ -157,6 +221,14 @@ BENCHMARK(crc32c_software_1KB_block, iters) { ...@@ -157,6 +221,14 @@ BENCHMARK(crc32c_software_1KB_block, iters) {
benchmarkSoftwareCRC32C(iters, 1024); benchmarkSoftwareCRC32C(iters, 1024);
} }
BENCHMARK(crc32_hardware_1KB_block, iters) {
benchmarkHardwareCRC32(iters, 1024);
}
BENCHMARK(crc32_software_1KB_block, iters) {
benchmarkSoftwareCRC32(iters, 1024);
}
BENCHMARK_DRAW_LINE(); BENCHMARK_DRAW_LINE();
// This test is too big for the L1 cache but fits in L2 // This test is too big for the L1 cache but fits in L2
...@@ -168,6 +240,14 @@ BENCHMARK(crc32c_software_64KB_block, iters) { ...@@ -168,6 +240,14 @@ BENCHMARK(crc32c_software_64KB_block, iters) {
benchmarkSoftwareCRC32C(iters, 64 * 1024); benchmarkSoftwareCRC32C(iters, 64 * 1024);
} }
BENCHMARK(crc32_hardware_64KB_block, iters) {
benchmarkHardwareCRC32(iters, 64 * 1024);
}
BENCHMARK(crc32_software_64KB_block, iters) {
benchmarkSoftwareCRC32(iters, 64 * 1024);
}
BENCHMARK_DRAW_LINE(); BENCHMARK_DRAW_LINE();
// This test is too big for the L2 cache but fits in L3 // This test is too big for the L2 cache but fits in L3
...@@ -179,6 +259,13 @@ BENCHMARK(crc32c_software_512KB_block, iters) { ...@@ -179,6 +259,13 @@ BENCHMARK(crc32c_software_512KB_block, iters) {
benchmarkSoftwareCRC32C(iters, 512 * 1024); benchmarkSoftwareCRC32C(iters, 512 * 1024);
} }
BENCHMARK(crc32_hardware_512KB_block, iters) {
benchmarkHardwareCRC32(iters, 512 * 1024);
}
BENCHMARK(crc32_software_512KB_block, iters) {
benchmarkSoftwareCRC32(iters, 512 * 1024);
}
int main(int argc, char** argv) { int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv); testing::InitGoogleTest(&argc, argv);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment