Commit 11e98e93 authored by Philip Pronin's avatar Philip Pronin Committed by Facebook Github Bot

switch back to inline assembly in compression::instructions

Summary:
D3278901 never worked, GCC (at least 4.9) failed to propagate
`FOLLY_TARGET_ATTRIBUTE("arch=haswell")` down, `blsr` and `select64`
weren't inlined in EF coding on Haswell, showing up in CPU profile.

This diff switches back to inline assembly, removing second dispatching
mechanism (IFUNC; in additional to caller-side `Default` / `Nehalem` / `Haswell`
dispatching) for `compression::instructions`, and disables functionality on
compilers not supporting AT&T syntax (MSVC++) for now.

Added `FOLLY_ALWAYS_INLINE` to make sure failure to inline results in
compilation failure.

Reviewed By: ot

Differential Revision: D3959438

fbshipit-source-id: e40573fbfbf38991caa2cd70293aeaeeec3afad7
parent 806a09e4
...@@ -17,15 +17,18 @@ ...@@ -17,15 +17,18 @@
#pragma once #pragma once
#include <glog/logging.h> #include <glog/logging.h>
#include <immintrin.h>
#ifdef __clang__
// Clang defines the intrinsics in weird places.
#include <popcntintrin.h>
#endif
#include <folly/CpuId.h> #include <folly/CpuId.h>
#include <folly/Portability.h>
#include <folly/portability/Builtins.h> #include <folly/portability/Builtins.h>
#if defined(__GNUC__) || defined(__clang__)
// For compilers supporting AT&T assembly syntax.
#define FOLLY_INSTRUCTIONS_SUPPORTED 1
#else
#define FOLLY_INSTRUCTIONS_SUPPORTED 0
#endif
namespace folly { namespace compression { namespace instructions { namespace folly { namespace compression { namespace instructions {
// NOTE: It's recommended to compile EF coding with -msse4.2, starting // NOTE: It's recommended to compile EF coding with -msse4.2, starting
...@@ -37,39 +40,37 @@ namespace folly { namespace compression { namespace instructions { ...@@ -37,39 +40,37 @@ namespace folly { namespace compression { namespace instructions {
// use explicitly. // use explicitly.
struct Default { struct Default {
static bool supported(const folly::CpuId& /* cpuId */ = {}) { return true; } static bool supported(const folly::CpuId& /* cpuId */ = {}) {
static inline uint64_t popcount(uint64_t value) { return true;
}
static FOLLY_ALWAYS_INLINE uint64_t popcount(uint64_t value) {
return __builtin_popcountll(value); return __builtin_popcountll(value);
} }
static inline int ctz(uint64_t value) { static FOLLY_ALWAYS_INLINE int ctz(uint64_t value) {
DCHECK_GT(value, 0); DCHECK_GT(value, 0);
return __builtin_ctzll(value); return __builtin_ctzll(value);
} }
static inline int clz(uint64_t value) { static FOLLY_ALWAYS_INLINE int clz(uint64_t value) {
DCHECK_GT(value, 0); DCHECK_GT(value, 0);
return __builtin_clzll(value); return __builtin_clzll(value);
} }
static inline uint64_t blsr(uint64_t value) { static FOLLY_ALWAYS_INLINE uint64_t blsr(uint64_t value) {
return value & (value - 1); return value & (value - 1);
} }
}; };
#if FOLLY_INSTRUCTIONS_SUPPORTED
struct Nehalem : public Default { struct Nehalem : public Default {
static bool supported(const folly::CpuId& cpuId = {}) { static bool supported(const folly::CpuId& cpuId = {}) {
return cpuId.popcnt(); return cpuId.popcnt();
} }
FOLLY_TARGET_ATTRIBUTE("popcnt") static FOLLY_ALWAYS_INLINE uint64_t popcount(uint64_t value) {
static inline uint64_t popcount(uint64_t value) {
// POPCNT is supported starting with Intel Nehalem, AMD K10. // POPCNT is supported starting with Intel Nehalem, AMD K10.
#if defined(__GNUC__) && !defined(__clang__) && !__GNUC_PREREQ(4, 9)
// GCC 4.8 doesn't support the intrinsics.
uint64_t result; uint64_t result;
asm ("popcntq %1, %0" : "=r" (result) : "r" (value)); asm ("popcntq %1, %0" : "=r" (result) : "r" (value));
return result; return result;
#else
return _mm_popcnt_u64(value);
#endif
} }
}; };
...@@ -78,19 +79,20 @@ struct Haswell : public Nehalem { ...@@ -78,19 +79,20 @@ struct Haswell : public Nehalem {
return Nehalem::supported(cpuId) && cpuId.bmi1(); return Nehalem::supported(cpuId) && cpuId.bmi1();
} }
FOLLY_TARGET_ATTRIBUTE("bmi") static FOLLY_ALWAYS_INLINE uint64_t blsr(uint64_t value) {
static inline uint64_t blsr(uint64_t value) {
// BMI1 is supported starting with Intel Haswell, AMD Piledriver. // BMI1 is supported starting with Intel Haswell, AMD Piledriver.
// BLSR combines two instuctions into one and reduces register pressure. // BLSR combines two instuctions into one and reduces register pressure.
#if defined(__GNUC__) && !defined(__clang__) && !__GNUC_PREREQ(4, 9)
// GCC 4.8 doesn't support the intrinsics.
uint64_t result; uint64_t result;
asm ("blsrq %1, %0" : "=r" (result) : "r" (value)); asm ("blsrq %1, %0" : "=r" (result) : "r" (value));
return result; return result;
#else
return _blsr_u64(value);
#endif
} }
}; };
}}} // namespaces #else // FOLLY_INSTRUCTIONS_SUPPORTED
struct Nehalem : public Default {};
struct Haswell : public Nehalem {};
#endif // FOLLY_INSTRUCTIONS_SUPPORTED
}}} // namespaces
...@@ -18,13 +18,14 @@ ...@@ -18,13 +18,14 @@
#include <glog/logging.h> #include <glog/logging.h>
#include <folly/Portability.h>
#include <folly/experimental/Instructions.h> #include <folly/experimental/Instructions.h>
namespace folly { namespace folly {
namespace detail { namespace detail {
extern const uint8_t kSelectInByte[2048]; extern const uint8_t kSelectInByte[2048];
} } // namespace detail
/** /**
* Returns the position of the k-th 1 in the 64-bit word x. * Returns the position of the k-th 1 in the 64-bit word x.
...@@ -62,12 +63,11 @@ inline uint64_t select64(uint64_t x, uint64_t k) { ...@@ -62,12 +63,11 @@ inline uint64_t select64(uint64_t x, uint64_t k) {
return place + detail::kSelectInByte[((x >> place) & 0xFF) | (byteRank << 8)]; return place + detail::kSelectInByte[((x >> place) & 0xFF) | (byteRank << 8)];
} }
#if FOLLY_INSTRUCTIONS_SUPPORTED
template <> template <>
FOLLY_TARGET_ATTRIBUTE("bmi,bmi2") FOLLY_ALWAYS_INLINE uint64_t
inline uint64_t select64<compression::instructions::Haswell>(uint64_t x, select64<compression::instructions::Haswell>(uint64_t x, uint64_t k) {
uint64_t k) {
#if defined(__GNUC__) && !defined(__clang__) && !__GNUC_PREREQ(4, 9)
// GCC 4.8 doesn't support the intrinsics.
uint64_t result = uint64_t(1) << k; uint64_t result = uint64_t(1) << k;
asm("pdep %1, %0, %0\n\t" asm("pdep %1, %0, %0\n\t"
...@@ -76,9 +76,8 @@ inline uint64_t select64<compression::instructions::Haswell>(uint64_t x, ...@@ -76,9 +76,8 @@ inline uint64_t select64<compression::instructions::Haswell>(uint64_t x,
: "r"(x)); : "r"(x));
return result; return result;
#else
return _tzcnt_u64(_pdep_u64(1ULL << k, x));
#endif
} }
#endif // FOLLY_INSTRUCTIONS_SUPPORTED
} // namespace folly } // namespace folly
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment