Commit 533d54f5 authored by Yedidya Feldblum's avatar Yedidya Feldblum Committed by Facebook GitHub Bot

Tweaks to hash primitives

Summary: [Folly] Tweaks to hash primitives: twang, jenkins_rev, fnv

Reviewed By: ot, luciang

Differential Revision: D23290185

fbshipit-source-id: 080dfa8a5f357065473f3973654c91d01c683c3c
parent bfde457b
...@@ -82,7 +82,7 @@ ByteRange DataHolder::data(size_t size) const { ...@@ -82,7 +82,7 @@ ByteRange DataHolder::data(size_t size) const {
} }
uint64_t hashIOBuf(const IOBuf* buf) { uint64_t hashIOBuf(const IOBuf* buf) {
uint64_t h = folly::hash::FNV_64_HASH_START; uint64_t h = folly::hash::fnv64_hash_start;
for (auto& range : *buf) { for (auto& range : *buf) {
h = folly::hash::fnv64_buf(range.data(), range.size(), h); h = folly::hash::fnv64_buf(range.data(), range.size(), h);
} }
......
...@@ -33,18 +33,33 @@ ...@@ -33,18 +33,33 @@
#include <folly/hash/SpookyHashV2.h> #include <folly/hash/SpookyHashV2.h>
#include <folly/lang/Bits.h> #include <folly/lang/Bits.h>
/*
* Various hashing functions.
*/
namespace folly { namespace folly {
namespace hash { namespace hash {
// This is the Hash128to64 function from Google's cityhash (available namespace detail {
// under the MIT License). We use it to reduce multiple 64 bit hashes
// into a single hash. namespace {
template <typename T>
constexpr bool is_hashable_byte_v = false;
template <>
constexpr bool is_hashable_byte_v<char> = true;
template <>
constexpr bool is_hashable_byte_v<signed char> = true;
template <>
constexpr bool is_hashable_byte_v<unsigned char> = true;
} // namespace
} // namespace detail
// hash_128_to_64
//
// The Hash128to64 function from Google's cityhash (under the MIT License).
//
// We use it to reduce multiple 64 bit hashes into a single hash.
FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER("unsigned-integer-overflow") FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER("unsigned-integer-overflow")
inline uint64_t hash_128_to_64( constexpr uint64_t hash_128_to_64(
const uint64_t upper, const uint64_t upper,
const uint64_t lower) noexcept { const uint64_t lower) noexcept {
// Murmur-inspired hashing. // Murmur-inspired hashing.
...@@ -57,14 +72,11 @@ inline uint64_t hash_128_to_64( ...@@ -57,14 +72,11 @@ inline uint64_t hash_128_to_64(
return b; return b;
} }
////////////////////////////////////////////////////////////////////// // twang_mix64
//
/* // Thomas Wang 64 bit mix hash function.
* Thomas Wang 64 bit mix hash function
*/
FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER("unsigned-integer-overflow") FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER("unsigned-integer-overflow")
inline uint64_t twang_mix64(uint64_t key) noexcept { constexpr uint64_t twang_mix64(uint64_t key) noexcept {
key = (~key) + (key << 21); // key *= (1 << 21) - 1; key -= 1; key = (~key) + (key << 21); // key *= (1 << 21) - 1; key -= 1;
key = key ^ (key >> 24); key = key ^ (key >> 24);
key = key + (key << 3) + (key << 8); // key *= 1 + (1 << 3) + (1 << 8) key = key + (key << 3) + (key << 8); // key *= 1 + (1 << 3) + (1 << 8)
...@@ -75,13 +87,10 @@ inline uint64_t twang_mix64(uint64_t key) noexcept { ...@@ -75,13 +87,10 @@ inline uint64_t twang_mix64(uint64_t key) noexcept {
return key; return key;
} }
/* // twang_unmix64
* Inverse of twang_mix64 //
* // Inverse of, and slower than, twang_mix64.
* Note that twang_unmix64 is significantly slower than twang_mix64. constexpr uint64_t twang_unmix64(uint64_t key) noexcept {
*/
inline uint64_t twang_unmix64(uint64_t key) noexcept {
// See the comments in jenkins_rev_unmix32 for an explanation as to how this // See the comments in jenkins_rev_unmix32 for an explanation as to how this
// was generated // was generated
key *= 4611686016279904257U; key *= 4611686016279904257U;
...@@ -94,25 +103,23 @@ inline uint64_t twang_unmix64(uint64_t key) noexcept { ...@@ -94,25 +103,23 @@ inline uint64_t twang_unmix64(uint64_t key) noexcept {
return key; return key;
} }
/* // twang_32from64
* Thomas Wang downscaling hash function //
*/ // Thomas Wang downscaling hash function.
constexpr uint32_t twang_32from64(uint64_t key) noexcept {
inline uint32_t twang_32from64(uint64_t key) noexcept {
key = (~key) + (key << 18); key = (~key) + (key << 18);
key = key ^ (key >> 31); key = key ^ (key >> 31);
key = key * 21; key = key * 21;
key = key ^ (key >> 11); key = key ^ (key >> 11);
key = key + (key << 6); key = key + (key << 6);
key = key ^ (key >> 22); key = key ^ (key >> 22);
return (uint32_t)key; return static_cast<uint32_t>(key);
} }
/* // jenkins_rev_mix32
* Robert Jenkins' reversible 32 bit mix hash function //
*/ // Robert Jenkins' reversible 32 bit mix hash function.
constexpr uint32_t jenkins_rev_mix32(uint32_t key) noexcept {
inline uint32_t jenkins_rev_mix32(uint32_t key) noexcept {
key += (key << 12); // key *= (1 + (1 << 12)) key += (key << 12); // key *= (1 + (1 << 12))
key ^= (key >> 22); key ^= (key >> 22);
key += (key << 4); // key *= (1 + (1 << 4)) key += (key << 4); // key *= (1 + (1 << 4))
...@@ -125,14 +132,10 @@ inline uint32_t jenkins_rev_mix32(uint32_t key) noexcept { ...@@ -125,14 +132,10 @@ inline uint32_t jenkins_rev_mix32(uint32_t key) noexcept {
return key; return key;
} }
/* // jenkins_rev_unmix32
* Inverse of jenkins_rev_mix32 //
* // Inverse of, and slower than, jenkins_rev_mix32.
* Note that jenkinks_rev_unmix32 is significantly slower than constexpr uint32_t jenkins_rev_unmix32(uint32_t key) noexcept {
* jenkins_rev_mix32.
*/
inline uint32_t jenkins_rev_unmix32(uint32_t key) noexcept {
// These are the modular multiplicative inverses (in Z_2^32) of the // These are the modular multiplicative inverses (in Z_2^32) of the
// multiplication factors in jenkins_rev_mix32, in reverse order. They were // multiplication factors in jenkins_rev_mix32, in reverse order. They were
// computed using the Extended Euclidean algorithm, see // computed using the Extended Euclidean algorithm, see
...@@ -155,109 +158,149 @@ inline uint32_t jenkins_rev_unmix32(uint32_t key) noexcept { ...@@ -155,109 +158,149 @@ inline uint32_t jenkins_rev_unmix32(uint32_t key) noexcept {
return key; return key;
} }
/* // fnv
* Fowler / Noll / Vo (FNV) Hash //
* http://www.isthe.com/chongo/tech/comp/fnv/ // Fowler / Noll / Vo (FNV) Hash
*/ // http://www.isthe.com/chongo/tech/comp/fnv/
//
const uint32_t FNV_32_HASH_START = 2166136261UL; // Discouraged for poor performance in the smhasher suite.
const uint64_t FNV_64_HASH_START = 14695981039346656037ULL;
const uint64_t FNVA_64_HASH_START = 14695981039346656037ULL; constexpr uint32_t fnv32_hash_start = 2166136261UL;
constexpr uint64_t fnv64_hash_start = 14695981039346656037ULL;
inline uint32_t fnv32( constexpr uint64_t fnva64_hash_start = 14695981039346656037ULL;
const char* buf,
uint32_t hash = FNV_32_HASH_START) noexcept { // fnv32_append(
constexpr uint32_t fnv32_append_byte(uint32_t hash, uint8_t c) {
hash = hash //
+ (hash << 1) //
+ (hash << 4) //
+ (hash << 7) //
+ (hash << 8) //
+ (hash << 24);
// forcing signed char, since other platforms can use unsigned // forcing signed char, since other platforms can use unsigned
const signed char* s = reinterpret_cast<const signed char*>(buf); hash ^= static_cast<int8_t>(c);
return hash;
}
for (; *s; ++s) { // fnv32_buf
hash += template <typename C, std::enable_if_t<detail::is_hashable_byte_v<C>, int> = 0>
(hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24); constexpr uint32_t
hash ^= *s; fnv32_buf(const C* buf, size_t n, uint32_t hash = fnv32_hash_start) noexcept {
for (size_t i = 0; i < n; ++i) {
hash = fnv32_append_byte(hash, static_cast<uint8_t>(buf[i]));
} }
return hash; return hash;
} }
inline uint32_t fnv32_buf( inline uint32_t fnv32_buf(
const void* buf, const void* buf,
size_t n, size_t n,
uint32_t hash = FNV_32_HASH_START) noexcept { uint32_t hash = fnv32_hash_start) noexcept {
// forcing signed char, since other platforms can use unsigned return fnv32_buf(reinterpret_cast<const uint8_t*>(buf), n, hash);
const signed char* char_buf = reinterpret_cast<const signed char*>(buf); }
for (size_t i = 0; i < n; ++i) { // fnv32
hash += constexpr uint32_t fnv32(
(hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24); const char* buf,
hash ^= char_buf[i]; uint32_t hash = fnv32_hash_start) noexcept {
for (; *buf; ++buf) {
hash = fnv32_append_byte(hash, static_cast<uint8_t>(*buf));
} }
return hash; return hash;
} }
// fnv32
inline uint32_t fnv32( inline uint32_t fnv32(
const std::string& str, const std::string& str,
uint32_t hash = FNV_32_HASH_START) noexcept { uint32_t hash = fnv32_hash_start) noexcept {
return fnv32_buf(str.data(), str.size(), hash); return fnv32_buf(str.data(), str.size(), hash);
} }
inline uint64_t fnv64( // fnv64_append_byte
const char* buf, constexpr uint64_t fnv64_append_byte(uint64_t hash, uint8_t c) {
uint64_t hash = FNV_64_HASH_START) noexcept { hash = hash //
+ (hash << 1) //
+ (hash << 4) //
+ (hash << 5) //
+ (hash << 7) //
+ (hash << 8) //
+ (hash << 40);
// forcing signed char, since other platforms can use unsigned // forcing signed char, since other platforms can use unsigned
const signed char* s = reinterpret_cast<const signed char*>(buf); hash ^= static_cast<int8_t>(c);
return hash;
}
for (; *s; ++s) { // fnv64_buf
hash += (hash << 1) + (hash << 4) + (hash << 5) + (hash << 7) + template <typename C, std::enable_if_t<detail::is_hashable_byte_v<C>, int> = 0>
(hash << 8) + (hash << 40); constexpr uint64_t
hash ^= *s; fnv64_buf(const C* buf, size_t n, uint64_t hash = fnv64_hash_start) noexcept {
for (size_t i = 0; i < n; ++i) {
hash = fnv64_append_byte(hash, static_cast<uint8_t>(buf[i]));
} }
return hash; return hash;
} }
inline uint64_t fnv64_buf( inline uint64_t fnv64_buf(
const void* buf, const void* buf,
size_t n, size_t n,
uint64_t hash = FNV_64_HASH_START) noexcept { uint64_t hash = fnv64_hash_start) noexcept {
// forcing signed char, since other platforms can use unsigned return fnv64_buf(reinterpret_cast<const uint8_t*>(buf), n, hash);
const signed char* char_buf = reinterpret_cast<const signed char*>(buf); }
for (size_t i = 0; i < n; ++i) { // fnv64
hash += (hash << 1) + (hash << 4) + (hash << 5) + (hash << 7) + constexpr uint64_t fnv64(
(hash << 8) + (hash << 40); const char* buf,
hash ^= char_buf[i]; uint64_t hash = fnv64_hash_start) noexcept {
for (; *buf; ++buf) {
hash = fnv64_append_byte(hash, static_cast<uint8_t>(*buf));
} }
return hash; return hash;
} }
// fnv64
inline uint64_t fnv64( inline uint64_t fnv64(
const std::string& str, const std::string& str,
uint64_t hash = FNV_64_HASH_START) noexcept { uint64_t hash = fnv64_hash_start) noexcept {
return fnv64_buf(str.data(), str.size(), hash); return fnv64_buf(str.data(), str.size(), hash);
} }
inline uint64_t fnva64_buf( // fnva64_append_byte
const void* buf, constexpr uint64_t fnva64_append_byte(uint64_t hash, uint8_t c) {
size_t n, hash ^= c;
uint64_t hash = FNVA_64_HASH_START) noexcept { hash = hash //
const uint8_t* char_buf = reinterpret_cast<const uint8_t*>(buf); + (hash << 1) //
+ (hash << 4) //
+ (hash << 5) //
+ (hash << 7) //
+ (hash << 8) //
+ (hash << 40);
return hash;
}
// fnva64_buf
template <typename C, std::enable_if_t<detail::is_hashable_byte_v<C>, int> = 0>
constexpr uint64_t
fnva64_buf(const C* buf, size_t n, uint64_t hash = fnva64_hash_start) noexcept {
for (size_t i = 0; i < n; ++i) { for (size_t i = 0; i < n; ++i) {
hash ^= char_buf[i]; hash = fnva64_append_byte(hash, static_cast<uint8_t>(buf[i]));
hash += (hash << 1) + (hash << 4) + (hash << 5) + (hash << 7) +
(hash << 8) + (hash << 40);
} }
return hash; return hash;
} }
inline uint64_t fnva64_buf(
const void* buf,
size_t n,
uint64_t hash = fnva64_hash_start) noexcept {
return fnva64_buf(reinterpret_cast<const uint8_t*>(buf), n, hash);
}
// fnva64
inline uint64_t fnva64( inline uint64_t fnva64(
const std::string& str, const std::string& str,
uint64_t hash = FNVA_64_HASH_START) noexcept { uint64_t hash = fnva64_hash_start) noexcept {
return fnva64_buf(str.data(), str.size(), hash); return fnva64_buf(str.data(), str.size(), hash);
} }
/* // hsieh
* Paul Hsieh: http://www.azillionmonkeys.com/qed/hash.html //
*/ // Paul Hsieh: http://www.azillionmonkeys.com/qed/hash.html
#define get16bits(d) folly::loadUnaligned<uint16_t>(d) #define get16bits(d) folly::loadUnaligned<uint16_t>(d)
...@@ -324,8 +367,6 @@ inline uint32_t hsieh_hash32_str(const std::string& str) noexcept { ...@@ -324,8 +367,6 @@ inline uint32_t hsieh_hash32_str(const std::string& str) noexcept {
return hsieh_hash32_buf(str.data(), str.size()); return hsieh_hash32_buf(str.data(), str.size());
} }
//////////////////////////////////////////////////////////////////////
} // namespace hash } // namespace hash
namespace detail { namespace detail {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment