Commit ff25e3b5 authored by Tudor Bosman's avatar Tudor Bosman

Add Bob Jenkins's SpookyHash to folly.

Summary:
http://burtleburtle.net/bob/hash/spooky.html
The code is in the public domain, so republishing it as such is fine.
The code needed modifications (the test was designed to run on Windows
only; removed unhygienic typedefs; moved to folly::hash namespace)

Also updated Makefile.am files.

Test Plan: test added

Reviewed By: soren@fb.com

FB internal diff: D549572
parent ce15293f
main 2016.07.26 v2022.02.14.00 v2022.02.07.00 v2022.01.31.00 v2022.01.24.00 v2022.01.17.00 v2022.01.10.00 v2022.01.03.00 v2021.12.27.00 v2021.12.20.00 v2021.12.13.00 v2021.12.06.00 v2021.11.29.00 v2021.11.15.00 v2021.11.08.00 v2021.11.01.00 v2021.10.25.00 v2021.10.18.00 v2021.10.11.00 v2021.10.04.00 v2021.09.27.00 v2021.09.20.00 v2021.09.13.00 v2021.09.06.00 v2021.08.30.00 v2021.08.23.00 v2021.08.02.00 v2021.07.22.00 v2021.07.20.01 v2021.07.20.00 v2021.06.28.00 v2021.06.14.00 v2021.06.07.00 v2021.05.31.00 v2021.05.24.00 v2021.05.17.00 v2021.05.10.00 v2021.05.03.00 v2021.04.26.00 v2021.04.19.00 v2021.04.12.00 v2021.04.05.00 v2021.03.29.00 v2021.03.22.00 v2021.03.15.00 v2021.03.08.00 v2021.03.01.00 v2021.02.22.00 v2021.02.15.00 v2021.02.08.00 v2021.02.01.00 v2021.01.25.00 v2021.01.18.01 v2021.01.18.00 v2021.01.11.00 v2021.01.04.00 v2020.12.28.00 v2020.12.21.00 v2020.12.14.00 v2020.12.07.00 v2020.11.30.00 v2020.11.23.00 v2020.11.16.00 v2020.11.09.00 v2020.11.02.00 v2020.10.26.00 v2020.10.19.00 v2020.10.12.00 v2020.10.05.00 v2020.09.28.00 v2020.09.21.00 v2020.09.14.00 v2020.09.07.00 v2020.08.31.00 v2020.08.24.00 v2020.08.17.00 v2020.08.10.00 v2020.08.03.00 v2020.07.27.00 v2020.07.20.00 v2020.07.13.00 v2020.07.06.00 v2020.06.29.00 v2020.06.15.00 v2020.06.08.00 v2020.06.01.00 v2020.05.25.00 v2020.05.18.00 v2020.05.11.00 v2020.05.04.00 v2020.04.27.00 v2020.04.20.00 v2020.04.13.00 v2020.04.06.00 v2020.03.30.00 v2020.03.23.00 v2020.03.16.00 v2020.03.09.00 v2020.03.02.00 v2020.02.24.00 v2020.02.17.00 v2020.02.10.00 v2020.02.03.00 v2020.01.27.00 v2020.01.20.00 v2020.01.13.00 v2020.01.06.00 v2019.12.30.00 v2019.12.23.00 v2019.12.16.00 v2019.12.09.00 v2019.12.06.00 v2019.12.02.00 v2019.11.11.00 v2019.11.04.00 v2019.10.28.00 v2019.10.21.00 v2019.10.14.00 v2019.10.07.00 v2019.09.30.00 v2019.09.23.00 v2019.09.16.00 v2019.09.09.00 v2019.09.02.00 v2019.08.26.00 v2019.08.19.00 v2019.08.12.00 v2019.08.05.00 v2019.07.29.00 v2019.07.22.00 v2019.06.17.00 v2019.06.10.00 v2019.06.03.00 v2019.05.27.00 v2019.05.20.00 v2019.05.13.00 v2019.05.06.00 v2019.04.29.00 v2019.04.22.00 v2019.04.15.00 v2019.04.08.00 v2019.04.01.00 v2019.03.25.00 v2019.03.18.00 v2019.03.04.00 v2019.02.25.00 v2019.02.18.00 v2019.02.11.00 v2019.02.04.00 v2019.01.28.00 v2019.01.21.00 v2019.01.14.00 v2019.01.07.00 v2018.12.31.00 v2018.12.24.00 v2018.12.17.00 v2018.12.10.00 v2018.12.03.00 v2018.11.26.00 v2018.11.19.00 v2018.11.12.00 v2018.11.05.00 v2018.10.29.00 v2018.10.22.00 v2018.10.15.00 v2018.10.08.00 v2018.10.01.00 v2018.09.24.00 v2018.09.17.00 v2018.09.10.01 v2018.09.10.00 v2018.09.03.01 v2018.09.03.00 v2018.08.27.00 v2018.08.20.00 v2018.08.13.00 v2018.08.09.00 v2018.08.06.00 v2018.07.30.00 v2018.07.23.00 v2018.07.16.00 v2018.07.09.00 v2018.07.02.00 v2018.06.25.00 v2018.06.18.00 v2018.06.11.00 v2018.06.04.00 v2018.05.28.00 v2018.05.21.00 v2018.05.14.00 v2018.05.07.00 v2018.04.30.00 v2018.04.23.00 v2018.04.16.00 v2018.04.09.00 v2018.04.02.00 v2018.03.26.00 v2018.03.19.00 v2018.03.12.00 v2018.03.05.00 v2018.02.26.00 v2018.02.19.00 v2018.02.12.00 v2018.02.05.00 v2018.01.29.00 v2018.01.22.00 v2018.01.15.00 v2018.01.08.00 v2018.01.01.00 v2017.12.25.00 v2017.12.18.00 v2017.12.11.00 v2017.12.04.00 v2017.11.27.00 v2017.11.20.00 v2017.11.13.00 v2017.11.06.00 v2017.10.30.00 v2017.10.23.00 v2017.10.16.00 v2017.10.09.00 v2017.10.02.00 v2017.09.25.00 v2017.09.18.00 v2017.09.11.00 v2017.09.04.00 v2017.08.28.00 v2017.08.21.00 v2017.08.14.00 v2017.08.07.00 v2017.07.31.00 v2017.07.24.00 v2017.07.17.01 v2017.07.17.00 v2017.07.10.00 v2017.07.03.00 v2017.06.26.01 v2017.06.26.00 v2017.06.19.00 v2017.06.12.00 v2017.06.05.00 v2017.05.29.00 v2017.05.22.00 v2017.05.15.00 v2017.05.08.00 v2017.05.01.00 v2017.04.24.00 v2017.04.17.00 v2017.04.10.00 v2017.04.03.00 v2017.03.27.00 v2017.03.20.00 v2017.03.13.00 v2017.03.06.00 v2016.12.19.00 v2016.12.12.00 v2016.12.05.00 v2016.11.28.00 v2016.11.21.00 v2016.11.14.00 v2016.11.07.00 v2016.10.31.00 v2016.10.24.00 v2016.10.17.00 v2016.10.10.00 v2016.10.03.00 v2016.09.26.00 v2016.09.19.00 v2016.09.12.01 v2016.09.12.00 v2016.09.05.00 v2016.08.29.00 v2016.08.22.00 v2016.08.15.00 v2016.08.08.00 v2016.08.01.00 v2016.07.29.00 v0.57.0 v0.56.0 v0.55.0 v0.54.0 v0.53.0 v0.52.0 v0.51.0 v0.50.0 v0.49.1 v0.49.0 v0.48.0 v0.47.0 v0.45.0 v0.43.0 v0.42.0 v0.41.0 v0.40.0 v0.39.0 v0.38.0 v0.37.0 v0.36.0 v0.35.0 v0.34.0 v0.33.0 v0.32.0 v0.31.0 v0.30.0 v0.29.0 v0.28.0 v0.27.0 v0.26.0 v0.25.0 v0.24.0 v0.23.0 v0.22.0 deprecate-dynamic-initializer
...@@ -22,6 +22,8 @@ ...@@ -22,6 +22,8 @@
#include <string> #include <string>
#include <utility> #include <utility>
#include "folly/SpookyHash.h"
/* /*
* Various hashing functions. * Various hashing functions.
*/ */
......
...@@ -66,6 +66,7 @@ nobase_follyinclude_HEADERS = \ ...@@ -66,6 +66,7 @@ nobase_follyinclude_HEADERS = \
SmallLocks.h \ SmallLocks.h \
small_vector.h \ small_vector.h \
sorted_vector_types.h \ sorted_vector_types.h \
SpookyHash.h \
StlAllocator.h \ StlAllocator.h \
String.h \ String.h \
String-inl.h \ String-inl.h \
...@@ -106,7 +107,8 @@ libfolly_la_SOURCES = \ ...@@ -106,7 +107,8 @@ libfolly_la_SOURCES = \
Format.cpp \ Format.cpp \
FormatTables.cpp \ FormatTables.cpp \
String.cpp \ String.cpp \
Bits.cpp Bits.cpp \
SpookyHash.cpp
libfolly_la_LIBADD = $(BOOST_THREAD_LIB) -lpthread libfolly_la_LIBADD = $(BOOST_THREAD_LIB) -lpthread
......
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Spooky Hash
// A 128-bit noncryptographic hash, for checksums and table lookup
// By Bob Jenkins. Public domain.
// Oct 31 2010: published framework, disclaimer ShortHash isn't right
// Nov 7 2010: disabled ShortHash
// Oct 31 2011: replace End, ShortMix, ShortEnd, enable ShortHash again
// April 10 2012: buffer overflow on platforms without unaligned reads
// July 12 2012: was passing out variables in final to in/out in short
// July 30 2012: I reintroduced the buffer overflow
#include "folly/SpookyHash.h"
#include <cstring>
#define ALLOW_UNALIGNED_READS 1
namespace folly {
namespace hash {
//
// short hash ... it could be used on any message,
// but it's used by Spooky just for short messages.
//
void SpookyHash::Short(
const void *message,
size_t length,
uint64_t *hash1,
uint64_t *hash2)
{
uint64_t buf[2*sc_numVars];
union
{
const uint8_t *p8;
uint32_t *p32;
uint64_t *p64;
size_t i;
} u;
u.p8 = (const uint8_t *)message;
if (!ALLOW_UNALIGNED_READS && (u.i & 0x7))
{
memcpy(buf, message, length);
u.p64 = buf;
}
size_t remainder = length%32;
uint64_t a=*hash1;
uint64_t b=*hash2;
uint64_t c=sc_const;
uint64_t d=sc_const;
if (length > 15)
{
const uint64_t *end = u.p64 + (length/32)*4;
// handle all complete sets of 32 bytes
for (; u.p64 < end; u.p64 += 4)
{
c += u.p64[0];
d += u.p64[1];
ShortMix(a,b,c,d);
a += u.p64[2];
b += u.p64[3];
}
//Handle the case of 16+ remaining bytes.
if (remainder >= 16)
{
c += u.p64[0];
d += u.p64[1];
ShortMix(a,b,c,d);
u.p64 += 2;
remainder -= 16;
}
}
// Handle the last 0..15 bytes, and its length
d = ((uint64_t)length) << 56;
switch (remainder)
{
case 15:
d += ((uint64_t)u.p8[14]) << 48;
case 14:
d += ((uint64_t)u.p8[13]) << 40;
case 13:
d += ((uint64_t)u.p8[12]) << 32;
case 12:
d += u.p32[2];
c += u.p64[0];
break;
case 11:
d += ((uint64_t)u.p8[10]) << 16;
case 10:
d += ((uint64_t)u.p8[9]) << 8;
case 9:
d += (uint64_t)u.p8[8];
case 8:
c += u.p64[0];
break;
case 7:
c += ((uint64_t)u.p8[6]) << 48;
case 6:
c += ((uint64_t)u.p8[5]) << 40;
case 5:
c += ((uint64_t)u.p8[4]) << 32;
case 4:
c += u.p32[0];
break;
case 3:
c += ((uint64_t)u.p8[2]) << 16;
case 2:
c += ((uint64_t)u.p8[1]) << 8;
case 1:
c += (uint64_t)u.p8[0];
break;
case 0:
c += sc_const;
d += sc_const;
}
ShortEnd(a,b,c,d);
*hash1 = a;
*hash2 = b;
}
// do the whole hash in one call
void SpookyHash::Hash128(
const void *message,
size_t length,
uint64_t *hash1,
uint64_t *hash2)
{
if (length < sc_bufSize)
{
Short(message, length, hash1, hash2);
return;
}
uint64_t h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11;
uint64_t buf[sc_numVars];
uint64_t *end;
union
{
const uint8_t *p8;
uint64_t *p64;
size_t i;
} u;
size_t remainder;
h0=h3=h6=h9 = *hash1;
h1=h4=h7=h10 = *hash2;
h2=h5=h8=h11 = sc_const;
u.p8 = (const uint8_t *)message;
end = u.p64 + (length/sc_blockSize)*sc_numVars;
// handle all whole sc_blockSize blocks of bytes
if (ALLOW_UNALIGNED_READS || ((u.i & 0x7) == 0))
{
while (u.p64 < end)
{
Mix(u.p64, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
u.p64 += sc_numVars;
}
}
else
{
while (u.p64 < end)
{
memcpy(buf, u.p64, sc_blockSize);
Mix(buf, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
u.p64 += sc_numVars;
}
}
// handle the last partial block of sc_blockSize bytes
remainder = (length - ((const uint8_t *)end-(const uint8_t *)message));
memcpy(buf, end, remainder);
memset(((uint8_t *)buf)+remainder, 0, sc_blockSize-remainder);
((uint8_t *)buf)[sc_blockSize-1] = remainder;
Mix(buf, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
// do some final mixing
End(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
*hash1 = h0;
*hash2 = h1;
}
// init spooky state
void SpookyHash::Init(uint64_t seed1, uint64_t seed2)
{
m_length = 0;
m_remainder = 0;
m_state[0] = seed1;
m_state[1] = seed2;
}
// add a message fragment to the state
void SpookyHash::Update(const void *message, size_t length)
{
uint64_t h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11;
size_t newLength = length + m_remainder;
uint8_t remainder;
union
{
const uint8_t *p8;
uint64_t *p64;
size_t i;
} u;
const uint64_t *end;
// Is this message fragment too short? If it is, stuff it away.
if (newLength < sc_bufSize)
{
memcpy(&((uint8_t *)m_data)[m_remainder], message, length);
m_length = length + m_length;
m_remainder = (uint8_t)newLength;
return;
}
// init the variables
if (m_length < sc_bufSize)
{
h0=h3=h6=h9 = m_state[0];
h1=h4=h7=h10 = m_state[1];
h2=h5=h8=h11 = sc_const;
}
else
{
h0 = m_state[0];
h1 = m_state[1];
h2 = m_state[2];
h3 = m_state[3];
h4 = m_state[4];
h5 = m_state[5];
h6 = m_state[6];
h7 = m_state[7];
h8 = m_state[8];
h9 = m_state[9];
h10 = m_state[10];
h11 = m_state[11];
}
m_length = length + m_length;
// if we've got anything stuffed away, use it now
if (m_remainder)
{
uint8_t prefix = sc_bufSize-m_remainder;
memcpy(&(((uint8_t *)m_data)[m_remainder]), message, prefix);
u.p64 = m_data;
Mix(u.p64, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
Mix(&u.p64[sc_numVars], h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
u.p8 = ((const uint8_t *)message) + prefix;
length -= prefix;
}
else
{
u.p8 = (const uint8_t *)message;
}
// handle all whole blocks of sc_blockSize bytes
end = u.p64 + (length/sc_blockSize)*sc_numVars;
remainder = (uint8_t)(length-((const uint8_t *)end-u.p8));
if (ALLOW_UNALIGNED_READS || (u.i & 0x7) == 0)
{
while (u.p64 < end)
{
Mix(u.p64, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
u.p64 += sc_numVars;
}
}
else
{
while (u.p64 < end)
{
memcpy(m_data, u.p8, sc_blockSize);
Mix(m_data, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
u.p64 += sc_numVars;
}
}
// stuff away the last few bytes
m_remainder = remainder;
memcpy(m_data, end, remainder);
// stuff away the variables
m_state[0] = h0;
m_state[1] = h1;
m_state[2] = h2;
m_state[3] = h3;
m_state[4] = h4;
m_state[5] = h5;
m_state[6] = h6;
m_state[7] = h7;
m_state[8] = h8;
m_state[9] = h9;
m_state[10] = h10;
m_state[11] = h11;
}
// report the hash for the concatenation of all message fragments so far
void SpookyHash::Final(uint64_t *hash1, uint64_t *hash2)
{
// init the variables
if (m_length < sc_bufSize)
{
*hash1 = m_state[0];
*hash2 = m_state[1];
Short( m_data, m_length, hash1, hash2);
return;
}
const uint64_t *data = (const uint64_t *)m_data;
uint8_t remainder = m_remainder;
uint64_t h0 = m_state[0];
uint64_t h1 = m_state[1];
uint64_t h2 = m_state[2];
uint64_t h3 = m_state[3];
uint64_t h4 = m_state[4];
uint64_t h5 = m_state[5];
uint64_t h6 = m_state[6];
uint64_t h7 = m_state[7];
uint64_t h8 = m_state[8];
uint64_t h9 = m_state[9];
uint64_t h10 = m_state[10];
uint64_t h11 = m_state[11];
if (remainder >= sc_blockSize)
{
// m_data can contain two blocks; handle any whole first block
Mix(data, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
data += sc_numVars;
remainder -= sc_blockSize;
}
// mix in the last partial block, and the length mod sc_blockSize
memset(&((uint8_t *)data)[remainder], 0, (sc_blockSize-remainder));
((uint8_t *)data)[sc_blockSize-1] = remainder;
Mix(data, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
// do some final mixing
End(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
*hash1 = h0;
*hash2 = h1;
}
} // namespace hash
} // namespace folly
This diff is collapsed.
...@@ -165,4 +165,16 @@ fingerprint_test_SOURCES = FingerprintTest.cpp ...@@ -165,4 +165,16 @@ fingerprint_test_SOURCES = FingerprintTest.cpp
fingerprint_test_LDADD = libgtest.la $(top_builddir)/libfolly.la $(top_builddir)/libfollyfingerprint.la $(top_builddir)/libfollybenchmark.la fingerprint_test_LDADD = libgtest.la $(top_builddir)/libfolly.la $(top_builddir)/libfollyfingerprint.la $(top_builddir)/libfollybenchmark.la
TESTS += fingerprint_test TESTS += fingerprint_test
portability_test_SOURCES = PortabilityTest.cpp
portability_test_LDADD = libgtestmain.la $(top_builddir)/libfolly.la
TESTS += portability_test
cpuid_test_SOURCES = CpuIdTest.cpp
cpuid_test_LDADD = libgtestmain.la $(top_builddir)/libfolly.la
TESTS += cpuid_test
spooky_hash_test_SOURCES = SpookyHashTest.cpp
spooky_hash_test_LDADD = $(top_builddir)/libfolly.la
TESTS += spooky_hash_test
check_PROGRAMS= $(TESTS) check_PROGRAMS= $(TESTS)
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment