Commit 166dfd2b authored by Philip Pronin's avatar Philip Pronin Committed by Sara Golemon

tune EF coding

Summary:
Change the way forward / skip pointers are encoded, so we can
expect that `uint32_t` will be enough to address ~4B elements instead
of ~2B as it is now.

Test Plan:
Ran benchmarks for both versions, saw no significant
difference.

Added tests.

Reviewed By: lucian@fb.com

FB internal diff: D1156950
parent 07580800
...@@ -32,12 +32,13 @@ ...@@ -32,12 +32,13 @@
#error EliasFanoCoding.h requires x86_64 #error EliasFanoCoding.h requires x86_64
#endif #endif
#include <cstdlib>
#include <algorithm> #include <algorithm>
#include <cstdlib>
#include <limits> #include <limits>
#include <type_traits> #include <type_traits>
#include <boost/noncopyable.hpp> #include <boost/noncopyable.hpp>
#include <glog/logging.h> #include <glog/logging.h>
#include "folly/Bits.h" #include "folly/Bits.h"
#include "folly/CpuId.h" #include "folly/CpuId.h"
#include "folly/Likely.h" #include "folly/Likely.h"
...@@ -49,23 +50,16 @@ ...@@ -49,23 +50,16 @@
namespace folly { namespace compression { namespace folly { namespace compression {
template <class Value,
class SkipValue = size_t,
size_t kSkipQuantum = 0, // 0 = disabled
size_t kForwardQuantum = 0> // 0 = disabled
struct EliasFanoCompressedList { struct EliasFanoCompressedList {
static_assert(std::is_integral<Value>::value &&
std::is_unsigned<Value>::value,
"Value should be unsigned integral");
typedef Value ValueType;
typedef SkipValue SkipValueType;
EliasFanoCompressedList() EliasFanoCompressedList()
: size(0), numLowerBits(0) { } : size(0), numLowerBits(0) { }
static constexpr size_t skipQuantum = kSkipQuantum; void free() {
static constexpr size_t forwardQuantum = kForwardQuantum; ::free(const_cast<unsigned char*>(lower.data()));
::free(const_cast<unsigned char*>(upper.data()));
::free(const_cast<unsigned char*>(skipPointers.data()));
::free(const_cast<unsigned char*>(forwardPointers.data()));
}
size_t size; size_t size;
uint8_t numLowerBits; uint8_t numLowerBits;
...@@ -79,13 +73,30 @@ struct EliasFanoCompressedList { ...@@ -79,13 +73,30 @@ struct EliasFanoCompressedList {
folly::ByteRange skipPointers; folly::ByteRange skipPointers;
folly::ByteRange forwardPointers; folly::ByteRange forwardPointers;
};
void free() { // Version history:
::free(const_cast<unsigned char*>(lower.data())); // In version 1 skip / forward pointers encoding has been changed,
::free(const_cast<unsigned char*>(upper.data())); // so SkipValue = uint32_t is able to address up to ~4B elements,
::free(const_cast<unsigned char*>(skipPointers.data())); // instead of only ~2B.
::free(const_cast<unsigned char*>(forwardPointers.data())); template <class Value,
} class SkipValue = size_t,
size_t kSkipQuantum = 0, // 0 = disabled
size_t kForwardQuantum = 0, // 0 = disabled
size_t kVersion = 0>
struct EliasFanoEncoder {
static_assert(std::is_integral<Value>::value &&
std::is_unsigned<Value>::value,
"Value should be unsigned integral");
typedef EliasFanoCompressedList CompressedList;
typedef Value ValueType;
typedef SkipValue SkipValueType;
static constexpr size_t skipQuantum = kSkipQuantum;
static constexpr size_t forwardQuantum = kForwardQuantum;
static constexpr size_t version = kVersion;
static uint8_t defaultNumLowerBits(size_t upperBound, size_t size) { static uint8_t defaultNumLowerBits(size_t upperBound, size_t size) {
if (size == 0 || upperBound < size) { if (size == 0 || upperBound < size) {
...@@ -166,9 +177,15 @@ struct EliasFanoCompressedList { ...@@ -166,9 +177,15 @@ struct EliasFanoCompressedList {
/* static */ if (skipQuantum != 0) { /* static */ if (skipQuantum != 0) {
// Workaround to avoid 'division by zero' compile-time error. // Workaround to avoid 'division by zero' compile-time error.
constexpr size_t q = skipQuantum ?: 1; constexpr size_t q = skipQuantum ?: 1;
/* static */ if (kVersion > 0) {
CHECK_LT(size, std::numeric_limits<SkipValueType>::max());
} else {
CHECK_LT(upperSizeBits, std::numeric_limits<SkipValueType>::max()); CHECK_LT(upperSizeBits, std::numeric_limits<SkipValueType>::max());
}
// 8 * upperSize is used here instead of upperSizeBits, as that is // 8 * upperSize is used here instead of upperSizeBits, as that is
// more serialization-friendly way. // more serialization-friendly way (upperSizeBits isn't known outside of
// this function, unlike upperSize; thus numSkipPointers could easily be
// deduced from upperSize).
numSkipPointers = (8 * upperSize - size) / q; numSkipPointers = (8 * upperSize - size) / q;
skipPointers = static_cast<SkipValueType*>( skipPointers = static_cast<SkipValueType*>(
numSkipPointers == 0 numSkipPointers == 0
...@@ -178,10 +195,15 @@ struct EliasFanoCompressedList { ...@@ -178,10 +195,15 @@ struct EliasFanoCompressedList {
for (size_t i = 0, pos = 0; i < size; ++i) { for (size_t i = 0, pos = 0; i < size; ++i) {
const ValueType upperBits = list[i] >> numLowerBits; const ValueType upperBits = list[i] >> numLowerBits;
for (; (pos + 1) * q <= upperBits; ++pos) { for (; (pos + 1) * q <= upperBits; ++pos) {
/* static */ if (kVersion > 0) {
// Since version 1, just the number of preceding 1-bits is stored.
skipPointers[pos] = i;
} else {
skipPointers[pos] = i + (pos + 1) * q; skipPointers[pos] = i + (pos + 1) * q;
} }
} }
} }
}
// *** Forward pointers. // *** Forward pointers.
// Store (1-indexed) position of every forwardQuantum-th // Store (1-indexed) position of every forwardQuantum-th
...@@ -201,9 +223,14 @@ struct EliasFanoCompressedList { ...@@ -201,9 +223,14 @@ struct EliasFanoCompressedList {
for (size_t i = q - 1, pos = 0; i < size; i += q, ++pos) { for (size_t i = q - 1, pos = 0; i < size; i += q, ++pos) {
const ValueType upperBits = list[i] >> numLowerBits; const ValueType upperBits = list[i] >> numLowerBits;
/* static */ if (kVersion > 0) {
// Since version 1, just the number of preceding 0-bits is stored.
forwardPointers[pos] = upperBits;
} else {
forwardPointers[pos] = upperBits + i + 1; forwardPointers[pos] = upperBits + i + 1;
} }
} }
}
// *** Result. // *** Result.
result.size = size; result.size = size;
...@@ -269,13 +296,13 @@ struct Fast : public Default { ...@@ -269,13 +296,13 @@ struct Fast : public Default {
namespace detail { namespace detail {
template <class CompressedList, class Instructions> template <class Encoder, class Instructions>
class UpperBitsReader { class UpperBitsReader {
typedef typename CompressedList::SkipValueType SkipValueType; typedef typename Encoder::SkipValueType SkipValueType;
public: public:
typedef typename CompressedList::ValueType ValueType; typedef typename Encoder::ValueType ValueType;
explicit UpperBitsReader(const CompressedList& list) explicit UpperBitsReader(const EliasFanoCompressedList& list)
: forwardPointers_(list.forwardPointers.data()), : forwardPointers_(list.forwardPointers.data()),
skipPointers_(list.skipPointers.data()), skipPointers_(list.skipPointers.data()),
start_(list.upper.data()), start_(list.upper.data()),
...@@ -308,17 +335,20 @@ class UpperBitsReader { ...@@ -308,17 +335,20 @@ class UpperBitsReader {
position_ += n; // n 1-bits will be read. position_ += n; // n 1-bits will be read.
// Use forward pointer. // Use forward pointer.
if (CompressedList::forwardQuantum > 0 && if (Encoder::forwardQuantum > 0 && n > Encoder::forwardQuantum) {
n > CompressedList::forwardQuantum) {
// Workaround to avoid 'division by zero' compile-time error. // Workaround to avoid 'division by zero' compile-time error.
constexpr size_t q = CompressedList::forwardQuantum ?: 1; constexpr size_t q = Encoder::forwardQuantum ?: 1;
const size_t steps = position_ / q; const size_t steps = position_ / q;
const size_t dest = const size_t dest =
folly::loadUnaligned<SkipValueType>( folly::loadUnaligned<SkipValueType>(
forwardPointers_ + (steps - 1) * sizeof(SkipValueType)); forwardPointers_ + (steps - 1) * sizeof(SkipValueType));
/* static */ if (Encoder::version > 0) {
reposition(dest + steps * q);
} else {
reposition(dest); reposition(dest);
}
n = position_ + 1 - steps * q; // n is > 0. n = position_ + 1 - steps * q; // n is > 0.
// correct inner_ will be set at the end. // correct inner_ will be set at the end.
} }
...@@ -353,18 +383,22 @@ class UpperBitsReader { ...@@ -353,18 +383,22 @@ class UpperBitsReader {
DCHECK_GE(v, value_); DCHECK_GE(v, value_);
// Use skip pointer. // Use skip pointer.
if (CompressedList::skipQuantum > 0 && if (Encoder::skipQuantum > 0 && v >= value_ + Encoder::skipQuantum) {
v >= value_ + CompressedList::skipQuantum) {
// Workaround to avoid 'division by zero' compile-time error. // Workaround to avoid 'division by zero' compile-time error.
constexpr size_t q = CompressedList::skipQuantum ?: 1; constexpr size_t q = Encoder::skipQuantum ?: 1;
const size_t steps = v / q; const size_t steps = v / q;
const size_t dest = const size_t dest =
folly::loadUnaligned<SkipValueType>( folly::loadUnaligned<SkipValueType>(
skipPointers_ + (steps - 1) * sizeof(SkipValueType)); skipPointers_ + (steps - 1) * sizeof(SkipValueType));
/* static */ if (Encoder::version > 0) {
reposition(dest + q * steps);
position_ = dest - 1;
} else {
reposition(dest); reposition(dest);
position_ = dest - q * steps - 1; position_ = dest - q * steps - 1;
}
// Correct inner_ and value_ will be set during the next() // Correct inner_ and value_ will be set during the next()
// call at the end. // call at the end.
...@@ -423,13 +457,13 @@ class UpperBitsReader { ...@@ -423,13 +457,13 @@ class UpperBitsReader {
} // namespace detail } // namespace detail
template <class CompressedList, template <class Encoder,
class Instructions = instructions::Default> class Instructions = instructions::Default>
class EliasFanoReader : private boost::noncopyable { class EliasFanoReader : private boost::noncopyable {
public: public:
typedef typename CompressedList::ValueType ValueType; typedef typename Encoder::ValueType ValueType;
explicit EliasFanoReader(const CompressedList& list) explicit EliasFanoReader(const EliasFanoCompressedList& list)
: list_(list), : list_(list),
lowerMask_((ValueType(1) << list_.numLowerBits) - 1), lowerMask_((ValueType(1) << list_.numLowerBits) - 1),
upper_(list), upper_(list),
...@@ -515,9 +549,9 @@ class EliasFanoReader : private boost::noncopyable { ...@@ -515,9 +549,9 @@ class EliasFanoReader : private boost::noncopyable {
return lowerMask_ & (ptrv >> (pos % 8)); return lowerMask_ & (ptrv >> (pos % 8));
} }
const CompressedList list_; const EliasFanoCompressedList list_;
const ValueType lowerMask_; const ValueType lowerMask_;
detail::UpperBitsReader<CompressedList, Instructions> upper_; detail::UpperBitsReader<Encoder, Instructions> upper_;
size_t progress_; size_t progress_;
ValueType value_; ValueType value_;
ValueType lastValue_; ValueType lastValue_;
......
...@@ -68,27 +68,6 @@ std::vector<uint32_t> loadList(const std::string& filename) { ...@@ -68,27 +68,6 @@ std::vector<uint32_t> loadList(const std::string& filename) {
return result; return result;
} }
template <class Reader, class List>
void testEmpty() {
List list;
const typename List::ValueType* const data = nullptr;
List::encode(data, 0, list);
{
Reader reader(list);
EXPECT_FALSE(reader.next());
EXPECT_EQ(reader.size(), 0);
}
{
Reader reader(list);
EXPECT_FALSE(reader.skip(1));
EXPECT_FALSE(reader.skip(10));
}
{
Reader reader(list);
EXPECT_FALSE(reader.skipTo(1));
}
}
template <class Reader, class List> template <class Reader, class List>
void testNext(const std::vector<uint32_t>& data, const List& list) { void testNext(const std::vector<uint32_t>& data, const List& list) {
Reader reader(list); Reader reader(list);
...@@ -169,10 +148,31 @@ void testSkipTo(const std::vector<uint32_t>& data, const List& list) { ...@@ -169,10 +148,31 @@ void testSkipTo(const std::vector<uint32_t>& data, const List& list) {
} }
} }
template <class Reader, class List> template <class Reader, class Encoder>
void testEmpty() {
typename Encoder::CompressedList list;
const typename Encoder::ValueType* const data = nullptr;
Encoder::encode(data, 0, list);
{
Reader reader(list);
EXPECT_FALSE(reader.next());
EXPECT_EQ(reader.size(), 0);
}
{
Reader reader(list);
EXPECT_FALSE(reader.skip(1));
EXPECT_FALSE(reader.skip(10));
}
{
Reader reader(list);
EXPECT_FALSE(reader.skipTo(1));
}
}
template <class Reader, class Encoder>
void testAll(const std::vector<uint32_t>& data) { void testAll(const std::vector<uint32_t>& data) {
List list; typename Encoder::CompressedList list;
List::encode(data.begin(), data.end(), list); Encoder::encode(data.begin(), data.end(), list);
testNext<Reader>(data, list); testNext<Reader>(data, list);
testSkip<Reader>(data, list); testSkip<Reader>(data, list);
testSkipTo<Reader>(data, list); testSkipTo<Reader>(data, list);
......
...@@ -14,54 +14,74 @@ ...@@ -14,54 +14,74 @@
* limitations under the License. * limitations under the License.
*/ */
#include "folly/experimental/test/CodingTestUtils.h"
#include "folly/experimental/EliasFanoCoding.h"
#include "folly/Benchmark.h" #include "folly/Benchmark.h"
#include "folly/experimental/EliasFanoCoding.h"
#include "folly/experimental/test/CodingTestUtils.h"
using namespace folly::compression; using namespace folly::compression;
template <class List> template <size_t kVersion>
void testAll() { struct TestType {
typedef EliasFanoReader<List> Reader; static constexpr size_t Version = kVersion;
testAll<Reader, List>(generateRandomList(100 * 1000, 10 * 1000 * 1000)); };
testAll<Reader, List>(generateSeqList(1, 100000, 100));
} template <class T>
class EliasFanoCodingTest : public ::testing::Test {
public:
void doTestEmpty() {
typedef EliasFanoEncoder<uint32_t, size_t, 0, 0, T::Version> Encoder;
typedef EliasFanoReader<Encoder> Reader;
testEmpty<Reader, Encoder>();
}
template <size_t kSkipQuantum, size_t kForwardQuantum>
void doTestAll() {
typedef EliasFanoEncoder<
uint32_t, uint32_t, kSkipQuantum, kForwardQuantum, T::Version> Encoder;
typedef EliasFanoReader<Encoder> Reader;
testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000));
testAll<Reader, Encoder>(generateSeqList(1, 100000, 100));
}
};
TEST(EliasFanoCompressedList, Empty) { typedef ::testing::Types<TestType<0>, TestType<1>> TestTypes;
typedef EliasFanoCompressedList<uint32_t> List; TYPED_TEST_CASE(EliasFanoCodingTest, TestTypes);
typedef EliasFanoReader<List> Reader;
testEmpty<Reader, List>(); TYPED_TEST(EliasFanoCodingTest, Empty) {
TestFixture::doTestEmpty();
} }
TEST(EliasFanoCompressedList, Simple) { TYPED_TEST(EliasFanoCodingTest, Simple) {
testAll<EliasFanoCompressedList<uint32_t> >(); TestFixture::template doTestAll<0, 0>();
} }
TEST(EliasFanoCompressedList, SkipPointers) { TYPED_TEST(EliasFanoCodingTest, SkipPointers) {
testAll<EliasFanoCompressedList<uint32_t, uint32_t, 128, 0> >(); TestFixture::template doTestAll<128, 0>();
} }
TEST(EliasFanoCompressedList, ForwardPointers) { TYPED_TEST(EliasFanoCodingTest, ForwardPointers) {
testAll<EliasFanoCompressedList<uint32_t, uint32_t, 0, 128> >(); TestFixture::template doTestAll<0, 128>();
} }
TEST(EliasFanoCompressedList, SkipForwardPointers) { TYPED_TEST(EliasFanoCodingTest, SkipForwardPointers) {
testAll<EliasFanoCompressedList<uint32_t, uint32_t, 128, 128> >(); TestFixture::template doTestAll<128, 128>();
} }
namespace bm { namespace bm {
constexpr size_t k1M = 1000000; constexpr size_t k1M = 1000000;
typedef EliasFanoCompressedList<uint32_t, uint32_t, 128, 128> List; constexpr size_t kVersion = 1;
typedef EliasFanoReader<List> Reader;
typedef EliasFanoEncoder<uint32_t, uint32_t, 128, 128, kVersion> Encoder;
typedef EliasFanoReader<Encoder> Reader;
std::vector<uint32_t> data; std::vector<uint32_t> data;
List list; typename Encoder::CompressedList list;
void init() { void init() {
data = generateRandomList(100 * 1000, 10 * 1000 * 1000); data = generateRandomList(100 * 1000, 10 * 1000 * 1000);
//data = loadList("/home/philipp/pl_test_dump.txt"); //data = loadList("/home/philipp/pl_test_dump.txt");
List::encode(data.data(), data.size(), bm::list); Encoder::encode(data.data(), data.size(), bm::list);
} }
void free() { void free() {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment