Commit 2c464254 authored by Giuseppe Ottaviano's avatar Giuseppe Ottaviano Committed by Facebook Github Bot

Document and test some EliasFanoCoding corner cases

Summary:
EliasFanoCoding supports sequences with duplicate values, but there is no test coverage and the behavior of some corner cases (`skipTo` and `jumpTo` from the middle of a run) is not documented.

This diff adds coverage and documentation.

Reviewed By: yfeldblum

Differential Revision: D18496744

fbshipit-source-id: 00e112a902a8b11ac9601520b858172a1d2c0f57
parent da41ae50
...@@ -289,7 +289,9 @@ class BitVectorReader : detail::ForwardPointers<Encoder::forwardQuantum>, ...@@ -289,7 +289,9 @@ class BitVectorReader : detail::ForwardPointers<Encoder::forwardQuantum>,
} }
bool skip(SizeType n) { bool skip(SizeType n) {
CHECK_GT(n, 0); if (n == 0) {
return valid();
}
if (!kUnchecked && position() + n >= size_) { if (!kUnchecked && position() + n >= size_) {
return setDone(); return setDone();
......
...@@ -641,8 +641,14 @@ class EliasFanoReader { ...@@ -641,8 +641,14 @@ class EliasFanoReader {
return true; return true;
} }
/**
* Advances by n elements. n = 0 is allowed and has no effect. Returns false
* if the end of the list is reached.
*/
bool skip(SizeType n) { bool skip(SizeType n) {
CHECK_GT(n, 0); if (n == 0) {
return valid();
}
if (kUnchecked || LIKELY(position() + n < size_)) { if (kUnchecked || LIKELY(position() + n < size_)) {
if (LIKELY(n < kLinearScanThreshold)) { if (LIKELY(n < kLinearScanThreshold)) {
...@@ -660,8 +666,12 @@ class EliasFanoReader { ...@@ -660,8 +666,12 @@ class EliasFanoReader {
return setDone(); return setDone();
} }
/**
* Skips to the first element >= value whose position is greater or equal to
* the current position. Requires that value >= value() (or that the reader is
* at position -1). Returns false if no such element exists.
*/
bool skipTo(ValueType value) { bool skipTo(ValueType value) {
// Also works when value_ == kInvalidValue.
if (value != kInvalidValue) { if (value != kInvalidValue) {
DCHECK_GE(value + 1, value_ + 1); DCHECK_GE(value + 1, value_ + 1);
} }
...@@ -711,14 +721,22 @@ class EliasFanoReader { ...@@ -711,14 +721,22 @@ class EliasFanoReader {
__builtin_prefetch(addr + kCacheLineSize); __builtin_prefetch(addr + kCacheLineSize);
} }
/**
* Jumps to the element at position n. The reader can be in any state. Returns
* false if n >= size().
*/
bool jump(SizeType n) { bool jump(SizeType n) {
if (LIKELY(n < size_)) { // Also checks that n != -1. if (LIKELY(n < size_)) {
value_ = readLowerPart(n) | (upper_.jump(n + 1) << numLowerBits_); value_ = readLowerPart(n) | (upper_.jump(n + 1) << numLowerBits_);
return true; return true;
} }
return setDone(); return setDone();
} }
/**
* Jumps to the first element >= value. The reader can be in any
* state. Returns false if no such element exists.
*/
bool jumpTo(ValueType value) { bool jumpTo(ValueType value) {
if (!kUnchecked && value > lastValue_) { if (!kUnchecked && value > lastValue_) {
return setDone(); return setDone();
......
...@@ -36,7 +36,11 @@ namespace folly { ...@@ -36,7 +36,11 @@ namespace folly {
namespace compression { namespace compression {
template <class URNG> template <class URNG>
std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId, URNG&& g) { std::vector<uint64_t> generateRandomList(
size_t n,
uint64_t maxId,
URNG&& g,
bool withDuplicates = false) {
CHECK_LT(n, 2 * maxId); CHECK_LT(n, 2 * maxId);
std::uniform_int_distribution<uint64_t> uid(1, maxId); std::uniform_int_distribution<uint64_t> uid(1, maxId);
std::unordered_set<uint64_t> dataset; std::unordered_set<uint64_t> dataset;
...@@ -48,13 +52,21 @@ std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId, URNG&& g) { ...@@ -48,13 +52,21 @@ std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId, URNG&& g) {
} }
std::vector<uint64_t> ids(dataset.begin(), dataset.end()); std::vector<uint64_t> ids(dataset.begin(), dataset.end());
if (withDuplicates && n > 0) {
// Ensure 20% of the list has at least 1 duplicate, 10% has at least 2, and
// 5% is a run of the same value.
std::copy(ids.begin() + ids.size() * 8 / 10, ids.end(), ids.begin());
std::copy(ids.begin() + ids.size() * 9 / 10, ids.end(), ids.begin());
std::fill(ids.begin(), ids.begin() + ids.size() / 20, ids[0]);
}
std::sort(ids.begin(), ids.end()); std::sort(ids.begin(), ids.end());
return ids; return ids;
} }
inline std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId) { inline std::vector<uint64_t>
generateRandomList(size_t n, uint64_t maxId, bool withDuplicates = false) {
std::mt19937 gen; std::mt19937 gen;
return generateRandomList(n, maxId, gen); return generateRandomList(n, maxId, gen, withDuplicates);
} }
inline std::vector<uint64_t> inline std::vector<uint64_t>
...@@ -142,10 +154,14 @@ void testSkip( ...@@ -142,10 +154,14 @@ void testSkip(
Reader reader(list); Reader reader(list);
for (size_t i = skipStep - 1; i < data.size(); i += skipStep) { for (size_t i = skipStep - 1; i < data.size(); i += skipStep) {
EXPECT_TRUE(reader.skip(skipStep)); // Also test that skip(0) stays in place.
EXPECT_TRUE(reader.valid()); for (auto step : {skipStep, size_t(0)}) {
EXPECT_EQ(reader.value(), data[i]); EXPECT_TRUE(reader.skip(step));
EXPECT_EQ(reader.position(), i); EXPECT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), data[i]);
EXPECT_EQ(reader.position(), i);
}
maybeTestPreviousValue(data, reader, i); maybeTestPreviousValue(data, reader, i);
maybeTestPrevious(data, reader, i); maybeTestPrevious(data, reader, i);
} }
...@@ -176,24 +192,42 @@ void testSkipTo( ...@@ -176,24 +192,42 @@ void testSkipTo(
Reader reader(list); Reader reader(list);
const uint64_t delta = std::max<uint64_t>(1, data.back() / skipToStep); const uint64_t delta = std::max<uint64_t>(1, data.back() / skipToStep);
ValueType value = delta; ValueType target = delta;
auto it = data.begin(); auto it = data.begin();
while (true) { while (true) {
it = std::lower_bound(it, data.end(), value); it = std::lower_bound(it, data.end(), target);
if (it == data.end()) { if (it == data.end()) {
EXPECT_FALSE(reader.skipTo(value)); EXPECT_FALSE(reader.skipTo(target));
break; break;
} }
EXPECT_TRUE(reader.skipTo(value));
EXPECT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), *it);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it));
maybeTestPreviousValue(data, reader, std::distance(data.begin(), it));
maybeTestPrevious(data, reader, std::distance(data.begin(), it));
value = reader.value() + delta; EXPECT_TRUE(reader.skipTo(target));
if (value < reader.value()) { // Wrapped around. // Test the whole group of equal values.
value = reader.value() + 1; // Can't be kInvalidValue. for (auto it2 = it; it2 != data.end() && *it2 == *it;
++it2, reader.next()) {
ASSERT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), *it2);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it2));
// The reader should stay in place even if we're in the middle of a group.
EXPECT_TRUE(reader.skipTo(*it2));
EXPECT_EQ(reader.value(), *it2);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it2));
maybeTestPreviousValue(data, reader, std::distance(data.begin(), it2));
maybeTestPrevious(data, reader, std::distance(data.begin(), it2));
}
// Reader is now past the group.
if (!reader.valid()) {
break;
}
target += delta;
if (target < reader.value()) {
// Value following the group is already greater than target, or delta is
// so large that target wrapped around.
target = reader.value();
} }
} }
EXPECT_FALSE(reader.valid()); EXPECT_FALSE(reader.valid());
...@@ -250,9 +284,12 @@ void testJump(const std::vector<uint64_t>& data, const List& list) { ...@@ -250,9 +284,12 @@ void testJump(const std::vector<uint64_t>& data, const List& list) {
Reader reader(list); Reader reader(list);
for (auto i : is) { for (auto i : is) {
EXPECT_TRUE(reader.jump(i)); // Also test idempotency.
EXPECT_EQ(reader.value(), data[i]); for (size_t round = 0; round < 2; ++round) {
EXPECT_EQ(reader.position(), i); EXPECT_TRUE(reader.jump(i));
EXPECT_EQ(reader.value(), data[i]);
EXPECT_EQ(reader.position(), i);
}
maybeTestPreviousValue(data, reader, i); maybeTestPreviousValue(data, reader, i);
maybeTestPrevious(data, reader, i); maybeTestPrevious(data, reader, i);
} }
...@@ -269,25 +306,34 @@ void testJumpTo(const std::vector<uint64_t>& data, const List& list) { ...@@ -269,25 +306,34 @@ void testJumpTo(const std::vector<uint64_t>& data, const List& list) {
Reader reader(list); Reader reader(list);
std::mt19937 gen; std::mt19937 gen;
std::uniform_int_distribution<ValueType> values(0, data.back()); std::uniform_int_distribution<ValueType> targets(0, data.back());
const size_t iters = Reader::EncoderType::skipQuantum == 0 ? 100 : 10000; const size_t iters = Reader::EncoderType::skipQuantum == 0 ? 100 : 10000;
for (size_t i = 0; i < iters; ++i) { for (size_t i = 0; i < iters; ++i) {
const uint64_t value = values(gen); uint64_t target;
auto it = std::lower_bound(data.begin(), data.end(), value); // Force boundary targets interleaved with random targets.
if (i == 10) {
target = data.back();
} else if (i == 20) {
target = 0;
} else {
target = targets(gen);
}
auto it = std::lower_bound(data.begin(), data.end(), target);
CHECK(it != data.end()); CHECK(it != data.end());
EXPECT_TRUE(reader.jumpTo(value)); EXPECT_TRUE(reader.jumpTo(target));
EXPECT_EQ(reader.value(), *it); // Test the whole group of equal values.
for (auto it2 = it; it2 != data.end() && *it2 == *it;
++it2, reader.next()) {
EXPECT_EQ(reader.value(), *it2);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it2));
}
// Calling jumpTo() on the current value should reposition on the beginning
// of the group.
EXPECT_TRUE(reader.jumpTo(*it));
EXPECT_EQ(reader.position(), std::distance(data.begin(), it)); EXPECT_EQ(reader.position(), std::distance(data.begin(), it));
} }
EXPECT_TRUE(reader.jumpTo(0));
EXPECT_EQ(reader.value(), data[0]);
EXPECT_EQ(reader.position(), 0);
EXPECT_TRUE(reader.jumpTo(data.back()));
EXPECT_EQ(reader.value(), data.back());
EXPECT_EQ(reader.position(), reader.size() - 1);
EXPECT_FALSE(reader.jumpTo(data.back() + 1)); EXPECT_FALSE(reader.jumpTo(data.back() + 1));
EXPECT_FALSE(reader.valid()); EXPECT_FALSE(reader.valid());
EXPECT_EQ(reader.position(), reader.size()); EXPECT_EQ(reader.position(), reader.size());
......
...@@ -108,6 +108,8 @@ class EliasFanoCodingTest : public ::testing::Test { ...@@ -108,6 +108,8 @@ class EliasFanoCodingTest : public ::testing::Test {
using Reader = EliasFanoReader<Encoder, instructions::Default, false>; using Reader = EliasFanoReader<Encoder, instructions::Default, false>;
testAll<Reader, Encoder>({0}); testAll<Reader, Encoder>({0});
testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000)); testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000));
testAll<Reader, Encoder>(generateRandomList(
100 * 1000, 10 * 1000 * 1000, /* withDuplicates */ true));
testAll<Reader, Encoder>(generateSeqList(1, 100000, 100)); testAll<Reader, Encoder>(generateSeqList(1, 100000, 100));
// max() cannot be read, as it is assumed an invalid value. // max() cannot be read, as it is assumed an invalid value.
// TODO(ott): It should be possible to lift this constraint. // TODO(ott): It should be possible to lift this constraint.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment