Commit 2c464254 authored by Giuseppe Ottaviano's avatar Giuseppe Ottaviano Committed by Facebook Github Bot

Document and test some EliasFanoCoding corner cases

Summary:
EliasFanoCoding supports sequences with duplicate values, but there is no test coverage and the behavior of some corner cases (`skipTo` and `jumpTo` from the middle of a run) is not documented.

This diff adds coverage and documentation.

Reviewed By: yfeldblum

Differential Revision: D18496744

fbshipit-source-id: 00e112a902a8b11ac9601520b858172a1d2c0f57
parent da41ae50
......@@ -289,7 +289,9 @@ class BitVectorReader : detail::ForwardPointers<Encoder::forwardQuantum>,
}
bool skip(SizeType n) {
CHECK_GT(n, 0);
if (n == 0) {
return valid();
}
if (!kUnchecked && position() + n >= size_) {
return setDone();
......
......@@ -641,8 +641,14 @@ class EliasFanoReader {
return true;
}
/**
* Advances by n elements. n = 0 is allowed and has no effect. Returns false
* if the end of the list is reached.
*/
bool skip(SizeType n) {
CHECK_GT(n, 0);
if (n == 0) {
return valid();
}
if (kUnchecked || LIKELY(position() + n < size_)) {
if (LIKELY(n < kLinearScanThreshold)) {
......@@ -660,8 +666,12 @@ class EliasFanoReader {
return setDone();
}
/**
* Skips to the first element >= value whose position is greater or equal to
* the current position. Requires that value >= value() (or that the reader is
* at position -1). Returns false if no such element exists.
*/
bool skipTo(ValueType value) {
// Also works when value_ == kInvalidValue.
if (value != kInvalidValue) {
DCHECK_GE(value + 1, value_ + 1);
}
......@@ -711,14 +721,22 @@ class EliasFanoReader {
__builtin_prefetch(addr + kCacheLineSize);
}
/**
* Jumps to the element at position n. The reader can be in any state. Returns
* false if n >= size().
*/
bool jump(SizeType n) {
if (LIKELY(n < size_)) { // Also checks that n != -1.
if (LIKELY(n < size_)) {
value_ = readLowerPart(n) | (upper_.jump(n + 1) << numLowerBits_);
return true;
}
return setDone();
}
/**
* Jumps to the first element >= value. The reader can be in any
* state. Returns false if no such element exists.
*/
bool jumpTo(ValueType value) {
if (!kUnchecked && value > lastValue_) {
return setDone();
......
......@@ -36,7 +36,11 @@ namespace folly {
namespace compression {
template <class URNG>
std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId, URNG&& g) {
std::vector<uint64_t> generateRandomList(
size_t n,
uint64_t maxId,
URNG&& g,
bool withDuplicates = false) {
CHECK_LT(n, 2 * maxId);
std::uniform_int_distribution<uint64_t> uid(1, maxId);
std::unordered_set<uint64_t> dataset;
......@@ -48,13 +52,21 @@ std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId, URNG&& g) {
}
std::vector<uint64_t> ids(dataset.begin(), dataset.end());
if (withDuplicates && n > 0) {
// Ensure 20% of the list has at least 1 duplicate, 10% has at least 2, and
// 5% is a run of the same value.
std::copy(ids.begin() + ids.size() * 8 / 10, ids.end(), ids.begin());
std::copy(ids.begin() + ids.size() * 9 / 10, ids.end(), ids.begin());
std::fill(ids.begin(), ids.begin() + ids.size() / 20, ids[0]);
}
std::sort(ids.begin(), ids.end());
return ids;
}
inline std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId) {
inline std::vector<uint64_t>
generateRandomList(size_t n, uint64_t maxId, bool withDuplicates = false) {
std::mt19937 gen;
return generateRandomList(n, maxId, gen);
return generateRandomList(n, maxId, gen, withDuplicates);
}
inline std::vector<uint64_t>
......@@ -142,10 +154,14 @@ void testSkip(
Reader reader(list);
for (size_t i = skipStep - 1; i < data.size(); i += skipStep) {
EXPECT_TRUE(reader.skip(skipStep));
EXPECT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), data[i]);
EXPECT_EQ(reader.position(), i);
// Also test that skip(0) stays in place.
for (auto step : {skipStep, size_t(0)}) {
EXPECT_TRUE(reader.skip(step));
EXPECT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), data[i]);
EXPECT_EQ(reader.position(), i);
}
maybeTestPreviousValue(data, reader, i);
maybeTestPrevious(data, reader, i);
}
......@@ -176,24 +192,42 @@ void testSkipTo(
Reader reader(list);
const uint64_t delta = std::max<uint64_t>(1, data.back() / skipToStep);
ValueType value = delta;
ValueType target = delta;
auto it = data.begin();
while (true) {
it = std::lower_bound(it, data.end(), value);
it = std::lower_bound(it, data.end(), target);
if (it == data.end()) {
EXPECT_FALSE(reader.skipTo(value));
EXPECT_FALSE(reader.skipTo(target));
break;
}
EXPECT_TRUE(reader.skipTo(value));
EXPECT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), *it);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it));
maybeTestPreviousValue(data, reader, std::distance(data.begin(), it));
maybeTestPrevious(data, reader, std::distance(data.begin(), it));
value = reader.value() + delta;
if (value < reader.value()) { // Wrapped around.
value = reader.value() + 1; // Can't be kInvalidValue.
EXPECT_TRUE(reader.skipTo(target));
// Test the whole group of equal values.
for (auto it2 = it; it2 != data.end() && *it2 == *it;
++it2, reader.next()) {
ASSERT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), *it2);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it2));
// The reader should stay in place even if we're in the middle of a group.
EXPECT_TRUE(reader.skipTo(*it2));
EXPECT_EQ(reader.value(), *it2);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it2));
maybeTestPreviousValue(data, reader, std::distance(data.begin(), it2));
maybeTestPrevious(data, reader, std::distance(data.begin(), it2));
}
// Reader is now past the group.
if (!reader.valid()) {
break;
}
target += delta;
if (target < reader.value()) {
// Value following the group is already greater than target, or delta is
// so large that target wrapped around.
target = reader.value();
}
}
EXPECT_FALSE(reader.valid());
......@@ -250,9 +284,12 @@ void testJump(const std::vector<uint64_t>& data, const List& list) {
Reader reader(list);
for (auto i : is) {
EXPECT_TRUE(reader.jump(i));
EXPECT_EQ(reader.value(), data[i]);
EXPECT_EQ(reader.position(), i);
// Also test idempotency.
for (size_t round = 0; round < 2; ++round) {
EXPECT_TRUE(reader.jump(i));
EXPECT_EQ(reader.value(), data[i]);
EXPECT_EQ(reader.position(), i);
}
maybeTestPreviousValue(data, reader, i);
maybeTestPrevious(data, reader, i);
}
......@@ -269,25 +306,34 @@ void testJumpTo(const std::vector<uint64_t>& data, const List& list) {
Reader reader(list);
std::mt19937 gen;
std::uniform_int_distribution<ValueType> values(0, data.back());
std::uniform_int_distribution<ValueType> targets(0, data.back());
const size_t iters = Reader::EncoderType::skipQuantum == 0 ? 100 : 10000;
for (size_t i = 0; i < iters; ++i) {
const uint64_t value = values(gen);
auto it = std::lower_bound(data.begin(), data.end(), value);
uint64_t target;
// Force boundary targets interleaved with random targets.
if (i == 10) {
target = data.back();
} else if (i == 20) {
target = 0;
} else {
target = targets(gen);
}
auto it = std::lower_bound(data.begin(), data.end(), target);
CHECK(it != data.end());
EXPECT_TRUE(reader.jumpTo(value));
EXPECT_EQ(reader.value(), *it);
EXPECT_TRUE(reader.jumpTo(target));
// Test the whole group of equal values.
for (auto it2 = it; it2 != data.end() && *it2 == *it;
++it2, reader.next()) {
EXPECT_EQ(reader.value(), *it2);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it2));
}
// Calling jumpTo() on the current value should reposition on the beginning
// of the group.
EXPECT_TRUE(reader.jumpTo(*it));
EXPECT_EQ(reader.position(), std::distance(data.begin(), it));
}
EXPECT_TRUE(reader.jumpTo(0));
EXPECT_EQ(reader.value(), data[0]);
EXPECT_EQ(reader.position(), 0);
EXPECT_TRUE(reader.jumpTo(data.back()));
EXPECT_EQ(reader.value(), data.back());
EXPECT_EQ(reader.position(), reader.size() - 1);
EXPECT_FALSE(reader.jumpTo(data.back() + 1));
EXPECT_FALSE(reader.valid());
EXPECT_EQ(reader.position(), reader.size());
......
......@@ -108,6 +108,8 @@ class EliasFanoCodingTest : public ::testing::Test {
using Reader = EliasFanoReader<Encoder, instructions::Default, false>;
testAll<Reader, Encoder>({0});
testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000));
testAll<Reader, Encoder>(generateRandomList(
100 * 1000, 10 * 1000 * 1000, /* withDuplicates */ true));
testAll<Reader, Encoder>(generateSeqList(1, 100000, 100));
// max() cannot be read, as it is assumed an invalid value.
// TODO(ott): It should be possible to lift this constraint.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment