Commit df7b6652 authored by Giuseppe Ottaviano's avatar Giuseppe Ottaviano Committed by Facebook Github Bot

Improve EliasFanoCodingTest coverage of 64-bits values and corner cases

Summary: `EliasFanoCoding` is used with 64-bits values in some logic, but we have no tests for that. Also, we weren't testing the larges admissible value. Finally, clarify in the comments what the various size types are for.

Reviewed By: philippv

Differential Revision: D18146549

fbshipit-source-id: e97ec9e1344db6e56a591a9b4c77c5929e10b22b
parent acf4b746
...@@ -90,7 +90,8 @@ typedef EliasFanoCompressedListBase<uint8_t*> MutableEliasFanoCompressedList; ...@@ -90,7 +90,8 @@ typedef EliasFanoCompressedListBase<uint8_t*> MutableEliasFanoCompressedList;
template < template <
class Value, class Value,
class SkipValue = size_t, // SkipValue must be wide enough to be able to represent the list length.
class SkipValue = uint64_t,
size_t kSkipQuantum = 0, // 0 = disabled size_t kSkipQuantum = 0, // 0 = disabled
size_t kForwardQuantum = 0, // 0 = disabled size_t kForwardQuantum = 0, // 0 = disabled
bool kUpperFirst = false> bool kUpperFirst = false>
...@@ -577,14 +578,17 @@ class UpperBitsReader : ForwardPointers<Encoder::forwardQuantum>, ...@@ -577,14 +578,17 @@ class UpperBitsReader : ForwardPointers<Encoder::forwardQuantum>,
} // namespace detail } // namespace detail
// If kUnchecked = true the caller must guarantee that all the // If kUnchecked = true the caller must guarantee that all the operations return
// operations return valid elements, i.e., they would never return // valid elements, i.e., they would never return false if checked.
// false if checked. //
// If the list length is known to be representable with a type narrower than the
// SkipValueType used in the format, the reader footprint can be reduced by
// passing the type as SizeType.
template < template <
class Encoder, class Encoder,
class Instructions = instructions::Default, class Instructions = instructions::Default,
bool kUnchecked = false, bool kUnchecked = false,
class SizeType = size_t> class SizeType = typename Encoder::SkipValueType>
class EliasFanoReader { class EliasFanoReader {
public: public:
typedef Encoder EncoderType; typedef Encoder EncoderType;
......
...@@ -69,11 +69,11 @@ namespace bm { ...@@ -69,11 +69,11 @@ namespace bm {
typedef BitVectorEncoder<uint32_t, uint32_t, 128, 128> Encoder; typedef BitVectorEncoder<uint32_t, uint32_t, 128, 128> Encoder;
std::vector<uint32_t> data; std::vector<uint64_t> data;
std::vector<size_t> order; std::vector<size_t> order;
std::vector<uint32_t> encodeSmallData; std::vector<uint64_t> encodeSmallData;
std::vector<uint32_t> encodeLargeData; std::vector<uint64_t> encodeLargeData;
typename Encoder::MutableCompressedList list; typename Encoder::MutableCompressedList list;
......
...@@ -36,43 +36,43 @@ namespace folly { ...@@ -36,43 +36,43 @@ namespace folly {
namespace compression { namespace compression {
template <class URNG> template <class URNG>
std::vector<uint32_t> generateRandomList(size_t n, uint32_t maxId, URNG&& g) { std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId, URNG&& g) {
CHECK_LT(n, 2 * maxId); CHECK_LT(n, 2 * maxId);
std::uniform_int_distribution<> uid(1, maxId); std::uniform_int_distribution<uint64_t> uid(1, maxId);
std::unordered_set<uint32_t> dataset; std::unordered_set<uint64_t> dataset;
while (dataset.size() < n) { while (dataset.size() < n) {
uint32_t value = uid(g); uint64_t value = uid(g);
if (dataset.count(value) == 0) { if (dataset.count(value) == 0) {
dataset.insert(value); dataset.insert(value);
} }
} }
std::vector<uint32_t> ids(dataset.begin(), dataset.end()); std::vector<uint64_t> ids(dataset.begin(), dataset.end());
std::sort(ids.begin(), ids.end()); std::sort(ids.begin(), ids.end());
return ids; return ids;
} }
inline std::vector<uint32_t> generateRandomList(size_t n, uint32_t maxId) { inline std::vector<uint64_t> generateRandomList(size_t n, uint64_t maxId) {
std::mt19937 gen; std::mt19937 gen;
return generateRandomList(n, maxId, gen); return generateRandomList(n, maxId, gen);
} }
inline std::vector<uint32_t> inline std::vector<uint64_t>
generateSeqList(uint32_t minId, uint32_t maxId, uint32_t step = 1) { generateSeqList(uint64_t minId, uint64_t maxId, uint64_t step = 1) {
CHECK_LE(minId, maxId); CHECK_LE(minId, maxId);
CHECK_GT(step, 0); CHECK_GT(step, 0);
std::vector<uint32_t> ids; std::vector<uint64_t> ids;
ids.reserve((maxId - minId) / step + 1); ids.reserve((maxId - minId) / step + 1);
for (uint32_t i = minId; i <= maxId; i += step) { for (uint64_t i = minId; i <= maxId; i += step) {
ids.push_back(i); ids.push_back(i);
} }
return ids; return ids;
} }
inline std::vector<uint32_t> loadList(const std::string& filename) { inline std::vector<uint64_t> loadList(const std::string& filename) {
std::ifstream fin(filename); std::ifstream fin(filename);
std::vector<uint32_t> result; std::vector<uint64_t> result;
uint32_t id; uint64_t id;
while (fin >> id) { while (fin >> id) {
result.push_back(id); result.push_back(id);
} }
...@@ -116,7 +116,7 @@ auto maybeTestPrevious(const Vector& data, Reader& reader, Index i) ...@@ -116,7 +116,7 @@ auto maybeTestPrevious(const Vector& data, Reader& reader, Index i)
} }
template <class Reader, class List> template <class Reader, class List>
void testNext(const std::vector<uint32_t>& data, const List& list) { void testNext(const std::vector<uint64_t>& data, const List& list) {
Reader reader(list); Reader reader(list);
EXPECT_FALSE(reader.valid()); EXPECT_FALSE(reader.valid());
...@@ -135,7 +135,7 @@ void testNext(const std::vector<uint32_t>& data, const List& list) { ...@@ -135,7 +135,7 @@ void testNext(const std::vector<uint32_t>& data, const List& list) {
template <class Reader, class List> template <class Reader, class List>
void testSkip( void testSkip(
const std::vector<uint32_t>& data, const std::vector<uint64_t>& data,
const List& list, const List& list,
size_t skipStep) { size_t skipStep) {
CHECK_GT(skipStep, 0); CHECK_GT(skipStep, 0);
...@@ -156,7 +156,7 @@ void testSkip( ...@@ -156,7 +156,7 @@ void testSkip(
} }
template <class Reader, class List> template <class Reader, class List>
void testSkip(const std::vector<uint32_t>& data, const List& list) { void testSkip(const std::vector<uint64_t>& data, const List& list) {
for (size_t skipStep = 1; skipStep < 25; ++skipStep) { for (size_t skipStep = 1; skipStep < 25; ++skipStep) {
testSkip<Reader, List>(data, list, skipStep); testSkip<Reader, List>(data, list, skipStep);
} }
...@@ -167,14 +167,16 @@ void testSkip(const std::vector<uint32_t>& data, const List& list) { ...@@ -167,14 +167,16 @@ void testSkip(const std::vector<uint32_t>& data, const List& list) {
template <class Reader, class List> template <class Reader, class List>
void testSkipTo( void testSkipTo(
const std::vector<uint32_t>& data, const std::vector<uint64_t>& data,
const List& list, const List& list,
size_t skipToStep) { size_t skipToStep) {
using ValueType = typename Reader::ValueType;
CHECK_GT(skipToStep, 0); CHECK_GT(skipToStep, 0);
Reader reader(list); Reader reader(list);
const uint32_t delta = std::max<uint32_t>(1, data.back() / skipToStep); const uint64_t delta = std::max<uint64_t>(1, data.back() / skipToStep);
uint32_t value = delta; ValueType value = delta;
auto it = data.begin(); auto it = data.begin();
while (true) { while (true) {
it = std::lower_bound(it, data.end(), value); it = std::lower_bound(it, data.end(), value);
...@@ -186,9 +188,13 @@ void testSkipTo( ...@@ -186,9 +188,13 @@ void testSkipTo(
EXPECT_TRUE(reader.valid()); EXPECT_TRUE(reader.valid());
EXPECT_EQ(reader.value(), *it); EXPECT_EQ(reader.value(), *it);
EXPECT_EQ(reader.position(), std::distance(data.begin(), it)); EXPECT_EQ(reader.position(), std::distance(data.begin(), it));
value = reader.value() + delta;
maybeTestPreviousValue(data, reader, std::distance(data.begin(), it)); maybeTestPreviousValue(data, reader, std::distance(data.begin(), it));
maybeTestPrevious(data, reader, std::distance(data.begin(), it)); maybeTestPrevious(data, reader, std::distance(data.begin(), it));
value = reader.value() + delta;
if (value < reader.value()) { // Wrapped around.
value = reader.value() + 1; // Can't be kInvalidValue.
}
} }
EXPECT_FALSE(reader.valid()); EXPECT_FALSE(reader.valid());
EXPECT_EQ(reader.position(), reader.size()); EXPECT_EQ(reader.position(), reader.size());
...@@ -196,7 +202,7 @@ void testSkipTo( ...@@ -196,7 +202,7 @@ void testSkipTo(
} }
template <class Reader, class List> template <class Reader, class List>
void testSkipTo(const std::vector<uint32_t>& data, const List& list) { void testSkipTo(const std::vector<uint64_t>& data, const List& list) {
for (size_t steps = 10; steps < 100; steps += 10) { for (size_t steps = 10; steps < 100; steps += 10) {
testSkipTo<Reader, List>(data, list, steps); testSkipTo<Reader, List>(data, list, steps);
} }
...@@ -231,7 +237,7 @@ void testSkipTo(const std::vector<uint32_t>& data, const List& list) { ...@@ -231,7 +237,7 @@ void testSkipTo(const std::vector<uint32_t>& data, const List& list) {
} }
template <class Reader, class List> template <class Reader, class List>
void testJump(const std::vector<uint32_t>& data, const List& list) { void testJump(const std::vector<uint64_t>& data, const List& list) {
std::mt19937 gen; std::mt19937 gen;
std::vector<size_t> is(data.size()); std::vector<size_t> is(data.size());
for (size_t i = 0; i < data.size(); ++i) { for (size_t i = 0; i < data.size(); ++i) {
...@@ -256,16 +262,17 @@ void testJump(const std::vector<uint32_t>& data, const List& list) { ...@@ -256,16 +262,17 @@ void testJump(const std::vector<uint32_t>& data, const List& list) {
} }
template <class Reader, class List> template <class Reader, class List>
void testJumpTo(const std::vector<uint32_t>& data, const List& list) { void testJumpTo(const std::vector<uint64_t>& data, const List& list) {
CHECK(!data.empty()); using ValueType = typename Reader::ValueType;
CHECK(!data.empty());
Reader reader(list); Reader reader(list);
std::mt19937 gen; std::mt19937 gen;
std::uniform_int_distribution<> values(0, data.back()); std::uniform_int_distribution<ValueType> values(0, data.back());
const size_t iters = Reader::EncoderType::skipQuantum == 0 ? 100 : 10000; const size_t iters = Reader::EncoderType::skipQuantum == 0 ? 100 : 10000;
for (size_t i = 0; i < iters; ++i) { for (size_t i = 0; i < iters; ++i) {
const uint32_t value = values(gen); const uint64_t value = values(gen);
auto it = std::lower_bound(data.begin(), data.end(), value); auto it = std::lower_bound(data.begin(), data.end(), value);
CHECK(it != data.end()); CHECK(it != data.end());
EXPECT_TRUE(reader.jumpTo(value)); EXPECT_TRUE(reader.jumpTo(value));
...@@ -310,7 +317,7 @@ void testEmpty() { ...@@ -310,7 +317,7 @@ void testEmpty() {
} }
template <class Reader, class Encoder> template <class Reader, class Encoder>
void testAll(const std::vector<uint32_t>& data) { void testAll(const std::vector<uint64_t>& data) {
auto list = Encoder::encode(data.begin(), data.end()); auto list = Encoder::encode(data.begin(), data.end());
testNext<Reader>(data, list); testNext<Reader>(data, list);
testSkip<Reader>(data, list); testSkip<Reader>(data, list);
...@@ -321,7 +328,7 @@ void testAll(const std::vector<uint32_t>& data) { ...@@ -321,7 +328,7 @@ void testAll(const std::vector<uint32_t>& data) {
} }
template <class Reader, class List> template <class Reader, class List>
void bmNext(const List& list, const std::vector<uint32_t>& data, size_t iters) { void bmNext(const List& list, const std::vector<uint64_t>& data, size_t iters) {
if (data.empty()) { if (data.empty()) {
return; return;
} }
...@@ -339,7 +346,7 @@ void bmNext(const List& list, const std::vector<uint32_t>& data, size_t iters) { ...@@ -339,7 +346,7 @@ void bmNext(const List& list, const std::vector<uint32_t>& data, size_t iters) {
template <class Reader, class List> template <class Reader, class List>
void bmSkip( void bmSkip(
const List& list, const List& list,
const std::vector<uint32_t>& /* data */, const std::vector<uint64_t>& /* data */,
size_t logAvgSkip, size_t logAvgSkip,
size_t iters) { size_t iters) {
size_t avg = (size_t(1) << logAvgSkip); size_t avg = (size_t(1) << logAvgSkip);
...@@ -360,7 +367,7 @@ void bmSkip( ...@@ -360,7 +367,7 @@ void bmSkip(
template <class Reader, class List> template <class Reader, class List>
void bmSkipTo( void bmSkipTo(
const List& list, const List& list,
const std::vector<uint32_t>& data, const std::vector<uint64_t>& data,
size_t logAvgSkip, size_t logAvgSkip,
size_t iters) { size_t iters) {
size_t avg = (size_t(1) << logAvgSkip); size_t avg = (size_t(1) << logAvgSkip);
...@@ -384,7 +391,7 @@ void bmSkipTo( ...@@ -384,7 +391,7 @@ void bmSkipTo(
template <class Reader, class List> template <class Reader, class List>
void bmJump( void bmJump(
const List& list, const List& list,
const std::vector<uint32_t>& data, const std::vector<uint64_t>& data,
const std::vector<size_t>& order, const std::vector<size_t>& order,
size_t iters) { size_t iters) {
CHECK(!data.empty()); CHECK(!data.empty());
...@@ -403,7 +410,7 @@ void bmJump( ...@@ -403,7 +410,7 @@ void bmJump(
template <class Reader, class List> template <class Reader, class List>
void bmJumpTo( void bmJumpTo(
const List& list, const List& list,
const std::vector<uint32_t>& data, const std::vector<uint64_t>& data,
const std::vector<size_t>& order, const std::vector<size_t>& order,
size_t iters) { size_t iters) {
CHECK(!data.empty()); CHECK(!data.empty());
......
...@@ -93,21 +93,25 @@ class EliasFanoCodingTest : public ::testing::Test { ...@@ -93,21 +93,25 @@ class EliasFanoCodingTest : public ::testing::Test {
template < template <
size_t kSkipQuantum, size_t kSkipQuantum,
size_t kForwardQuantum, size_t kForwardQuantum,
class SizeType, class ValueType,
bool kUpperFirst> bool kUpperFirst>
void doTestAll() { void doTestAll() {
// SkipValueType and SizeType could both be narrower than ValueType, but
// testing all combinations would be slow, so assume they are all the same.
typedef EliasFanoEncoderV2< typedef EliasFanoEncoderV2<
uint32_t, ValueType,
uint32_t, ValueType,
kSkipQuantum, kSkipQuantum,
kForwardQuantum, kForwardQuantum,
kUpperFirst> kUpperFirst>
Encoder; Encoder;
using Reader = using Reader = EliasFanoReader<Encoder, instructions::Default, false>;
EliasFanoReader<Encoder, instructions::Default, false, SizeType>;
testAll<Reader, Encoder>({0}); testAll<Reader, Encoder>({0});
testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000)); testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000));
testAll<Reader, Encoder>(generateSeqList(1, 100000, 100)); testAll<Reader, Encoder>(generateSeqList(1, 100000, 100));
// max() cannot be read, as it is assumed an invalid value.
// TODO(ott): It should be possible to lift this constraint.
testAll<Reader, Encoder>({0, 1, std::numeric_limits<uint32_t>::max() - 1});
} }
}; };
...@@ -118,29 +122,29 @@ TEST_F(EliasFanoCodingTest, Empty) { ...@@ -118,29 +122,29 @@ TEST_F(EliasFanoCodingTest, Empty) {
TEST_F(EliasFanoCodingTest, Simple) { TEST_F(EliasFanoCodingTest, Simple) {
doTestAll<0, 0, uint32_t, false>(); doTestAll<0, 0, uint32_t, false>();
doTestAll<0, 0, uint32_t, true>(); doTestAll<0, 0, uint32_t, true>();
doTestAll<0, 0, size_t, false>(); doTestAll<0, 0, uint64_t, false>();
doTestAll<0, 0, size_t, true>(); doTestAll<0, 0, uint64_t, true>();
} }
TEST_F(EliasFanoCodingTest, SkipPointers) { TEST_F(EliasFanoCodingTest, SkipPointers) {
doTestAll<128, 0, uint32_t, false>(); doTestAll<128, 0, uint32_t, false>();
doTestAll<128, 0, uint32_t, true>(); doTestAll<128, 0, uint32_t, true>();
doTestAll<128, 0, size_t, false>(); doTestAll<128, 0, uint64_t, false>();
doTestAll<128, 0, size_t, true>(); doTestAll<128, 0, uint64_t, true>();
} }
TEST_F(EliasFanoCodingTest, ForwardPointers) { TEST_F(EliasFanoCodingTest, ForwardPointers) {
doTestAll<0, 128, uint32_t, false>(); doTestAll<0, 128, uint32_t, false>();
doTestAll<0, 128, uint32_t, true>(); doTestAll<0, 128, uint32_t, true>();
doTestAll<0, 128, size_t, false>(); doTestAll<0, 128, uint64_t, false>();
doTestAll<0, 128, size_t, true>(); doTestAll<0, 128, uint64_t, true>();
} }
TEST_F(EliasFanoCodingTest, SkipForwardPointers) { TEST_F(EliasFanoCodingTest, SkipForwardPointers) {
doTestAll<128, 128, uint32_t, false>(); doTestAll<128, 128, uint32_t, false>();
doTestAll<128, 128, uint32_t, true>(); doTestAll<128, 128, uint32_t, true>();
doTestAll<128, 128, size_t, false>(); doTestAll<128, 128, uint64_t, false>();
doTestAll<128, 128, size_t, true>(); doTestAll<128, 128, uint64_t, true>();
} }
TEST_F(EliasFanoCodingTest, BugLargeGapInUpperBits) { // t16274876 TEST_F(EliasFanoCodingTest, BugLargeGapInUpperBits) { // t16274876
...@@ -171,11 +175,11 @@ namespace bm { ...@@ -171,11 +175,11 @@ namespace bm {
typedef EliasFanoEncoderV2<uint32_t, uint32_t, 128, 128> Encoder; typedef EliasFanoEncoderV2<uint32_t, uint32_t, 128, 128> Encoder;
std::vector<uint32_t> data; std::vector<uint64_t> data;
std::vector<size_t> order; std::vector<size_t> order;
std::vector<uint32_t> encodeSmallData; std::vector<uint64_t> encodeSmallData;
std::vector<uint32_t> encodeLargeData; std::vector<uint64_t> encodeLargeData;
std::vector<std::pair<size_t, size_t>> numLowerBitsInput; std::vector<std::pair<size_t, size_t>> numLowerBitsInput;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment