Commit d92be25e authored by Giuseppe Ottaviano's avatar Giuseppe Ottaviano Committed by Facebook Github Bot

Fix EliasFanoReader when both kUpperFirst and kChecked are true

Summary: The computation of the upper bits size is incorrect when both `kUpperFirst` and `kChecked` are true. In addition, when `kUpperFirst` is true, we might actually need 8 additional bytes in the buffer, for when `numLowerBits = 0`, so clarify the comment.

Reviewed By: philippv

Differential Revision: D18145144

fbshipit-source-id: 1372d39023e2d5d1c5ccbfb8e6c7af9a5a930f1d
parent 6f97a09f
...@@ -58,6 +58,7 @@ struct EliasFanoCompressedListBase { ...@@ -58,6 +58,7 @@ struct EliasFanoCompressedListBase {
const EliasFanoCompressedListBase<OtherPointer>& other) const EliasFanoCompressedListBase<OtherPointer>& other)
: size(other.size), : size(other.size),
numLowerBits(other.numLowerBits), numLowerBits(other.numLowerBits),
upperSizeBytes(other.upperSizeBytes),
data(other.data), data(other.data),
skipPointers(reinterpret_cast<Pointer>(other.skipPointers)), skipPointers(reinterpret_cast<Pointer>(other.skipPointers)),
forwardPointers(reinterpret_cast<Pointer>(other.forwardPointers)), forwardPointers(reinterpret_cast<Pointer>(other.forwardPointers)),
...@@ -69,15 +70,13 @@ struct EliasFanoCompressedListBase { ...@@ -69,15 +70,13 @@ struct EliasFanoCompressedListBase {
return ::free(data.data()); return ::free(data.data());
} }
size_t upperSize() const {
return size_t(data.end() - upper);
}
size_t size = 0; size_t size = 0;
uint8_t numLowerBits = 0; uint8_t numLowerBits = 0;
size_t upperSizeBytes = 0;
// WARNING: EliasFanoCompressedList has no ownership of data. The 7 // WARNING: EliasFanoCompressedList has no ownership of data. The 7 bytes
// bytes following the last byte should be readable. // following the last byte should be readable if kUpperFirst = false, 8 bytes
// otherwise.
folly::Range<Pointer> data; folly::Range<Pointer> data;
Pointer skipPointers = nullptr; Pointer skipPointers = nullptr;
...@@ -297,6 +296,7 @@ struct EliasFanoEncoderV2< ...@@ -297,6 +296,7 @@ struct EliasFanoEncoderV2<
EliasFanoCompressedListBase<typename Range::iterator> result; EliasFanoCompressedListBase<typename Range::iterator> result;
result.size = size; result.size = size;
result.numLowerBits = numLowerBits; result.numLowerBits = numLowerBits;
result.upperSizeBytes = upper;
result.data = buf.subpiece(0, bytes()); result.data = buf.subpiece(0, bytes());
auto advance = [&](size_t n) { auto advance = [&](size_t n) {
...@@ -321,12 +321,12 @@ struct EliasFanoEncoderV2< ...@@ -321,12 +321,12 @@ struct EliasFanoEncoderV2<
MutableCompressedList allocList() const { MutableCompressedList allocList() const {
uint8_t* buf = nullptr; uint8_t* buf = nullptr;
// WARNING: Current read/write logic assumes that the 7 bytes // WARNING: Current read/write logic assumes that the 7 bytes
// following the last byte of lower and upper sequences are // following the upper bytes and the 8 bytes following the lower bytes
// readable (stored value doesn't matter and won't be changed), so // sequences are readable (stored value doesn't matter and won't be
// we allocate additional 7 bytes, but do not include them in size // changed), so we allocate additional 8 bytes, but do not include them in
// of returned value. // size of returned value.
if (size > 0) { if (size > 0) {
buf = static_cast<uint8_t*>(malloc(bytes() + 7)); buf = static_cast<uint8_t*>(malloc(bytes() + 8));
} }
folly::MutableByteRange bufRange(buf, bytes()); folly::MutableByteRange bufRange(buf, bytes());
return openList(bufRange); return openList(bufRange);
...@@ -604,8 +604,8 @@ class EliasFanoReader { ...@@ -604,8 +604,8 @@ class EliasFanoReader {
lastValue_ = 0; lastValue_ = 0;
return; return;
} }
ValueType lastUpperValue = ValueType(8 * list.upperSize() - size_); ValueType lastUpperValue = ValueType(8 * list.upperSizeBytes - size_);
auto it = list.upper + list.upperSize() - 1; auto it = list.upper + list.upperSizeBytes - 1;
DCHECK_NE(*it, 0); DCHECK_NE(*it, 0);
lastUpperValue -= 8 - folly::findLastSet(*it); lastUpperValue -= 8 - folly::findLastSet(*it);
lastValue_ = readLowerPart(size_ - 1) | (lastUpperValue << numLowerBits_); lastValue_ = readLowerPart(size_ - 1) | (lastUpperValue << numLowerBits_);
......
...@@ -90,13 +90,18 @@ class EliasFanoCodingTest : public ::testing::Test { ...@@ -90,13 +90,18 @@ class EliasFanoCodingTest : public ::testing::Test {
testEmpty<Reader, Encoder>(); testEmpty<Reader, Encoder>();
} }
template <size_t kSkipQuantum, size_t kForwardQuantum, class SizeType> template <
size_t kSkipQuantum,
size_t kForwardQuantum,
class SizeType,
bool kUpperFirst>
void doTestAll() { void doTestAll() {
typedef EliasFanoEncoderV2< typedef EliasFanoEncoderV2<
uint32_t, uint32_t,
uint32_t, uint32_t,
kSkipQuantum, kSkipQuantum,
kForwardQuantum> kForwardQuantum,
kUpperFirst>
Encoder; Encoder;
using Reader = using Reader =
EliasFanoReader<Encoder, instructions::Default, false, SizeType>; EliasFanoReader<Encoder, instructions::Default, false, SizeType>;
...@@ -111,23 +116,31 @@ TEST_F(EliasFanoCodingTest, Empty) { ...@@ -111,23 +116,31 @@ TEST_F(EliasFanoCodingTest, Empty) {
} }
TEST_F(EliasFanoCodingTest, Simple) { TEST_F(EliasFanoCodingTest, Simple) {
doTestAll<0, 0, uint32_t>(); doTestAll<0, 0, uint32_t, false>();
doTestAll<0, 0, size_t>(); doTestAll<0, 0, uint32_t, true>();
doTestAll<0, 0, size_t, false>();
doTestAll<0, 0, size_t, true>();
} }
TEST_F(EliasFanoCodingTest, SkipPointers) { TEST_F(EliasFanoCodingTest, SkipPointers) {
doTestAll<128, 0, uint32_t>(); doTestAll<128, 0, uint32_t, false>();
doTestAll<128, 0, size_t>(); doTestAll<128, 0, uint32_t, true>();
doTestAll<128, 0, size_t, false>();
doTestAll<128, 0, size_t, true>();
} }
TEST_F(EliasFanoCodingTest, ForwardPointers) { TEST_F(EliasFanoCodingTest, ForwardPointers) {
doTestAll<0, 128, uint32_t>(); doTestAll<0, 128, uint32_t, false>();
doTestAll<0, 128, size_t>(); doTestAll<0, 128, uint32_t, true>();
doTestAll<0, 128, size_t, false>();
doTestAll<0, 128, size_t, true>();
} }
TEST_F(EliasFanoCodingTest, SkipForwardPointers) { TEST_F(EliasFanoCodingTest, SkipForwardPointers) {
doTestAll<128, 128, uint32_t>(); doTestAll<128, 128, uint32_t, false>();
doTestAll<128, 128, size_t>(); doTestAll<128, 128, uint32_t, true>();
doTestAll<128, 128, size_t, false>();
doTestAll<128, 128, size_t, true>();
} }
TEST_F(EliasFanoCodingTest, BugLargeGapInUpperBits) { // t16274876 TEST_F(EliasFanoCodingTest, BugLargeGapInUpperBits) { // t16274876
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment