Commit b797de38 authored by Matt Ma's avatar Matt Ma Committed by Facebook Github Bot

Move QuotientMultiSet to folly/experimental/

Summary:
QuotientMultiSet is a space-efficient static data structure to store a non-decreasing sequence of b-bit integers.

If the integers are uniformly distributed lookup is O(1)-time and performs a single random memory lookup with high probability.

Reviewed By: ot

Differential Revision: D17506766

fbshipit-source-id: b3e7a22dd193672fadb07d4cccb8b01bedae7cf9
parent ccd541a4
...@@ -578,6 +578,7 @@ if (BUILD_TESTS) ...@@ -578,6 +578,7 @@ if (BUILD_TESTS)
TEST lock_free_ring_buffer_test SOURCES LockFreeRingBufferTest.cpp TEST lock_free_ring_buffer_test SOURCES LockFreeRingBufferTest.cpp
#TEST nested_command_line_app_test SOURCES NestedCommandLineAppTest.cpp #TEST nested_command_line_app_test SOURCES NestedCommandLineAppTest.cpp
#TEST program_options_test SOURCES ProgramOptionsTest.cpp #TEST program_options_test SOURCES ProgramOptionsTest.cpp
TEST quotient_multiset_test SOURCES QuotientMultiSetTest.cpp
# Depends on liburcu # Depends on liburcu
#TEST read_mostly_shared_ptr_test SOURCES ReadMostlySharedPtrTest.cpp #TEST read_mostly_shared_ptr_test SOURCES ReadMostlySharedPtrTest.cpp
#TEST ref_count_test SOURCES RefCountTest.cpp #TEST ref_count_test SOURCES RefCountTest.cpp
......
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/experimental/QuotientMultiSet.h>
#include <folly/Format.h>
#include <folly/Portability.h>
#include <folly/experimental/Bits.h>
#include <folly/experimental/Select64.h>
#include <folly/lang/Bits.h>
#include <folly/lang/SafeAssert.h>
#include <glog/logging.h>
#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
namespace folly {
namespace qms_detail {
/*
* Reference: Faster Remainder by Direct Computation: Applications to Compilers
* and Software Libraries, Software: Practice and Experience 49 (6), 2019.
*/
FOLLY_ALWAYS_INLINE UInt64InverseType getInverse(uint64_t divisor) {
UInt64InverseType fraction = UInt64InverseType(-1);
fraction /= divisor;
fraction += 1;
return fraction;
}
FOLLY_ALWAYS_INLINE uint64_t
mul128(UInt64InverseType lowbits, uint64_t divisor) {
UInt64InverseType bottomHalf =
(lowbits & UINT64_C(0xFFFFFFFFFFFFFFFF)) * divisor;
bottomHalf >>= 64;
UInt64InverseType topHalf = (lowbits >> 64) * divisor;
UInt64InverseType bothHalves = bottomHalf + topHalf;
bothHalves >>= 64;
return static_cast<uint64_t>(bothHalves);
}
FOLLY_ALWAYS_INLINE std::pair<uint64_t, uint64_t> getQuotientAndRemainder(
uint64_t dividend,
uint64_t divisor,
UInt64InverseType inverse) {
if (FOLLY_UNLIKELY(divisor == 1)) {
return {dividend, 0};
}
auto quotient = mul128(inverse, dividend);
auto remainder = dividend - quotient * divisor;
DCHECK_LT(remainder, divisor);
return {quotient, remainder};
}
// Max value for given bits.
FOLLY_ALWAYS_INLINE uint64_t maxValue(uint32_t nbits) {
return nbits == 64 ? std::numeric_limits<uint64_t>::max()
: (uint64_t(1) << nbits) - 1;
}
} // namespace qms_detail
template <class Instructions>
struct QuotientMultiSet<Instructions>::Metadata {
// Total number of blocks.
uint64_t numBlocks;
uint64_t numKeys;
uint64_t divisor;
uint8_t keyBits;
uint8_t remainderBits;
std::string debugString() const {
return sformat(
"Number of blocks: {}\n"
"Number of elements: {}\n"
"Divisor: {}\n"
"Key bits: {}\n"
"Remainder bits: {}",
numBlocks,
numKeys,
divisor,
keyBits,
remainderBits);
}
} FOLLY_PACK_ATTR;
template <class Instructions>
struct QuotientMultiSet<Instructions>::Block {
static const Block* get(const char* data) {
return reinterpret_cast<const Block*>(data);
}
static BlockPtr make(size_t remainderBits) {
auto ptr = reinterpret_cast<Block*>(calloc(blockSize(remainderBits), 1));
return {ptr, free};
}
uint64_t payload;
uint64_t occupieds;
uint64_t offset;
uint64_t runends;
char remainders[0];
static uint64_t blockSize(size_t remainderBits) {
return sizeof(Block) + remainderBits * 8;
}
FOLLY_ALWAYS_INLINE bool isOccupied(size_t offsetInBlock) const {
return ((occupieds >> offsetInBlock) & uint64_t(1)) != 0;
}
FOLLY_ALWAYS_INLINE bool isRunend(size_t offsetInBlock) const {
return ((runends >> offsetInBlock) & uint64_t(1)) != 0;
}
FOLLY_ALWAYS_INLINE uint64_t getRemainder(
size_t offsetInBlock,
size_t remainderBits,
size_t remainderMask) const {
DCHECK_LE(remainderBits, 56);
const size_t bitPos = offsetInBlock * remainderBits;
const uint64_t remainderWord =
loadUnaligned<uint64_t>(remainders + (bitPos / 8));
return (remainderWord >> (bitPos % 8)) & remainderMask;
}
void setOccupied(size_t offsetInBlock) {
occupieds |= uint64_t(1) << offsetInBlock;
}
void setRunend(size_t offsetInBlock) {
runends |= uint64_t(1) << offsetInBlock;
}
void
setRemainder(size_t offsetInBlock, size_t remainderBits, uint64_t remainder) {
DCHECK_LT(offsetInBlock, kBlockSize);
if (FOLLY_UNLIKELY(remainderBits == 0)) {
return;
}
Bits<uint64_t>::set(
reinterpret_cast<uint64_t*>(remainders),
offsetInBlock * remainderBits,
remainderBits,
remainder);
}
} FOLLY_PACK_ATTR;
template <class Instructions>
QuotientMultiSet<Instructions>::QuotientMultiSet(StringPiece data) {
static_assert(
kIsLittleEndian, "QuotientMultiSet requires little endianness.");
StringPiece sp = data;
CHECK_GE(sp.size(), sizeof(Metadata));
sp.advance(sp.size() - sizeof(Metadata));
metadata_ = reinterpret_cast<const Metadata*>(sp.data());
VLOG(2) << "Metadata: " << metadata_->debugString();
numBlocks_ = metadata_->numBlocks;
numSlots_ = numBlocks_ * kBlockSize;
divisor_ = metadata_->divisor;
fraction_ = qms_detail::getInverse(divisor_);
keyBits_ = metadata_->keyBits;
maxKey_ = qms_detail::maxValue(keyBits_);
remainderBits_ = metadata_->remainderBits;
remainderMask_ = qms_detail::maxValue(remainderBits_);
blockSize_ = Block::blockSize(remainderBits_);
CHECK_EQ(data.size(), numBlocks_ * blockSize_ + sizeof(Metadata));
data_ = data.data();
}
template <class Instructions>
auto QuotientMultiSet<Instructions>::equalRange(uint64_t key) const
-> SlotRange {
if (key > maxKey_) {
return {0, 0};
}
const auto qr = qms_detail::getQuotientAndRemainder(key, divisor_, fraction_);
const auto& quotient = qr.first;
const auto& remainder = qr.second;
const size_t blockIndex = quotient / kBlockSize;
const size_t offsetInBlock = quotient % kBlockSize;
if (FOLLY_UNLIKELY(blockIndex >= numBlocks_)) {
return {0, 0};
}
const auto* occBlock = getBlock(blockIndex);
__builtin_prefetch(reinterpret_cast<const char*>(&occBlock->occupieds) + 64);
const auto firstRunend = occBlock->offset;
if (!occBlock->isOccupied(offsetInBlock)) {
// Return a position that depends on the contents of the block so
// we can create a dependency in benchmarks.
return {firstRunend, firstRunend};
}
// Look for the right runend for the given key.
const uint64_t occupiedRank = Instructions::popcount(
Instructions::bzhi(occBlock->occupieds, offsetInBlock));
auto runend = findRunend(occupiedRank, firstRunend);
auto& slot = runend.first;
auto& block = runend.second;
// Iterates over the run backwards to find the slots whose remainder
// matches the key.
SlotRange range = {slot + 1, slot + 1};
while (true) {
uint64_t slotRemainder =
block->getRemainder(slot % kBlockSize, remainderBits_, remainderMask_);
if (slotRemainder > remainder) {
range.begin = slot;
range.end = slot;
} else if (slotRemainder == remainder) {
range.begin = slot;
} else {
break;
}
if (FOLLY_UNLIKELY(slot % kBlockSize == 0)) {
// Reached block start and the run starts from a prev block.
size_t slotBlockIndex = slot / kBlockSize;
if (slotBlockIndex > blockIndex) {
block = getBlock(slotBlockIndex - 1);
} else {
break;
}
}
--slot;
// Encounters the previous run.
if (block->isRunend(slot % kBlockSize)) {
break;
}
}
return range;
}
template <class Instructions>
auto QuotientMultiSet<Instructions>::findRunend(
uint64_t occupiedRank,
uint64_t firstRunend) const -> std::pair<uint64_t, const Block*> {
// Look for the right runend.
size_t slotBlockIndex = firstRunend / kBlockSize;
auto block = getBlock(slotBlockIndex);
uint64_t runendWord =
block->runends & (uint64_t(-1) << (firstRunend % kBlockSize));
while (true) {
DCHECK_LE(slotBlockIndex, numBlocks_);
const size_t numRuns = Instructions::popcount(runendWord);
if (FOLLY_LIKELY(numRuns > occupiedRank)) {
break;
}
occupiedRank -= numRuns;
++slotBlockIndex;
block = getBlock(slotBlockIndex);
runendWord = block->runends;
}
return {slotBlockIndex * kBlockSize +
select64<Instructions>(runendWord, occupiedRank),
block};
}
template <class Instructions>
uint64_t QuotientMultiSet<Instructions>::getBlockPayload(
uint64_t blockIndex) const {
DCHECK_LT(blockIndex, numBlocks_);
return getBlock(blockIndex)->payload;
}
template <class Instructions>
QuotientMultiSet<Instructions>::Iterator::Iterator(
QuotientMultiSet<Instructions>* qms)
: qms_(qms),
key_(0),
occBlockIndex_(-1),
occOffsetInBlock_(0),
occWord_(0),
occBlock_(nullptr),
pos_(-1) {}
template <class Instructions>
bool QuotientMultiSet<Instructions>::Iterator::next() {
if (pos_ == size_t(-1) ||
qms_->getBlock(pos_ / kBlockSize)->isRunend(pos_ % kBlockSize)) {
// Move to start of next run.
if (!nextOccupied()) {
return setEnd();
}
// Next run either starts at pos + 1 or the start of block
// specified by occupied slot.
pos_ = std::max<uint64_t>(pos_ + 1, occBlockIndex_ * kBlockSize);
} else {
// Move to next slot since a run must be contiguous.
pos_++;
}
const Block* block = qms_->getBlock(pos_ / kBlockSize);
uint64_t quotient =
(occBlockIndex_ * kBlockSize + occOffsetInBlock_) * qms_->divisor_;
key_ = quotient +
block->getRemainder(
pos_ % kBlockSize, qms_->remainderBits_, qms_->remainderMask_);
DCHECK_LT(pos_, qms_->numBlocks_ * kBlockSize);
return true;
}
template <class Instructions>
bool QuotientMultiSet<Instructions>::Iterator::skipTo(uint64_t key) {
if (key > qms_->maxKey_) {
return false;
}
const auto qr =
qms_detail::getQuotientAndRemainder(key, qms_->divisor_, qms_->fraction_);
const auto& quotient = qr.first;
occBlockIndex_ = quotient / kBlockSize;
occOffsetInBlock_ = quotient % kBlockSize;
if (FOLLY_UNLIKELY(occBlockIndex_ >= qms_->numBlocks_)) {
return setEnd();
}
occBlock_ = qms_->getBlock(occBlockIndex_);
occWord_ = occBlock_->occupieds & (uint64_t(-1) << occOffsetInBlock_);
if (!nextOccupied()) {
return setEnd();
}
// Search for the next runend.
uint64_t occupiedRank = Instructions::popcount(
Instructions::bzhi(occBlock_->occupieds, occOffsetInBlock_));
auto runend = qms_->findRunend(occupiedRank, occBlock_->offset);
auto& slot = runend.first;
auto& block = runend.second;
uint64_t slotBlockIndex = slot / kBlockSize;
uint64_t slotOffsetInBlock = slot % kBlockSize;
pos_ = slot;
uint64_t nextQuotient =
(occBlockIndex_ * kBlockSize + occOffsetInBlock_) * qms_->divisor_;
uint64_t nextRemainder = block->getRemainder(
slotOffsetInBlock, qms_->remainderBits_, qms_->remainderMask_);
if (nextQuotient + nextRemainder < key) {
// Lower bound element is at the start of next run.
return next();
}
// Iterate over the run backwards to find the first key that is larger than
// or equal to the given key.
while (true) {
uint64_t slotRemainder = block->getRemainder(
slotOffsetInBlock, qms_->remainderBits_, qms_->remainderMask_);
if (nextQuotient + slotRemainder < key) {
break;
}
pos_ = slot;
nextRemainder = slotRemainder;
if (FOLLY_UNLIKELY(slotOffsetInBlock == 0)) {
// Reached block start and the run starts from a prev block.
if (slotBlockIndex > occBlockIndex_) {
--slotBlockIndex;
block = qms_->getBlock(slotBlockIndex);
slotOffsetInBlock = kBlockSize;
} else {
break;
}
}
--slot;
--slotOffsetInBlock;
// Encounters the previous run.
if (block->isRunend(slotOffsetInBlock)) {
break;
}
}
key_ = nextQuotient + nextRemainder;
return true;
}
template <class Instructions>
bool QuotientMultiSet<Instructions>::Iterator::nextOccupied() {
while (FOLLY_UNLIKELY(occWord_ == 0)) {
if (FOLLY_UNLIKELY(++occBlockIndex_ >= qms_->numBlocks_)) {
return false;
}
occBlock_ = qms_->getBlock(occBlockIndex_);
occWord_ = occBlock_->occupieds;
}
occOffsetInBlock_ = Instructions::ctz(occWord_);
occWord_ = Instructions::blsr(occWord_);
return true;
}
} // namespace folly
#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/experimental/QuotientMultiSet.h>
#include <math.h>
#include <folly/Math.h>
#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
namespace folly {
QuotientMultiSetBuilder::QuotientMultiSetBuilder(
size_t keyBits,
size_t expectedElements,
double loadFactor)
: keyBits_(keyBits), maxKey_(qms_detail::maxValue(keyBits_)) {
expectedElements = std::max<size_t>(expectedElements, 1);
uint64_t numSlots = ceil(expectedElements / loadFactor);
// Make sure 1:1 mapping between key space and <divisor, remainder> pairs.
divisor_ = divCeil(maxKey_, numSlots);
remainderBits_ = findLastSet(divisor_ - 1);
// We only support remainders as long as 56 bits. If the set is very
// sparse, force the maximum allowed remainder size. This will waste
// up to 3 extra blocks (because of 8-bit quotients) but be correct.
if (remainderBits_ > 56) {
remainderBits_ = 56;
divisor_ = uint64_t(1) << remainderBits_;
}
blockSize_ = Block::blockSize(remainderBits_);
fraction_ = qms_detail::getInverse(divisor_);
}
QuotientMultiSetBuilder::~QuotientMultiSetBuilder() = default;
bool QuotientMultiSetBuilder::maybeAllocateBlocks(size_t limitIndex) {
bool blockAllocated = false;
for (; numBlocks_ <= limitIndex; numBlocks_++) {
auto block = Block::make(remainderBits_);
blocks_.emplace_back(std::move(block), numBlocks_);
blockAllocated = true;
}
return blockAllocated;
}
bool QuotientMultiSetBuilder::insert(uint64_t key) {
FOLLY_SAFE_CHECK(key <= maxKey_, "Invalid key");
FOLLY_SAFE_CHECK(
key >= prevKey_, "Keys need to be inserted in nondecreasing order");
const auto qr = qms_detail::getQuotientAndRemainder(key, divisor_, fraction_);
const auto& quotient = qr.first;
const auto& remainder = qr.second;
const size_t blockIndex = quotient / kBlockSize;
const size_t offsetInBlock = quotient % kBlockSize;
bool newBlockAllocated = false;
// Allocate block for the given key if necessary.
newBlockAllocated |= maybeAllocateBlocks(
std::max<uint64_t>(blockIndex, nextSlot_ / kBlockSize));
auto block = getBlock(nextSlot_ / kBlockSize).block.get();
// Start a new run.
if (prevOccupiedQuotient_ != quotient) {
closePreviousRun();
if (blockIndex > nextSlot_ / kBlockSize) {
nextSlot_ = (blockIndex * kBlockSize);
newBlockAllocated |= maybeAllocateBlocks(blockIndex);
block = getBlock(blockIndex).block.get();
}
// Update previous run info.
prevRunStart_ = nextSlot_;
prevOccupiedQuotient_ = quotient;
}
block->setRemainder(nextSlot_ % kBlockSize, remainderBits_, remainder);
// Set occupied bit for the given key.
block = getBlock(blockIndex).block.get();
block->setOccupied(offsetInBlock);
nextSlot_++;
prevKey_ = key;
numKeys_++;
return newBlockAllocated;
}
void QuotientMultiSetBuilder::setBlockPayload(uint64_t payload) {
DCHECK(!blocks_.empty());
blocks_.back().block->payload = payload;
}
void QuotientMultiSetBuilder::closePreviousRun() {
if (FOLLY_UNLIKELY(nextSlot_ == 0)) {
return;
}
// Mark runend for previous run.
const auto runEnd = nextSlot_ - 1;
auto block = getBlock(runEnd / kBlockSize).block.get();
block->setRunend(runEnd % kBlockSize);
numRuns_++;
// Set the offset of previous block if this run is the first one in that
// block.
auto prevRunOccupiedBlock =
getBlock(prevOccupiedQuotient_ / kBlockSize).block.get();
if (isPowTwo(prevRunOccupiedBlock->occupieds)) {
prevRunOccupiedBlock->offset = runEnd;
}
// Update mark all blocks before prevOccupiedQuotient_ + 1 to be ready.
size_t limitIndex = (prevOccupiedQuotient_ + 1) / kBlockSize;
for (size_t idx = readyBlocks_; idx < blocks_.size(); idx++) {
if (blocks_[idx].index < limitIndex) {
blocks_[idx].ready = true;
readyBlocks_++;
} else {
break;
}
}
}
void QuotientMultiSetBuilder::moveReadyBlocks(IOBufQueue& buff) {
while (!blocks_.empty()) {
if (!blocks_.front().ready) {
break;
}
buff.append(
IOBuf::takeOwnership(blocks_.front().block.release(), blockSize_));
blocks_.pop_front();
}
}
void QuotientMultiSetBuilder::flush(IOBufQueue& buff) {
moveReadyBlocks(buff);
readyBlocks_ = 0;
}
void QuotientMultiSetBuilder::close(IOBufQueue& buff) {
closePreviousRun();
// Mark all blocks as ready.
for (auto iter = blocks_.rbegin(); iter != blocks_.rend(); iter++) {
if (iter->ready) {
break;
}
iter->ready = true;
}
moveReadyBlocks(buff);
// Add metadata trailer. This will also allows getRemainder() to access whole
// 64-bits at any position without bounds-checking.
static_assert(sizeof(Metadata) > 7, "getRemainder() is not safe");
auto metadata = reinterpret_cast<Metadata*>(calloc(1, sizeof(Metadata)));
metadata->numBlocks = numBlocks_;
metadata->numKeys = numKeys_;
metadata->divisor = divisor_;
metadata->keyBits = keyBits_;
metadata->remainderBits = remainderBits_;
VLOG(2) << "Metadata: " << metadata->debugString();
buff.append(IOBuf::takeOwnership(metadata, sizeof(Metadata)));
}
} // namespace folly
#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <deque>
#include <utility>
#include <folly/Portability.h>
#include <folly/Range.h>
#include <folly/experimental/Instructions.h>
#include <folly/io/IOBuf.h>
#include <folly/io/IOBufQueue.h>
// A 128-bit integer type is needed for fast division.
#define FOLLY_QUOTIENT_MULTI_SET_SUPPORTED FOLLY_HAVE_INT128_T
#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
namespace folly {
namespace qms_detail {
using UInt64InverseType = __uint128_t;
} // namespace qms_detail
/**
* A space-efficient static data structure to store a non-decreasing sequence of
* b-bit integers. If the integers are uniformly distributed lookup is O(1)-time
* and performs a single random memory lookup with high probability.
*
* Space for n keys is bounded by (5 + b - log(n / loadFactor)) / loadFactor
* bits per element, which makes it particularly efficient for very dense
* sets. Note that 1 bit is taken up by the user-provided block payloads, and 1
* depends on how close the table size is to a power of 2. Experimentally,
* performance is good up to load factor 95%.
*
* Lookup returns a range of positions in the table. The intended use case is to
* store hashes, as the first layer of a multi-layer hash table. If b is sized
* to floor(log(n)) + k, the probability of a false positive (a non-empty range
* is returned for a non-existent key) is approximately 2^-k, which makes it
* competitive with a Bloom filter for low FP probabilities, with the additional
* benefit that it also returns a range of positions to restrict the search in
* subsequent layers.
*
* The data structure is inspired by the Rank-Select Quotient Filter
* introduced in
*
* Prashant Pandey, Michael A. Bender, Rob Johnson and Robert Patro,
* A General-Purpose Counting Filter: Making Every Bit Count, SIGMOD, 2017
*
* Besides being static, QuotientMultiSet differs from the data structure from
* the paper in the following ways:
*
* - The table size can be arbitrary, rather than just powers-of-2. This can
* waste up to a bit for each residual, but it prevents 2x overhead when the
* desired table size is slightly larger than a power of 2.
*
* - Within each block all the holes are moved at the end. This enables
* efficient iteration, and makes the returned positions a contiguous range
* for each block, which allows to use them to index into a secondary data
* structure. An arbitrary 64-bit payload can be attached to each block; for
* example, this can be used to store the number of elements up to that block,
* so that positions can be translated to the element rank. Alternatively, the
* payload can be used to address blocks in the secondary data structure.
*
* - Correctness does not depend on the keys being uniformly distributed.
* However, performance does, as for arbitrary keys the worst-case lookup time
* can be linear.
*
* Implemented by Matt Ma based on a prototype by Giuseppe Ottaviano and
* Sebastiano Vigna.
*
* Data layout:
* ------------------------------------------------------------------------
* | Block | Block | Block | Block | ... | Block |
* ------------------------------------------------------------------------
* / |
* ------------------------------------------------------------------------
* | Payload | Occupieds | Offset | Runends | Remainders * 64 |
* ------------------------------------------------------------------------
*
* Each block contains 64 slots. Keys mapping to the same slot are stored
* contiguously in a run. The occupieds and runends bitvectors are the
* concatenation of the corresponding words in each block.
*
* - Occupieds bit indicates whether there is a key mapping to this quotient.
*
* - Offset stores the position of the runend of the first run in this block.
*
* - Runends bit indicates whether the slot is the end of some run. 1s in
* occupieds and runends bits are in 1-1 correspondence: the i-th 1 in the
* runends vector marks the run end of the i-th 1 in the occupieds.
*/
template <class Instructions = compression::instructions::Default>
class QuotientMultiSet final {
public:
explicit QuotientMultiSet(StringPiece data);
// Each block contains 64 elements.
static constexpr size_t kBlockSize = 64;
// Position range of given key. End is not included. Range can be empty if the
// key is not found, in which case the values of begin and end are
// unspecified.
struct SlotRange {
size_t begin = 0;
size_t end = 0;
explicit operator bool() const {
DCHECK_LE(begin, end);
return begin < end;
}
};
class Iterator;
// Get the position range for the given key.
SlotRange equalRange(uint64_t key) const;
// Get payload of given block.
uint64_t getBlockPayload(uint64_t blockIndex) const;
friend class QuotientMultiSetBuilder;
private:
// Metadata to describe a quotient table.
struct Metadata;
// Block contains payload, occupieds, runends, offsets and 64 remainders.
struct Block;
using BlockPtr = std::unique_ptr<Block, decltype(free)*>;
const Block* getBlock(size_t blockIndex) const {
return Block::get(data_ + blockIndex * blockSize_);
}
FOLLY_ALWAYS_INLINE std::pair<uint64_t, const Block*> findRunend(
uint64_t occupiedRank,
uint64_t startPos) const;
const Metadata* metadata_;
const char* data_;
// Total number of blocks.
size_t numBlocks_;
size_t numSlots_;
// Number of bytes per block.
size_t blockSize_;
// Divisor for mapping from keys to slots.
uint64_t divisor_;
// fraction_ = 1 / divisor_.
qms_detail::UInt64InverseType fraction_;
// Number of key bits.
size_t keyBits_;
uint64_t maxKey_;
// Number of remainder bits.
size_t remainderBits_;
uint64_t remainderMask_;
};
template <class Instructions>
class QuotientMultiSet<Instructions>::Iterator {
public:
explicit Iterator(QuotientMultiSet<Instructions>* qms);
// Advance to the next key.
bool next();
// Skip forward to the first key >= the given key.
bool skipTo(uint64_t key);
bool done() const {
return pos_ == qms_->numSlots_;
}
// Return current key.
uint64_t key() const {
return key_;
}
// Return current position in quotient multiset.
size_t pos() const {
return pos_;
}
private:
// Position the iterator at the end and return false.
// Shortcut for use when implementing doNext, etc: return setEnd();
bool setEnd() {
pos_ = qms_->numSlots_;
return false;
}
// Move to next occupied.
bool nextOccupied();
QuotientMultiSet<Instructions>* qms_;
uint64_t key_;
// State members for the quotient occupied position.
// Block index of key_'s occupied slot.
size_t occBlockIndex_;
// Block offset of key_'s occupied slot.
uint64_t occOffsetInBlock_;
// Occupied words of the occupiedBlock_ after quotientBlockOffset_.
uint64_t occWord_;
// Block of the current occupied slot.
const Block* occBlock_;
// State member for the actual key position.
// Position of the current key_.
size_t pos_;
};
/*
* Class to build a QuotientMultiSet.
*
* The builder requires inserting elements in non-decreasing order.
* Example usage:
* QuotientMultiSetBuilder builder(...);
* while (...) {
* if (builder.insert(key)) {
* builder.setBlockPayload(payload);
* }
* if (builder.numReadyBlocks() > N) {
* buff = builder.flush();
* write(buff);
* }
* }
* buff = builder.close();
* write(buff)
*/
class QuotientMultiSetBuilder final {
public:
QuotientMultiSetBuilder(
size_t keyBits,
size_t expectedElements,
double loadFactor = kDefaultMaxLoadFactor);
~QuotientMultiSetBuilder();
using Metadata = QuotientMultiSet<>::Metadata;
using Block = QuotientMultiSet<>::Block;
// Keeps load factor <= 0.95.
constexpr static double kDefaultMaxLoadFactor = 0.95;
constexpr static size_t kBlockSize = QuotientMultiSet<>::kBlockSize;
// Returns whether the key's slot is in a newly created block.
// Only allows insert keys in nondecreasing order.
bool insert(uint64_t key);
// Set payload of the latest created block.
// Can only be called immediately after an add() that returns true.
void setBlockPayload(uint64_t payload);
// Return all ready blocks till now. The ownership of these blocks will be
// transferred to the caller.
void flush(IOBufQueue& buff);
// Return all remaining blocks since last flush call and the final quotient
// table metadata. The ownership of these blocks will be transferred to the
// caller.
void close(folly::IOBufQueue& buff);
size_t numReadyBlocks() {
return readyBlocks_;
}
private:
using BlockPtr = QuotientMultiSet<>::BlockPtr;
struct BlockWithState {
BlockWithState(BlockPtr ptr, size_t idx)
: block(std::move(ptr)), index(idx), ready(false) {}
BlockPtr block;
size_t index;
bool ready;
};
// Allocate space for blocks until limitIndex (included).
bool maybeAllocateBlocks(size_t limitIndex);
// Close the previous run.
void closePreviousRun();
// Move ready blocks to given IOBufQueue.
void moveReadyBlocks(IOBufQueue& buff);
// Get block for given block index.
BlockWithState& getBlock(uint64_t blockIndex) {
CHECK_GE(blockIndex, blocks_.front().index);
return blocks_[blockIndex - blocks_.front().index];
}
// Number of key bits.
const size_t keyBits_;
const uint64_t maxKey_;
// Total number of blocks.
size_t numBlocks_ = 0;
// Number of bytes per block.
size_t blockSize_ = 0;
// Divisor for mapping from keys to slots.
uint64_t divisor_;
// fraction_ = 1 / divisor_.
qms_detail::UInt64InverseType fraction_;
// Number of remainder bits.
uint64_t remainderBits_;
size_t numKeys_ = 0;
size_t numRuns_ = 0;
uint64_t prevKey_ = 0;
// Next slot to be used.
size_t nextSlot_ = 0;
// The actual start of previous run.
size_t prevRunStart_ = 0;
// The quotient of previous run.
size_t prevOccupiedQuotient_ = 0;
// Number of ready blocks in deque.
size_t readyBlocks_ = 0;
// Contains blocks since last flush call.
std::deque<BlockWithState> blocks_;
IOBufQueue buff_;
};
} // namespace folly
#include <folly/experimental/QuotientMultiSet-inl.h>
#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/experimental/QuotientMultiSet.h>
#include <boost/sort/spreadsort/integer_sort.hpp>
#include <folly/Benchmark.h>
#include <folly/Format.h>
#include <folly/Random.h>
#include <folly/String.h>
#include <folly/container/Enumerate.h>
#include <folly/container/F14Set.h>
#include <folly/container/Foreach.h>
#include <folly/experimental/EliasFanoCoding.h>
#include <folly/experimental/test/CodingTestUtils.h>
#include <folly/init/Init.h>
DEFINE_int64(
key_bits,
32,
"The number of key bits used in quotient multiset. Should always <= 64");
DEFINE_uint64(
num_elements,
100000000,
"The number of elements inserted into quotient multiset");
DEFINE_double(load_factor, 0.95, "Load factor of the multiset");
#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
namespace {
static const unsigned int kRunsPerIteration = 5000000;
std::mt19937 rng;
// Uniformly distributed keys.
std::vector<uint64_t> uniform;
std::string qmsData;
uint64_t maxValue(uint32_t nbits) {
return nbits == 64 ? std::numeric_limits<uint64_t>::max()
: (uint64_t(1) << nbits) - 1;
}
// Hash 64 bits into 1.
uint64_t mix64_to_1(uint64_t word) {
// Constant from MurmurHash3 final mixer.
return (word * UINT64_C(0xff51afd7ed558ccd)) >> 63;
}
void buildQuotientMultiSet(std::vector<uint64_t>& keys) {
folly::QuotientMultiSetBuilder builder(
FLAGS_key_bits, keys.size(), FLAGS_load_factor);
folly::IOBufQueue buff;
for (const auto& iter : folly::enumerate(keys)) {
if (builder.insert(*iter)) {
builder.setBlockPayload(iter.index);
}
if (builder.numReadyBlocks() >= 1) {
builder.flush(buff);
}
}
builder.close(buff);
qmsData = buff.move()->coalesce().toString();
}
std::vector<uint64_t> makeLookupKeys(size_t n, double hitRate) {
folly::BenchmarkSuspender guard;
std::vector<uint64_t> keys;
keys.reserve(n);
uint64_t maxKey = maxValue(FLAGS_key_bits);
for (uint64_t idx = 0; idx < n; idx++) {
keys.push_back(
(folly::Random::randDouble01(rng) < hitRate)
? uniform[folly::Random::rand64(uniform.size(), rng)]
: folly::Random::rand64(rng) & maxKey);
}
return keys;
}
const folly::F14FastSet<uint64_t>& getF14Baseline() {
folly::BenchmarkSuspender guard;
static const auto set = [] {
folly::F14FastSet<uint64_t> ret(uniform.begin(), uniform.end());
LOG(INFO) << folly::format(
"Built F14FastSet, size: {}, space: {}",
ret.size(),
folly::prettyPrint(
ret.getAllocatedMemorySize(), folly::PrettyType::PRETTY_BYTES_IEC));
return ret;
}();
return set;
}
using EFEncoder =
folly::compression::EliasFanoEncoderV2<uint64_t, uint64_t, 128, 128>;
const folly::compression::MutableEliasFanoCompressedList& getEFBaseline() {
folly::BenchmarkSuspender guard;
static auto list = [] {
auto ret = EFEncoder::encode(uniform.begin(), uniform.end());
LOG(INFO) << folly::format(
"Built Elias-Fano list, space: {}",
folly::prettyPrint(
ret.data.size(), folly::PrettyType::PRETTY_BYTES_IEC));
return ret;
}();
return list;
}
template <class Lookup>
size_t benchmarkHits(const Lookup& lookup) {
auto keys = makeLookupKeys(kRunsPerIteration, /* hitRate */ 1);
for (const auto& key : keys) {
auto found = lookup(key);
folly::doNotOptimizeAway(found);
DCHECK(found);
}
return kRunsPerIteration;
}
template <class Lookup>
size_t benchmarkRandom(const Lookup& lookup) {
auto keys = makeLookupKeys(kRunsPerIteration, /* hitRate */ 0);
for (const auto& key : keys) {
auto found = lookup(key);
folly::doNotOptimizeAway(found);
}
return kRunsPerIteration;
}
template <class Lookup>
size_t benchmarkMixtureSerialized(const Lookup& lookup) {
// Make the result unpredictable so we can use it to introduce a
// serializing dependency in the loop.
auto keys = makeLookupKeys(kRunsPerIteration * 2, /* hitRate */ 0.5);
size_t unpredictableBit = 0;
for (size_t i = 0; i < kRunsPerIteration * 2; i += 2) {
unpredictableBit = lookup(keys[i + unpredictableBit]) ? 1 : 0;
folly::doNotOptimizeAway(unpredictableBit);
}
return kRunsPerIteration;
}
template <class Lookup>
size_t benchmarkSerializedOnResultBits(const Lookup& lookup, double hitRate) {
auto keys = makeLookupKeys(kRunsPerIteration * 2, hitRate);
size_t unpredictableBit = 0;
for (size_t i = 0; i < kRunsPerIteration * 2; i += 2) {
// When all keys are hits we can use a hash of the location of the
// found key to introduce a serializing dependency in the loop.
unpredictableBit = mix64_to_1(lookup(keys[i + unpredictableBit]));
folly::doNotOptimizeAway(unpredictableBit);
}
return kRunsPerIteration;
}
template <class Lookup>
size_t benchmarkHitsSerialized(const Lookup& lookup) {
return benchmarkSerializedOnResultBits(lookup, /* hitRate */ 1);
}
template <class Lookup>
size_t benchmarkRandomSerialized(const Lookup& lookup) {
return benchmarkSerializedOnResultBits(lookup, /* hitRate */ 0);
}
template <class Lookup>
size_t benchmarkHitsSmallWorkingSet(const Lookup& lookup) {
// Loop over a small set of keys so that after the first iteration
// all the relevant parts of the data structure are in LLC cache.
constexpr size_t kLookupSetSize = 1 << 12; // Should be a power of 2.
auto keys = makeLookupKeys(kLookupSetSize, /* hitRate */ 1);
for (size_t i = 0; i < kRunsPerIteration; ++i) {
auto found = lookup(keys[i % kLookupSetSize]);
folly::doNotOptimizeAway(found);
DCHECK(found);
}
return kRunsPerIteration;
}
} // namespace
void benchmarkSetup() {
rng.seed(UINT64_C(12345678));
uint64_t maxKey = maxValue(FLAGS_key_bits);
// Uniformly distributed keys.
const uint64_t size = FLAGS_num_elements;
for (uint64_t idx = 0; idx < size; idx++) {
uint64_t key = folly::Random::rand64(rng) & maxKey;
uniform.emplace_back(key);
}
boost::sort::spreadsort::integer_sort(uniform.begin(), uniform.end());
buildQuotientMultiSet(uniform);
LOG(INFO) << folly::format(
"Built QuotientMultiSet, space: {}",
folly::prettyPrint(qmsData.size(), folly::PrettyType::PRETTY_BYTES_IEC));
}
BENCHMARK_MULTI(QuotientMultiSetGetHits) {
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = folly::QuotientMultiSet<decltype(instructions)>(qmsData);
ret = benchmarkHits([&](uint64_t key) { return reader.equalRange(key); });
});
return ret;
}
BENCHMARK_MULTI(QuotientMultiSetGetRandom) {
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = folly::QuotientMultiSet<decltype(instructions)>(qmsData);
ret = benchmarkRandom([&](uint64_t key) { return reader.equalRange(key); });
});
return ret;
}
BENCHMARK_MULTI(QuotientMultiSetGetHitsSmallWorkingSet) {
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = folly::QuotientMultiSet<decltype(instructions)>(qmsData);
ret = benchmarkHitsSmallWorkingSet(
[&](uint64_t key) { return reader.equalRange(key); });
});
return ret;
}
BENCHMARK_MULTI(QuotientMultiSetGetMixtureSerialized) {
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = folly::QuotientMultiSet<decltype(instructions)>(qmsData);
ret = benchmarkMixtureSerialized(
[&](uint64_t key) { return reader.equalRange(key); });
});
return ret;
}
BENCHMARK_MULTI(QuotientMultiSetGetHitsSerialized) {
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = folly::QuotientMultiSet<decltype(instructions)>(qmsData);
ret = benchmarkHitsSerialized(
[&](uint64_t key) { return reader.equalRange(key).begin; });
});
return ret;
}
BENCHMARK_MULTI(QuotientMultiSetGetRandomSerialized) {
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = folly::QuotientMultiSet<decltype(instructions)>(qmsData);
ret = benchmarkRandomSerialized([&](uint64_t key) {
// Even without a match, begin will depend on the whole
// computation.
return reader.equalRange(key).begin;
});
});
return ret;
}
BENCHMARK_DRAW_LINE();
BENCHMARK_MULTI(F14MapHits) {
const auto& baseline = getF14Baseline();
return benchmarkHits(
[&](uint64_t key) { return baseline.find(key) != baseline.end(); });
}
BENCHMARK_MULTI(F14MapRandom) {
const auto& baseline = getF14Baseline();
return benchmarkRandom(
[&](uint64_t key) { return baseline.find(key) != baseline.end(); });
}
BENCHMARK_MULTI(F14MapHitsSmallWorkingSet) {
const auto& baseline = getF14Baseline();
return benchmarkHitsSmallWorkingSet(
[&](uint64_t key) { return baseline.find(key) != baseline.end(); });
}
BENCHMARK_MULTI(F14MapMixtureSerialized) {
const auto& baseline = getF14Baseline();
return benchmarkMixtureSerialized(
[&](uint64_t key) { return baseline.find(key) != baseline.end(); });
}
BENCHMARK_MULTI(F14MapHitsSerialized) {
const auto& baseline = getF14Baseline();
return benchmarkHitsSerialized([&](uint64_t key) {
return reinterpret_cast<uintptr_t>(&*baseline.find(key));
});
}
// benchmarkRandomSerialized() cannot be used with F14.
BENCHMARK_DRAW_LINE();
using folly::compression::EliasFanoReader;
BENCHMARK_MULTI(EliasFanoGetHits) {
auto list = getEFBaseline();
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = EliasFanoReader<EFEncoder, decltype(instructions)>(list);
ret = benchmarkHits([&](uint64_t key) {
return reader.jumpTo(key) && reader.value() == key;
});
});
return ret;
}
BENCHMARK_MULTI(EliasFanoGetRandom) {
auto list = getEFBaseline();
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = EliasFanoReader<EFEncoder, decltype(instructions)>(list);
ret = benchmarkRandom([&](uint64_t key) {
return reader.jumpTo(key) && reader.value() == key;
});
});
return ret;
}
BENCHMARK_MULTI(EliasFanoGetHitsSmallWorkingSet) {
auto list = getEFBaseline();
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = EliasFanoReader<EFEncoder, decltype(instructions)>(list);
ret = benchmarkHitsSmallWorkingSet([&](uint64_t key) {
return reader.jumpTo(key) && reader.value() == key;
});
});
return ret;
}
BENCHMARK_MULTI(EliasFanoGetMixtureSerialized) {
auto list = getEFBaseline();
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = EliasFanoReader<EFEncoder, decltype(instructions)>(list);
ret = benchmarkMixtureSerialized([&](uint64_t key) {
return reader.jumpTo(key) && reader.value() == key;
});
});
return ret;
}
BENCHMARK_MULTI(EliasFanoGetHitsSerialized) {
auto list = getEFBaseline();
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = EliasFanoReader<EFEncoder, decltype(instructions)>(list);
ret = benchmarkHitsSerialized([&](uint64_t key) {
reader.jumpTo(key);
DCHECK(reader.valid());
return reader.position();
});
});
return ret;
}
BENCHMARK_MULTI(EliasFanoGetRandomSerialized) {
auto list = getEFBaseline();
size_t ret = 0;
folly::compression::dispatchInstructions([&](auto instructions) {
auto reader = EliasFanoReader<EFEncoder, decltype(instructions)>(list);
ret = benchmarkRandomSerialized([&](uint64_t key) {
reader.jumpTo(key);
return reader.position();
});
});
return ret;
}
#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
int main(int argc, char** argv) {
folly::init(&argc, &argv);
#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
benchmarkSetup();
folly::runBenchmarks();
#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
}
#if 0
// Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GHz, built with Clang.
$ buck run @mode/opt folly/experimental/test:quotient_multiset_benchmark -- --benchmark --bm_max_secs 3
I0917 10:28:54.497815 2874547 QuotientMultiSetBenchmark.cpp:195] Built QuotientMultiSet, space: 124.9 MiB
============================================================================
folly/experimental/test/QuotientMultiSetBenchmark.cpprelative time/iter iters/s
============================================================================
QuotientMultiSetGetHits 114.94ns 8.70M
QuotientMultiSetGetRandom 72.87ns 13.72M
QuotientMultiSetGetHitsSmallWorkingSet 58.75ns 17.02M
QuotientMultiSetGetMixtureSerialized 138.73ns 7.21M
QuotientMultiSetGetHitsSerialized 138.56ns 7.22M
QuotientMultiSetGetRandomSerialized 130.11ns 7.69M
----------------------------------------------------------------------------
I0917 10:29:33.808831 2874547 QuotientMultiSetBenchmark.cpp:83] Built F14FastSet, size: 98843868, space: 1 GiB
F14MapHits 34.69ns 28.83M
F14MapRandom 47.22ns 21.18M
F14MapHitsSmallWorkingSet 14.42ns 69.34M
F14MapMixtureSerialized 94.64ns 10.57M
F14MapHitsSerialized 140.74ns 7.11M
----------------------------------------------------------------------------
I0917 10:29:52.722596 2874547 QuotientMultiSetBenchmark.cpp:100] Built Elias-Fano list, space: 101.5 MiB
EliasFanoGetHits 258.08ns 3.87M
EliasFanoGetRandom 247.18ns 4.05M
EliasFanoGetHitsSmallWorkingSet 81.54ns 12.26M
EliasFanoGetMixtureSerialized 253.63ns 3.94M
EliasFanoGetHitsSerialized 161.57ns 6.19M
EliasFanoGetRandomSerialized 161.55ns 6.19M
============================================================================
#endif
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/experimental/QuotientMultiSet.h>
#include <random>
#include <folly/Format.h>
#include <folly/Random.h>
#include <folly/container/Enumerate.h>
#include <folly/io/IOBufQueue.h>
#include <folly/portability/GTest.h>
#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
namespace {
class QuotientMultiSetTest : public ::testing::Test {
protected:
static constexpr uint64_t kBlockSize = folly::QuotientMultiSet<>::kBlockSize;
void SetUp() override {
rng.seed(folly::randomNumberSeed());
}
void buildAndValidate(
std::vector<uint64_t>& keys,
uint64_t keyBits,
double loadFactor) {
// Elements must be added in ascending order.
std::sort(keys.begin(), keys.end());
folly::QuotientMultiSetBuilder builder(keyBits, keys.size(), loadFactor);
folly::IOBufQueue buff;
for (const auto& iter : folly::enumerate(keys)) {
if (builder.insert(*iter)) {
// Set payload to relative position of the first key in block.
builder.setBlockPayload(iter.index);
}
if (builder.numReadyBlocks() >= 1) {
builder.flush(buff);
}
}
builder.close(buff);
auto spBuf = buff.move();
folly::StringPiece data(spBuf->coalesce());
folly::QuotientMultiSet reader(data);
size_t index = 0;
folly::QuotientMultiSet<>::Iterator iter(&reader);
while (index < keys.size()) {
uint64_t start = index;
uint64_t& key = keys[start];
auto debugInfo = [&] {
return folly::sformat("Index: {} Key: {}", index, key);
};
while (index < keys.size() && keys[index] == key) {
index++;
}
auto range = reader.equalRange(key);
size_t pos = reader.getBlockPayload(range.begin / kBlockSize);
EXPECT_EQ(start, pos + range.begin % kBlockSize) << debugInfo();
pos = reader.getBlockPayload((range.end - 1) / kBlockSize);
EXPECT_EQ(index - 1, pos + (range.end - 1) % kBlockSize) << debugInfo();
iter.skipTo(key);
EXPECT_EQ(key, iter.key()) << debugInfo();
EXPECT_EQ(range.begin, iter.pos()) << debugInfo();
EXPECT_EQ(
start,
reader.getBlockPayload(iter.pos() / kBlockSize) +
iter.pos() % kBlockSize)
<< debugInfo();
if (start < keys.size() - 1) {
EXPECT_TRUE(iter.next());
EXPECT_EQ(keys[start + 1], iter.key());
}
// Verify getting keys not in keys returns false.
uint64_t prevKey = (start == 0 ? 0 : keys[start - 1]);
if (prevKey + 1 < key) {
uint64_t missedKey = folly::Random::rand64(prevKey + 1, key, rng);
EXPECT_FALSE(reader.equalRange(missedKey)) << key;
iter.skipTo(missedKey);
EXPECT_EQ(key, iter.key()) << debugInfo();
EXPECT_EQ(range.begin, iter.pos()) << debugInfo();
EXPECT_EQ(
start,
reader.getBlockPayload(iter.pos() / kBlockSize) +
iter.pos() % kBlockSize)
<< debugInfo();
if (start < keys.size() - 1) {
EXPECT_TRUE(iter.next());
EXPECT_EQ(keys[start + 1], iter.key());
}
}
}
const auto maxKey = folly::qms_detail::maxValue(keyBits);
if (keys.back() < maxKey) {
uint64_t key = folly::Random::rand64(keys.back(), maxKey, rng) + 1;
EXPECT_FALSE(reader.equalRange(key)) << keys.back() << " " << key;
EXPECT_FALSE(iter.skipTo(key));
EXPECT_TRUE(iter.done());
}
folly::QuotientMultiSet<>::Iterator nextIter(&reader);
for (const auto key : keys) {
EXPECT_TRUE(nextIter.next());
EXPECT_EQ(key, nextIter.key());
}
EXPECT_FALSE(nextIter.next());
EXPECT_TRUE(nextIter.done());
}
std::mt19937 rng;
};
} // namespace
TEST_F(QuotientMultiSetTest, Simple) {
std::vector<uint64_t> keys = {
100, 1000, 1 << 14, 10 << 14, 0, 10, 100 << 14, 1000 << 14};
buildAndValidate(keys, 32, 0.95);
}
TEST_F(QuotientMultiSetTest, Empty) {
folly::QuotientMultiSetBuilder builder(32, 1024);
folly::IOBufQueue buff;
builder.close(buff);
auto spBuf = buff.move();
folly::StringPiece data(spBuf->coalesce());
folly::QuotientMultiSet reader(data);
for (size_t idx = 0; idx < 1024; idx++) {
uint64_t key = folly::Random::rand32(rng);
EXPECT_FALSE(reader.equalRange(key));
}
}
TEST_F(QuotientMultiSetTest, ZeroKeyBits) {
std::vector<uint64_t> keys(67, 0);
buildAndValidate(keys, 0, 0.95);
}
TEST_F(QuotientMultiSetTest, Uniform) {
constexpr auto kLoadFactor =
folly::QuotientMultiSetBuilder::kDefaultMaxLoadFactor;
constexpr uint64_t kAvgSize = 1 << 16;
auto randSize = [&](uint64_t avgSize) {
return folly::Random::rand64(avgSize / 2, avgSize * 3 / 2, rng);
};
std::vector<std::tuple<int, uint64_t, double>> testCases = {
{1, randSize(1 << 9), kLoadFactor},
{8, randSize(1 << 10), kLoadFactor},
{9, randSize(1 << 11), kLoadFactor},
{12, randSize(kAvgSize), kLoadFactor},
{32, randSize(kAvgSize), kLoadFactor},
{48, randSize(kAvgSize), kLoadFactor},
{64, randSize(kAvgSize), kLoadFactor},
{32, randSize(kAvgSize), 1}, // Full
{12, 3800, kLoadFactor}, // Almost full
{64, randSize(16), kLoadFactor}, // Sparse, long keys.
};
for (const auto& testCase : testCases) {
const auto& [keyBits, size, loadFactor] = testCase;
SCOPED_TRACE(folly::sformat(
"Key bits: {} Size: {} Load factor: {}", keyBits, size, loadFactor));
std::vector<uint64_t> keys;
for (uint64_t idx = 0; idx < size; idx++) {
keys.emplace_back(
folly::Random::rand64(rng) & folly::qms_detail::maxValue(keyBits));
}
buildAndValidate(keys, keyBits, loadFactor);
}
}
TEST_F(QuotientMultiSetTest, UniformDistributionFullLoadFactor) {
const uint64_t numElements = 1 << 16;
std::vector<uint64_t> keys;
for (uint64_t idx = 0; idx < numElements; idx++) {
uint64_t key = folly::Random::rand32(idx << 16, (idx + 1) << 16, rng);
keys.emplace_back(key);
}
buildAndValidate(keys, 32, 1.0);
}
TEST_F(QuotientMultiSetTest, Overflow) {
const uint64_t numElements = 1 << 12;
std::vector<uint64_t> keys;
for (uint64_t idx = 0; idx < numElements; idx++) {
keys.emplace_back(idx);
keys.emplace_back(idx);
keys.emplace_back(idx);
}
buildAndValidate(keys, 12, 0.95);
}
TEST_F(QuotientMultiSetTest, RandomLengthRuns) {
const uint64_t numElements = 1 << 16;
std::vector<uint64_t> keys;
for (uint64_t idx = 0; idx < (numElements >> 4); idx++) {
uint64_t key = folly::Random::rand32(rng);
uint64_t length = folly::Random::rand32(0, 10, rng);
for (uint64_t k = 0; k < length; k++) {
keys.emplace_back(key + k);
}
}
buildAndValidate(keys, 32, 0.95);
}
TEST_F(QuotientMultiSetTest, RunAcrossBlocks) {
const uint64_t numElements = 1 << 10;
std::vector<uint64_t> keys;
// Add keys with cluster size 137.
for (uint64_t idx = 0; idx < (numElements >> 4); idx++) {
uint64_t key = folly::Random::rand32(rng);
for (uint64_t k = 0; k < 136; k++) {
key += k;
keys.emplace_back(key);
}
}
buildAndValidate(keys, 32, 0.95);
}
TEST_F(QuotientMultiSetTest, PackAtHeadSlots) {
const uint64_t numElements = 1 << 12;
std::vector<uint64_t> keys;
for (uint64_t idx = 0; idx < numElements; idx++) {
uint64_t key = folly::Random::rand32(idx << 8, (idx + 1) << 8, rng);
keys.emplace_back(key);
}
buildAndValidate(keys, 32, 0.95);
}
TEST_F(QuotientMultiSetTest, PackAtTailSlots) {
const uint64_t numElements = 1 << 12;
std::vector<uint64_t> keys;
uint64_t key = (1 << 30);
for (uint64_t idx = 0; idx < numElements; idx++) {
keys.emplace_back(key + idx);
}
buildAndValidate(keys, 32, 0.95);
}
TEST_F(QuotientMultiSetTest, KeysOnlyInHeadAndTail) {
const uint64_t numElements = 1 << 11;
std::vector<uint64_t> keys;
for (uint64_t idx = 0; idx < numElements; idx++) {
keys.emplace_back(idx);
}
uint64_t key = (1 << 30);
for (uint64_t idx = 0; idx < numElements; idx++) {
keys.emplace_back(key + idx);
}
buildAndValidate(keys, 32, 0.95);
}
TEST_F(QuotientMultiSetTest, RunendRightBeforeFirstOccupiedRunend) {
std::vector<uint64_t> keys;
// 60 ranges [0, 67] with occupied slot 20.
for (size_t idx = 0; idx < 68; idx++) {
keys.push_back(60);
}
// 60 ranges [68, 68] with occupied slot 66.
for (size_t idx = 0; idx < 1; idx++) {
keys.push_back(200);
}
// 60 ranges [69, 88] with occupied slot 83.
for (size_t idx = 0; idx < 20; idx++) {
keys.push_back(250);
}
buildAndValidate(keys, 8, 0.95);
}
#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment