Commit de95d5f9 authored by Tingzhe Zhou's avatar Tingzhe Zhou Committed by Facebook Github Bot

Improve blocking and accuracy

Summary:
Blocking algorithm:
 Push:
1.  push
2.  increase the PTicket number
3.  put (PTicket << 1) to the corresponding place in futex array.
4.  if the original  futex value & 1  == 1,  wake up ()

Pop:
1.  increase the CTicket number
2.  spin if the corresponding CTicket futex is not ready
3.  set the right bit of the CTicket futex  to 1, wait on it.
4. pop

Add additional control to improve the accuracy:
For the top several levels (top 0 - 3 levels by default), it does not arbitrarily insert any node in the middle of the list. Thus guaranteeing no later coming lower priorities can be insert at the top levels (pruning strategy still valid for the top four levels). For the bottom levels (start from level 7 by default), if insertion requires to hold parent + child locks, it traverses the parent list to check whether the parent list has lower priority nodes than the new node. If the condition holds, insert the new node to the parent and get the last node in the parent list and insert it to the child list.

Reviewed By: magedm

Differential Revision: D9094903

fbshipit-source-id: fa4d31fb0c21da746570029e0e3b4b3fbeaa2024
parent 7f721452
...@@ -112,6 +112,8 @@ ...@@ -112,6 +112,8 @@
// size_t size() // size_t size()
// bool empty() // bool empty()
namespace folly {
template < template <
typename T, typename T,
bool MayBlock = false, bool MayBlock = false,
...@@ -128,6 +130,8 @@ class RelaxedConcurrentPriorityQueue { ...@@ -128,6 +130,8 @@ class RelaxedConcurrentPriorityQueue {
// Align size for the shared buffer node // Align size for the shared buffer node
static constexpr size_t Align = 1u << 7; static constexpr size_t Align = 1u << 7;
static constexpr int LevelForForceInsert = 3;
static constexpr int LevelForTraverseParent = 7;
static_assert(PopBatch <= 256, "PopBatch must be <= 256"); static_assert(PopBatch <= 256, "PopBatch must be <= 256");
static_assert( static_assert(
...@@ -185,12 +189,14 @@ class RelaxedConcurrentPriorityQueue { ...@@ -185,12 +189,14 @@ class RelaxedConcurrentPriorityQueue {
std::unique_ptr<BufferNode[]> shared_buffer_; std::unique_ptr<BufferNode[]> shared_buffer_;
alignas(Align) Atom<int> top_loc_; alignas(Align) Atom<int> top_loc_;
// Numbers of active waiting threads /// Blocking algorithm
// (Written by consumer; read by producer) // Numbers of futexs in the array
alignas(Align) Atom<int> active_waiting_; static constexpr size_t NumFutex = 128;
// Monotonically increasing value (overflow to 0 is allowed) // The index gap for accessing futex in the array
// (Written by producer; read by consumer) static constexpr size_t Stride = 33;
folly::detail::Futex<Atom> seq_number_; std::unique_ptr<folly::detail::Futex<Atom>[]> futex_array_;
alignas(Align) Atom<uint32_t> cticket_;
alignas(Align) Atom<uint32_t> pticket_;
// Two counters to calculate size of the queue // Two counters to calculate size of the queue
alignas(Align) Atom<size_t> counter_p_; alignas(Align) Atom<size_t> counter_p_;
...@@ -199,7 +205,11 @@ class RelaxedConcurrentPriorityQueue { ...@@ -199,7 +205,11 @@ class RelaxedConcurrentPriorityQueue {
public: public:
/// Constructor /// Constructor
RelaxedConcurrentPriorityQueue() RelaxedConcurrentPriorityQueue()
: active_waiting_(0), seq_number_(0), counter_p_(0), counter_c_(0) { : cticket_(1), pticket_(1), counter_p_(0), counter_c_(0) {
if (MayBlock) {
futex_array_.reset(new folly::detail::Futex<Atom>[NumFutex]);
}
if (PopBatch > 0) { if (PopBatch > 0) {
top_loc_ = -1; top_loc_ = -1;
shared_buffer_.reset(new BufferNode[PopBatch]); shared_buffer_.reset(new BufferNode[PopBatch]);
...@@ -220,6 +230,9 @@ class RelaxedConcurrentPriorityQueue { ...@@ -220,6 +230,9 @@ class RelaxedConcurrentPriorityQueue {
if (PopBatch > 0) { if (PopBatch > 0) {
deleteSharedBuffer(); deleteSharedBuffer();
} }
if (MayBlock) {
futex_array_.reset();
}
Position pos; Position pos;
pos.level = pos.index = 0; pos.level = pos.index = 0;
deleteAllNodes(pos); deleteAllNodes(pos);
...@@ -412,7 +425,8 @@ class RelaxedConcurrentPriorityQueue { ...@@ -412,7 +425,8 @@ class RelaxedConcurrentPriorityQueue {
path = false; path = false;
seed = ++loc; seed = ++loc;
return pos; return pos;
} else if (getElementSize(pos) < ListTargetSize) { } else if (
b > LevelForForceInsert && getElementSize(pos) < ListTargetSize) {
// [fast path] conservative implementation // [fast path] conservative implementation
// it makes sure every tree element should // it makes sure every tree element should
// have more than the given number of nodes. // have more than the given number of nodes.
...@@ -647,7 +661,6 @@ class RelaxedConcurrentPriorityQueue { ...@@ -647,7 +661,6 @@ class RelaxedConcurrentPriorityQueue {
return false; return false;
} }
// TODO a good place to improve accuracy
// insert to an inner node // insert to an inner node
Position parent = parentOf(pos); Position parent = parentOf(pos);
if (!trylockNode(parent)) { if (!trylockNode(parent)) {
...@@ -660,11 +673,51 @@ class RelaxedConcurrentPriorityQueue { ...@@ -660,11 +673,51 @@ class RelaxedConcurrentPriorityQueue {
T pv = readValue(parent); T pv = readValue(parent);
T nv = readValue(pos); T nv = readValue(pos);
if (LIKELY(pv > val && nv <= val)) { if (LIKELY(pv > val && nv <= val)) {
newNode->next = getList(pos); // improve the accuracy by getting the node(R) with less priority than the
setTreeNode(pos, newNode); // new value from parent level, insert the new node to the parent list
// and insert R to the current list.
// It only happens at >= LevelForTraverseParent for reducing contention
uint32_t sz = getElementSize(pos); uint32_t sz = getElementSize(pos);
if (pos.level >= LevelForTraverseParent) {
Node* start = getList(parent);
while (start->next != nullptr && start->next->val >= val) {
start = start->next;
}
if (start->next != nullptr) {
newNode->next = start->next;
start->next = newNode;
while (start->next->next != nullptr) {
start = start->next;
}
newNode = start->next;
start->next = nullptr;
}
unlockNode(parent);
Node* curList = getList(pos);
if (curList == nullptr) {
newNode->next = nullptr;
setTreeNode(pos, newNode);
} else {
Node* p = curList;
if (p->val <= newNode->val) {
newNode->next = curList;
setTreeNode(pos, newNode);
} else {
while (p->next != nullptr && p->next->val >= newNode->val) {
p = p->next;
}
newNode->next = p->next;
p->next = newNode;
}
}
setElementSize(pos, sz + 1); setElementSize(pos, sz + 1);
} else {
unlockNode(parent); unlockNode(parent);
newNode->next = getList(pos);
setTreeNode(pos, newNode);
setElementSize(pos, sz + 1);
}
if (UNLIKELY(sz > PruningSize)) { if (UNLIKELY(sz > PruningSize)) {
startPruning(pos); startPruning(pos);
} else { } else {
...@@ -689,14 +742,24 @@ class RelaxedConcurrentPriorityQueue { ...@@ -689,14 +742,24 @@ class RelaxedConcurrentPriorityQueue {
if (sz >= ListTargetSize) { if (sz >= ListTargetSize) {
return false; return false;
} }
Node* p = getList(pos);
newNode->next = p; Node* curList = getList(pos);
if (curList == nullptr) {
newNode->next = nullptr;
setTreeNode(pos, newNode); setTreeNode(pos, newNode);
p = newNode; } else {
while (p != nullptr && p->next != nullptr && p->val <= p->next->val) { Node* p = curList;
std::swap(p->val, p->next->val); if (p->val <= newNode->val) {
newNode->next = curList;
setTreeNode(pos, newNode);
} else {
while (p->next != nullptr && p->next->val >= newNode->val) {
p = p->next; p = p->next;
} }
newNode->next = p->next;
p->next = newNode;
}
}
setElementSize(pos, sz + 1); setElementSize(pos, sz + 1);
return true; return true;
} }
...@@ -962,23 +1025,30 @@ class RelaxedConcurrentPriorityQueue { ...@@ -962,23 +1025,30 @@ class RelaxedConcurrentPriorityQueue {
return true; return true;
} }
FOLLY_ALWAYS_INLINE bool toWakeUp() { void blockingPushImpl() {
std::atomic_thread_fence(std::memory_order_seq_cst); auto p = pticket_.fetch_add(1, std::memory_order_acq_rel);
auto c = active_waiting_.load(std::memory_order_acquire); auto loc = getFutexArrayLoc(p);
return c > 0; uint32_t curfutex = futex_array_[loc].load(std::memory_order_acquire);
}
void wakeUp() { while (true) {
seq_number_.fetch_add(1, std::memory_order_seq_cst); uint32_t ready = p << 1; // get the lower 31 bits
seq_number_.futexWake(1); // avoid the situation that push has larger ticket already set the value
if (UNLIKELY(
ready + 1 < curfutex ||
((curfutex > ready) && (curfutex - ready > 0x40000000)))) {
return;
} }
void blockingPushImpl() { if (futex_array_[loc].compare_exchange_strong(curfutex, ready)) {
if (LIKELY(!toWakeUp())) { if (curfutex &
1) { // One or more consumers may be blocked on this futex
futex_array_[loc].futexWake();
}
return; return;
} else {
curfutex = futex_array_[loc].load(std::memory_order_acquire);
}
} }
// slow path, likely to have a futexWake call
wakeUp();
} }
// This could guarentee the Mound is empty // This could guarentee the Mound is empty
...@@ -993,25 +1063,70 @@ class RelaxedConcurrentPriorityQueue { ...@@ -993,25 +1063,70 @@ class RelaxedConcurrentPriorityQueue {
return top_loc_.load(std::memory_order_acquire) < 0; return top_loc_.load(std::memory_order_acquire) < 0;
} }
bool isEmpty() { FOLLY_ALWAYS_INLINE bool isEmpty() {
if (PopBatch > 0) { if (PopBatch > 0) {
return isMoundEmpty() && isSharedBufferEmpty(); return isMoundEmpty() && isSharedBufferEmpty();
} }
return isMoundEmpty(); return isMoundEmpty();
} }
void blockingPopImpl() { FOLLY_ALWAYS_INLINE bool futexIsReady(const size_t& curticket) {
active_waiting_.fetch_add(1, std::memory_order_seq_cst); auto loc = getFutexArrayLoc(curticket);
auto w = seq_number_.load(std::memory_order_acquire); auto curfutex = futex_array_[loc].load(std::memory_order_acquire);
uint32_t short_cticket = curticket & 0x7FFFFFFF;
uint32_t futex_ready = curfutex >> 1;
// handle unsigned 31 bits overflow
return futex_ready >= short_cticket ||
short_cticket - futex_ready > 0x40000000;
}
std::atomic_thread_fence(std::memory_order_seq_cst); template <typename Clock, typename Duration>
if (!isEmpty()) { // not empty FOLLY_NOINLINE bool trySpinBeforeBlock(
active_waiting_.fetch_sub(1, std::memory_order_seq_cst); const size_t& curticket,
const std::chrono::time_point<Clock, Duration>& deadline,
const folly::WaitOptions& opt = wait_options()) {
return folly::detail::spin_pause_until(deadline, opt, [=] {
return futexIsReady(curticket);
}) == folly::detail::spin_result::success;
}
void tryBlockingPop(const size_t& curticket) {
auto loc = getFutexArrayLoc(curticket);
auto curfutex = futex_array_[loc].load(std::memory_order_acquire);
if (curfutex &
1) { /// The last round consumers are still waiting, go to sleep
futex_array_[loc].futexWait(curfutex);
}
if (trySpinBeforeBlock(
curticket,
std::chrono::time_point<std::chrono::steady_clock>::max())) {
return; /// Spin until the push ticket is ready
}
while (true) {
curfutex = futex_array_[loc].load(std::memory_order_acquire);
if (curfutex &
1) { /// The last round consumers are still waiting, go to sleep
futex_array_[loc].futexWait(curfutex);
} else if (!futexIsReady(curticket)) { // current ticket < pop ticket
uint32_t blocking_futex = curfutex + 1;
if (futex_array_[loc].compare_exchange_strong(
curfutex, blocking_futex)) {
futex_array_[loc].futexWait(blocking_futex);
}
} else {
return; return;
} }
}
}
seq_number_.futexWait(w); void blockingPopImpl() {
active_waiting_.fetch_sub(1, std::memory_order_seq_cst); auto ct = cticket_.fetch_add(1, std::memory_order_acq_rel);
// fast path check
if (futexIsReady(ct)) {
return;
}
// Blocking
tryBlockingPop(ct);
} }
bool tryPopFromMound(T& val) { bool tryPopFromMound(T& val) {
...@@ -1048,19 +1163,15 @@ class RelaxedConcurrentPriorityQueue { ...@@ -1048,19 +1163,15 @@ class RelaxedConcurrentPriorityQueue {
} }
// Spinning strategy // Spinning strategy
if (!MayBlock) {
while (true) { while (true) {
auto res = folly::detail::spin_yield_until( auto res =
deadline, [=] { return !isEmpty(); }); folly::detail::spin_yield_until(deadline, [=] { return !isEmpty(); });
if (res == folly::detail::spin_result::success) { if (res == folly::detail::spin_result::success) {
return true; return true;
} else if (res == folly::detail::spin_result::timeout) { } else if (res == folly::detail::spin_result::timeout) {
return false; return false;
} }
} }
} else { // Blocking strategy
blockingPopImpl();
}
return true; return true;
} }
...@@ -1079,7 +1190,15 @@ class RelaxedConcurrentPriorityQueue { ...@@ -1079,7 +1190,15 @@ class RelaxedConcurrentPriorityQueue {
return false; return false;
} }
size_t getFutexArrayLoc(size_t s) {
return ((s - 1) * Stride) & (NumFutex - 1);
}
void moundPop(T& val) { void moundPop(T& val) {
if (MayBlock) {
blockingPopImpl();
}
if (PopBatch > 0) { if (PopBatch > 0) {
if (tryPopFromSharedBuffer(val)) { if (tryPopFromSharedBuffer(val)) {
return; return;
...@@ -1097,3 +1216,5 @@ class RelaxedConcurrentPriorityQueue { ...@@ -1097,3 +1216,5 @@ class RelaxedConcurrentPriorityQueue {
} }
} }
}; };
} // namespace folly
...@@ -24,6 +24,8 @@ ...@@ -24,6 +24,8 @@
#include <folly/test/DeterministicSchedule.h> #include <folly/test/DeterministicSchedule.h>
#include <glog/logging.h> #include <glog/logging.h>
using namespace folly;
DEFINE_int32(reps, 1, "number of reps"); DEFINE_int32(reps, 1, "number of reps");
DEFINE_int64(ops, 32, "number of operations per rep"); DEFINE_int64(ops, 32, "number of operations per rep");
DEFINE_int64(elems, 64, "number of elements"); DEFINE_int64(elems, 64, "number of elements");
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment