Commit 82f9889c authored by Maged Michael's avatar Maged Michael Committed by Facebook GitHub Bot

hazard pointers: Consolidate non-cohort and untagged cohort retired objects

Summary:
Consolidate retired_, the list of non-cohort retired objects, with untagged_, the list of cohort untagged objects, in the domain.

Eliminate redundant functions.

Reviewed By: davidtgoldblatt

Differential Revision: D30806882

fbshipit-source-id: bdcbc4990a3d114f7f6bea294df498c5743bed43
parent b3833ef1
...@@ -142,17 +142,10 @@ hazptr_domain<Atom>& default_hazptr_domain(); ...@@ -142,17 +142,10 @@ hazptr_domain<Atom>& default_hazptr_domain();
template <template <typename> class Atom = std::atomic> template <template <typename> class Atom = std::atomic>
hazard_pointer_domain<Atom>& hazard_pointer_default_domain(); hazard_pointer_domain<Atom>& hazard_pointer_default_domain();
/** hazptr_domain_push_list */
template <template <typename> class Atom = std::atomic>
void hazptr_domain_push_list(
hazptr_obj_list<Atom>& l,
hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) noexcept;
/** hazptr_domain_push_retired */ /** hazptr_domain_push_retired */
template <template <typename> class Atom = std::atomic> template <template <typename> class Atom = std::atomic>
void hazptr_domain_push_retired( void hazptr_domain_push_retired(
hazptr_obj_list<Atom>& l, hazptr_obj_list<Atom>& l,
bool check = true,
hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) noexcept; hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) noexcept;
/** hazptr_retire */ /** hazptr_retire */
......
...@@ -117,13 +117,11 @@ class hazptr_domain { ...@@ -117,13 +117,11 @@ class hazptr_domain {
} }
Atom<hazptr_rec<Atom>*> hazptrs_{nullptr}; Atom<hazptr_rec<Atom>*> hazptrs_{nullptr};
Atom<hazptr_obj<Atom>*> retired_{nullptr};
Atom<uint64_t> sync_time_{0}; Atom<uint64_t> sync_time_{0};
/* Using signed int for rcount_ because it may transiently be negative. /* Using signed int for rcount_ because it may transiently be negative.
Using signed int for all integer variables that may be involved in Using signed int for all integer variables that may be involved in
calculations related to the value of rcount_. */ calculations related to the value of rcount_. */
Atom<int> hcount_{0}; Atom<int> hcount_{0};
Atom<int> rcount_{0};
Atom<uint16_t> num_bulk_reclaims_{0}; Atom<uint16_t> num_bulk_reclaims_{0};
bool shutdown_{false}; bool shutdown_{false};
RetiredList untagged_; RetiredList untagged_;
...@@ -174,12 +172,11 @@ class hazptr_domain { ...@@ -174,12 +172,11 @@ class hazptr_domain {
delete static_cast<hazptr_retire_node*>(p); delete static_cast<hazptr_retire_node*>(p);
}; };
hazptr_obj_list<Atom> l(node); hazptr_obj_list<Atom> l(node);
push_retired(l); push_list(l);
} }
/** cleanup */ /** cleanup */
void cleanup() noexcept { void cleanup() noexcept {
relaxed_cleanup();
inc_num_bulk_reclaims(); inc_num_bulk_reclaims();
do_reclamation(0); do_reclamation(0);
wait_for_zero_bulk_reclaims(); // wait for concurrent bulk_reclaim-s wait_for_zero_bulk_reclaims(); // wait for concurrent bulk_reclaim-s
...@@ -214,10 +211,8 @@ class hazptr_domain { ...@@ -214,10 +211,8 @@ class hazptr_domain {
hazptr_rec<Atom>, hazptr_rec<Atom>,
FixedAlign<alignof(hazptr_rec<Atom>)>>; FixedAlign<alignof(hazptr_rec<Atom>)>>;
friend void hazptr_domain_push_list<Atom>(
hazptr_obj_list<Atom>&, hazptr_domain<Atom>&) noexcept;
friend void hazptr_domain_push_retired<Atom>( friend void hazptr_domain_push_retired<Atom>(
hazptr_obj_list<Atom>&, bool check, hazptr_domain<Atom>&) noexcept; hazptr_obj_list<Atom>&, hazptr_domain<Atom>&) noexcept;
friend hazptr_holder<Atom> make_hazard_pointer<Atom>(hazptr_domain<Atom>&); friend hazptr_holder<Atom> make_hazard_pointer<Atom>(hazptr_domain<Atom>&);
friend class hazptr_holder<Atom>; friend class hazptr_holder<Atom>;
friend class hazptr_obj<Atom>; friend class hazptr_obj<Atom>;
...@@ -275,26 +270,6 @@ class hazptr_domain { ...@@ -275,26 +270,6 @@ class hazptr_domain {
/** hprec_release */ /** hprec_release */
void hprec_release(hazptr_rec<Atom>* hprec) noexcept { hprec->release(); } void hprec_release(hazptr_rec<Atom>* hprec) noexcept { hprec->release(); }
/** push_retired */
void push_retired(hazptr_obj_list<Atom>& l, bool check = true) {
/*** Full fence ***/ asymmetricLightBarrier();
while (true) {
auto r = retired();
l.tail()->set_next(r);
if (retired_.compare_exchange_weak(
r,
l.head(),
std::memory_order_release,
std::memory_order_acquire)) {
break;
}
}
rcount_.fetch_add(l.count(), std::memory_order_release);
if (check) {
check_cleanup_and_reclaim();
}
}
/** push_list */ /** push_list */
void push_list(ObjList& l) { void push_list(ObjList& l) {
if (l.empty()) { if (l.empty()) {
...@@ -438,7 +413,7 @@ class hazptr_domain { ...@@ -438,7 +413,7 @@ class hazptr_domain {
} }
/** match_reclaim_untagged */ /** match_reclaim_untagged */
int match_reclaim_untagged(Obj* untagged, Set& hs) { int match_reclaim_untagged(Obj* untagged, Set& hs, bool& done) {
ObjList match, nomatch; ObjList match, nomatch;
list_match_condition(untagged, match, nomatch, [&](Obj* o) { list_match_condition(untagged, match, nomatch, [&](Obj* o) {
return hs.count(o->raw_ptr()) > 0; return hs.count(o->raw_ptr()) > 0;
...@@ -446,6 +421,7 @@ class hazptr_domain { ...@@ -446,6 +421,7 @@ class hazptr_domain {
ObjList children; ObjList children;
int count = nomatch.count(); int count = nomatch.count();
reclaim_unprotected(nomatch.head(), children); reclaim_unprotected(nomatch.head(), children);
done = untagged_.empty() && children.empty();
count -= children.count(); count -= children.count();
match.splice(children); match.splice(children);
List l(match.head(), match.tail()); List l(match.head(), match.tail());
...@@ -458,36 +434,23 @@ class hazptr_domain { ...@@ -458,36 +434,23 @@ class hazptr_domain {
while (true) { while (true) {
Obj* untagged; Obj* untagged;
Obj* tagged[kNumShards]; Obj* tagged[kNumShards];
bool done = true;
if (extract_retired_objects(untagged, tagged)) { if (extract_retired_objects(untagged, tagged)) {
/*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED); /*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
Set hs = load_hazptr_vals(); Set hs = load_hazptr_vals();
rcount -= match_tagged(tagged, hs); rcount -= match_tagged(tagged, hs);
rcount -= match_reclaim_untagged(untagged, hs); rcount -= match_reclaim_untagged(untagged, hs, done);
} }
if (rcount) { if (rcount) {
add_count(rcount); add_count(rcount);
} }
rcount = check_count_threshold(); rcount = check_count_threshold();
if (rcount == 0) if (rcount == 0 && done)
break; break;
} }
dec_num_bulk_reclaims(); dec_num_bulk_reclaims();
} }
/** lookup_and_reclaim */
void lookup_and_reclaim(Obj* obj, const Set& hs, ObjList& keep) {
while (obj) {
auto next = obj->next();
DCHECK_NE(obj, next);
if (hs.count(obj->raw_ptr()) == 0) {
(*(obj->reclaim()))(obj, keep);
} else {
keep.push(obj);
}
obj = next;
}
}
/** list_match_condition */ /** list_match_condition */
template <typename Cond> template <typename Cond>
void list_match_condition( void list_match_condition(
...@@ -526,26 +489,12 @@ class hazptr_domain { ...@@ -526,26 +489,12 @@ class hazptr_domain {
return hazptrs_.load(std::memory_order_acquire); return hazptrs_.load(std::memory_order_acquire);
} }
hazptr_obj<Atom>* retired() const noexcept {
return retired_.load(std::memory_order_acquire);
}
int hcount() const noexcept { int hcount() const noexcept {
return hcount_.load(std::memory_order_acquire); return hcount_.load(std::memory_order_acquire);
} }
int rcount() const noexcept {
return rcount_.load(std::memory_order_acquire);
}
bool reached_threshold(int rc, int hc) const noexcept {
return rc >= kThreshold && rc >= kMultiplier * hc;
}
void reclaim_all_objects() { void reclaim_all_objects() {
auto head = retired_.exchange(nullptr); Obj* head = untagged_.pop_all(RetiredList::kDontLock);
reclaim_list_transitive(head);
head = untagged_.pop_all(RetiredList::kDontLock);
reclaim_list_transitive(head); reclaim_list_transitive(head);
} }
...@@ -573,103 +522,12 @@ class hazptr_domain { ...@@ -573,103 +522,12 @@ class hazptr_domain {
} }
} }
void check_cleanup_and_reclaim() {
if (try_timed_cleanup()) {
return;
}
if (reached_threshold(rcount(), hcount())) {
try_bulk_reclaim();
}
}
void relaxed_cleanup() noexcept {
rcount_.store(0, std::memory_order_release);
bulk_reclaim(true);
}
void wait_for_zero_bulk_reclaims() { void wait_for_zero_bulk_reclaims() {
while (load_num_bulk_reclaims() > 0) { while (load_num_bulk_reclaims() > 0) {
std::this_thread::yield(); std::this_thread::yield();
} }
} }
void try_bulk_reclaim() {
auto hc = hcount();
auto rc = rcount();
if (!reached_threshold(rc, hc)) {
return;
}
rc = rcount_.exchange(0, std::memory_order_release);
if (!reached_threshold(rc, hc)) {
/* No need to add rc back to rcount_. At least one concurrent
try_bulk_reclaim will proceed to bulk_reclaim. */
return;
}
bulk_reclaim();
}
void bulk_reclaim(bool transitive = false) {
inc_num_bulk_reclaims();
while (true) {
auto obj = retired_.exchange(nullptr, std::memory_order_acquire);
/*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
auto rec = hazptrs_.load(std::memory_order_acquire);
/* Part 1 - read hazard pointer values into private search structure */
std::unordered_set<const void*> hashset;
for (; rec; rec = rec->next()) {
hashset.insert(rec->hazptr());
}
/* Part 2 - for each retired object, reclaim if no match */
if (bulk_lookup_and_reclaim(obj, hashset) || !transitive) {
break;
}
}
dec_num_bulk_reclaims();
}
bool bulk_lookup_and_reclaim(
hazptr_obj<Atom>* obj, const std::unordered_set<const void*>& hashset) {
hazptr_obj_list<Atom> children;
hazptr_obj_list<Atom> matched;
while (obj) {
auto next = obj->next();
DCHECK_NE(obj, next);
if (hashset.count(obj->raw_ptr()) == 0) {
(*(obj->reclaim()))(obj, children);
} else {
matched.push(obj);
}
obj = next;
}
bool done = ((children.count() == 0) && (retired() == nullptr));
matched.splice(children);
if (matched.count() > 0) {
push_retired(matched, false /* don't call bulk_reclaim recursively */);
}
return done;
}
bool check_sync_time(Atom<uint64_t>& sync_time) {
uint64_t time = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
auto prevtime = sync_time.load(std::memory_order_relaxed);
if (time < prevtime ||
!sync_time.compare_exchange_strong(
prevtime, time + kSyncTimePeriod, std::memory_order_relaxed)) {
return false;
}
return true;
}
bool try_timed_cleanup() {
if (!check_sync_time(sync_time_)) {
return false;
}
relaxed_cleanup(); // calling regular cleanup may self deadlock
return true;
}
hazptr_rec<Atom>* try_acquire_existing_hprec() { hazptr_rec<Atom>* try_acquire_existing_hprec() {
auto rec = head(); auto rec = head();
while (rec) { while (rec) {
...@@ -791,15 +649,6 @@ hazard_pointer_default_domain() { ...@@ -791,15 +649,6 @@ hazard_pointer_default_domain() {
/** hazptr_domain_push_retired: push a list of retired objects into a domain */ /** hazptr_domain_push_retired: push a list of retired objects into a domain */
template <template <typename> class Atom> template <template <typename> class Atom>
void hazptr_domain_push_retired( void hazptr_domain_push_retired(
hazptr_obj_list<Atom>& l,
bool check,
hazptr_domain<Atom>& domain) noexcept {
domain.push_retired(l, check);
}
/** hazptr_domain_push_list */
template <template <typename> class Atom>
void hazptr_domain_push_list(
hazptr_obj_list<Atom>& l, hazptr_domain<Atom>& domain) noexcept { hazptr_obj_list<Atom>& l, hazptr_domain<Atom>& domain) noexcept {
domain.push_list(l); domain.push_list(l);
} }
......
...@@ -146,8 +146,6 @@ class hazptr_obj { ...@@ -146,8 +146,6 @@ class hazptr_obj {
friend class hazptr_domain<Atom>; friend class hazptr_domain<Atom>;
template <typename, template <typename> class, typename> template <typename, template <typename> class, typename>
friend class hazptr_obj_base; friend class hazptr_obj_base;
template <typename, template <typename> class, typename>
friend class hazptr_obj_base_refcounted;
friend class hazptr_obj_cohort<Atom>; friend class hazptr_obj_cohort<Atom>;
Obj* next() const noexcept { return next_; } Obj* next() const noexcept { return next_; }
...@@ -177,7 +175,7 @@ class hazptr_obj { ...@@ -177,7 +175,7 @@ class hazptr_obj {
void push_to_retired(hazptr_domain<Atom>& domain) { void push_to_retired(hazptr_domain<Atom>& domain) {
hazptr_obj_list<Atom> l(this); hazptr_obj_list<Atom> l(this);
hazptr_domain_push_retired(l, true, domain); hazptr_domain_push_retired(l, domain);
} }
FOLLY_NOINLINE void pre_retire_check_fail() noexcept { FOLLY_NOINLINE void pre_retire_check_fail() noexcept {
...@@ -368,7 +366,7 @@ class hazptr_obj_cohort { ...@@ -368,7 +366,7 @@ class hazptr_obj_cohort {
} }
if (!children.empty()) { if (!children.empty()) {
if (active()) { if (active()) {
hazptr_domain_push_list<Atom>(children); hazptr_domain_push_retired<Atom>(children);
} else { } else {
obj = children.head(); obj = children.head();
} }
...@@ -393,7 +391,7 @@ class hazptr_obj_cohort { ...@@ -393,7 +391,7 @@ class hazptr_obj_cohort {
} }
} }
hazptr_obj_list<Atom> l(ll.head(), ll.tail(), c); hazptr_obj_list<Atom> l(ll.head(), ll.tail(), c);
hazptr_domain_push_list<Atom>(l); hazptr_domain_push_retired<Atom>(l);
return; return;
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment