Commit 520148aa authored by Maged Michael's avatar Maged Michael Committed by Facebook Github Bot

Fix destruction order

Summary:
- Added `hazptr.cpp` and `memory_resource.cpp`. Moved singleton code to source.
  - Changed the singleton for the default `hazptr_domain` to a global.
  - Changed hazptr_stats singleton to a global.
  - Moved the thread caching calls from the hazptr_domain functions to the constructor/destructor of hazptr_holder.
  - Changed the TLS singletons to TLS globals.
  - Changed some inlining directives.
  - Leak the hazptr_rec-s in the default domain

Reviewed By: davidtgoldblatt, djwatson

Differential Revision: D5553753

fbshipit-source-id: da69eecec55c0f78fb8ef5591f9aeffee99ff3fa
parent cccde53e
...@@ -98,7 +98,6 @@ inline uint64_t bench(std::string name, int ops, const RepFunc& repFn) { ...@@ -98,7 +98,6 @@ inline uint64_t bench(std::string name, int ops, const RepFunc& repFn) {
inline uint64_t listBench(std::string name, int nthreads, int size) { inline uint64_t listBench(std::string name, int nthreads, int size) {
int ops = 100000; int ops = 100000;
auto repFn = [&] { auto repFn = [&] {
hazptr_holder dummy[100];
SWMRListSet<uint64_t> s; SWMRListSet<uint64_t> s;
auto init = [&] { auto init = [&] {
for (int i = 0; i < size; ++i) { for (int i = 0; i < size; ++i) {
...@@ -119,7 +118,6 @@ inline uint64_t listBench(std::string name, int nthreads, int size) { ...@@ -119,7 +118,6 @@ inline uint64_t listBench(std::string name, int nthreads, int size) {
inline uint64_t holderBench(std::string name, int nthreads) { inline uint64_t holderBench(std::string name, int nthreads) {
int ops = 100000; int ops = 100000;
auto repFn = [&] { auto repFn = [&] {
hazptr_holder dummy[100];
auto init = [] {}; auto init = [] {};
auto fn = [&](int tid) { auto fn = [&](int tid) {
for (int j = tid; j < ops; j += nthreads) { for (int j = tid; j < ops; j += nthreads) {
...@@ -138,7 +136,6 @@ inline uint64_t retireBench(std::string name, int nthreads) { ...@@ -138,7 +136,6 @@ inline uint64_t retireBench(std::string name, int nthreads) {
}; };
int ops = 100000; int ops = 100000;
auto repFn = [&] { auto repFn = [&] {
hazptr_holder dummy[100];
auto init = [] {}; auto init = [] {};
auto fn = [&](int tid) { auto fn = [&](int tid) {
for (int j = tid; j < ops; j += nthreads) { for (int j = tid; j < ops; j += nthreads) {
......
...@@ -53,7 +53,6 @@ class SWMRListSet { ...@@ -53,7 +53,6 @@ class SWMRListSet {
}; };
std::atomic<Node*> head_ = {nullptr}; std::atomic<Node*> head_ = {nullptr};
hazptr_domain& domain_;
/* Used by the single writer */ /* Used by the single writer */
void locate_lower_bound(const T& v, std::atomic<Node*>*& prev) const { void locate_lower_bound(const T& v, std::atomic<Node*>*& prev) const {
...@@ -67,9 +66,6 @@ class SWMRListSet { ...@@ -67,9 +66,6 @@ class SWMRListSet {
} }
public: public:
explicit SWMRListSet(hazptr_domain& domain = default_hazptr_domain())
: domain_(domain) {}
~SWMRListSet() { ~SWMRListSet() {
Node* next; Node* next;
for (auto p = head_.load(); p; p = next) { for (auto p = head_.load(); p; p = next) {
...@@ -97,14 +93,15 @@ class SWMRListSet { ...@@ -97,14 +93,15 @@ class SWMRListSet {
prev->store(curr_next, std::memory_order_release); prev->store(curr_next, std::memory_order_release);
// ...and only then null out the removed node. // ...and only then null out the removed node.
curr->next_.store(nullptr, std::memory_order_release); curr->next_.store(nullptr, std::memory_order_release);
curr->retire(domain_); curr->retire();
return true; return true;
} }
/* Used by readers */ /* Used by readers */
bool contains(const T& val) const { bool contains(const T& val) const {
/* Acquire two hazard pointers for hand-over-hand traversal. */ /* Acquire two hazard pointers for hand-over-hand traversal. */
hazptr_holder hptr_prev(domain_); hazptr_holder hptr_prev;
hazptr_holder hptr_curr(domain_); hazptr_holder hptr_curr;
while (true) { while (true) {
auto prev = &head_; auto prev = &head_;
auto curr = prev->load(std::memory_order_acquire); auto curr = prev->load(std::memory_order_acquire);
......
...@@ -69,16 +69,15 @@ namespace folly { ...@@ -69,16 +69,15 @@ namespace folly {
namespace hazptr { namespace hazptr {
/** /**
* helper classes and functions * Helper classes and functions
*/ */
/** hazptr_stats */ /** hazptr_stats */
class hazptr_stats; class hazptr_stats;
hazptr_stats& hazptr_stats();
#if HAZPTR_STATS #if HAZPTR_STATS
#define INC_HAZPTR_STATS(x) hazptr_stats().x() #define INC_HAZPTR_STATS(x) hazptr_stats_.x()
#else #else
#define INC_HAZPTR_STATS(x) #define INC_HAZPTR_STATS(x)
#endif #endif
...@@ -91,65 +90,97 @@ class hazptr_mb { ...@@ -91,65 +90,97 @@ class hazptr_mb {
static void heavy(); static void heavy();
}; };
/** hazptr_tc /**
* TLS structures
*/
/** TLS life state */
enum hazptr_tls_state { TLS_ALIVE, TLS_UNINITIALIZED, TLS_DESTROYED };
/** hazptr_tc structures
* Thread caching of hazptr_rec-s that belong to the default domain. * Thread caching of hazptr_rec-s that belong to the default domain.
*/ */
class hazptr_tc_entry { struct hazptr_tc_entry {
friend class hazptr_tc; hazptr_rec* hprec_;
hazptr_rec* hprec_{nullptr};
public:
void fill(hazptr_rec* hprec); void fill(hazptr_rec* hprec);
hazptr_rec* get(); hazptr_rec* get();
void evict(); void evict();
}; };
class hazptr_tc { static_assert(
hazptr_tc_entry tc_[HAZPTR_TC_SIZE]; std::is_trivial<hazptr_tc_entry>::value,
int count_{0}; "hazptr_tc_entry must be trivial"
" to avoid a branch to check initialization");
struct hazptr_tc {
hazptr_tc_entry entry_[HAZPTR_TC_SIZE];
int count_;
public: public:
hazptr_tc();
~hazptr_tc();
hazptr_rec* get(); hazptr_rec* get();
bool put(hazptr_rec* hprec); bool put(hazptr_rec* hprec);
}; };
hazptr_tc& hazptr_tc(); static_assert(
std::is_trivial<hazptr_tc>::value,
"hazptr_tc must be trivial to avoid a branch to check initialization");
void hazptr_tc_init();
void hazptr_tc_shutdown();
hazptr_rec* hazptr_tc_try_get();
bool hazptr_tc_try_put(hazptr_rec* hprec);
/** hazptr_priv /** hazptr_priv structures
* Thread private lists of retired objects that belong to the default domain. * Thread private lists of retired objects that belong to the default domain.
*/ */
class hazptr_priv { struct hazptr_priv {
hazptr_domain* domain_{&default_hazptr_domain()}; hazptr_obj* head_;
hazptr_obj* head_{nullptr}; hazptr_obj* tail_;
hazptr_obj* tail_{nullptr}; int rcount_;
int rcount_{0}; bool active_;
bool active_{true};
public:
hazptr_priv();
~hazptr_priv();
void push(hazptr_obj* obj); void push(hazptr_obj* obj);
private:
void pushAllToDomain(); void pushAllToDomain();
}; };
hazptr_priv& hazptr_priv(); static_assert(
std::is_trivial<hazptr_priv>::value,
"hazptr_priv must be trivial to avoid a branch to check initialization");
void hazptr_priv_init();
void hazptr_priv_shutdown();
bool hazptr_priv_try_retire(hazptr_obj* obj);
/** hazptr_tls_life */
struct hazptr_tls_life {
hazptr_tls_life();
~hazptr_tls_life();
};
void tls_life_odr_use();
/** tls globals */
extern thread_local hazptr_tls_state tls_state_;
extern thread_local hazptr_tc tls_tc_data_;
extern thread_local hazptr_priv tls_priv_data_;
extern thread_local hazptr_tls_life tls_life_; // last
/** /**
* public functions * hazptr_domain
*/ */
/** hazptr_domain */ inline constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept
constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept
: mr_(mr) {} : mr_(mr) {}
/** hazptr_obj_base */ /**
* hazptr_obj_base
*/
template <typename T, typename D> template <typename T, typename D>
inline void hazptr_obj_base<T, D>::retire(hazptr_domain& domain, D deleter) { inline void hazptr_obj_base<T, D>::retire(hazptr_domain& domain, D deleter) {
...@@ -162,18 +193,21 @@ inline void hazptr_obj_base<T, D>::retire(hazptr_domain& domain, D deleter) { ...@@ -162,18 +193,21 @@ inline void hazptr_obj_base<T, D>::retire(hazptr_domain& domain, D deleter) {
}; };
if (HAZPTR_PRIV && if (HAZPTR_PRIV &&
(HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) { (HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) {
hazptr_priv().push(this); if (hazptr_priv_try_retire(this)) {
} else { return;
domain.objRetire(this); }
} }
domain.objRetire(this);
} }
/** hazptr_rec */ /**
* hazptr_rec
*/
class hazptr_rec { class hazptr_rec {
friend class hazptr_domain; friend class hazptr_domain;
friend class hazptr_holder; friend class hazptr_holder;
friend class hazptr_tc_entry; friend struct hazptr_tc_entry;
FOLLY_ALIGN_TO_AVOID_FALSE_SHARING FOLLY_ALIGN_TO_AVOID_FALSE_SHARING
std::atomic<const void*> hazptr_{nullptr}; std::atomic<const void*> hazptr_{nullptr};
...@@ -188,16 +222,28 @@ class hazptr_rec { ...@@ -188,16 +222,28 @@ class hazptr_rec {
void release() noexcept; void release() noexcept;
}; };
/** hazptr_holder */ /**
* hazptr_holder
*/
FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_domain& domain) { FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_domain& domain) {
domain_ = &domain; domain_ = &domain;
if (LIKELY(
HAZPTR_TC &&
(HAZPTR_ONE_DOMAIN || &domain == &default_hazptr_domain()))) {
auto hprec = hazptr_tc_try_get();
if (LIKELY(hprec != nullptr)) {
hazptr_ = hprec;
DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
return;
}
}
hazptr_ = domain_->hazptrAcquire(); hazptr_ = domain_->hazptrAcquire();
DEBUG_PRINT(this << " " << domain_ << " " << hazptr_); DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
if (hazptr_ == nullptr) { std::bad_alloc e; throw e; } if (hazptr_ == nullptr) { std::bad_alloc e; throw e; }
} }
inline hazptr_holder::hazptr_holder(std::nullptr_t) { FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(std::nullptr_t) {
domain_ = nullptr; domain_ = nullptr;
hazptr_ = nullptr; hazptr_ = nullptr;
DEBUG_PRINT(this << " " << domain_ << " " << hazptr_); DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
...@@ -205,22 +251,30 @@ inline hazptr_holder::hazptr_holder(std::nullptr_t) { ...@@ -205,22 +251,30 @@ inline hazptr_holder::hazptr_holder(std::nullptr_t) {
FOLLY_ALWAYS_INLINE hazptr_holder::~hazptr_holder() { FOLLY_ALWAYS_INLINE hazptr_holder::~hazptr_holder() {
DEBUG_PRINT(this); DEBUG_PRINT(this);
if (domain_) { if (LIKELY(hazptr_ != nullptr)) {
hazptr_->clear(); hazptr_->clear();
if (LIKELY(
HAZPTR_TC &&
(HAZPTR_ONE_DOMAIN || domain_ == &default_hazptr_domain()))) {
if (LIKELY(hazptr_tc_try_put(hazptr_))) {
return;
}
}
domain_->hazptrRelease(hazptr_); domain_->hazptrRelease(hazptr_);
} }
} }
inline hazptr_holder::hazptr_holder(hazptr_holder&& rhs) noexcept { FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_holder&& rhs) noexcept {
domain_ = rhs.domain_; domain_ = rhs.domain_;
hazptr_ = rhs.hazptr_; hazptr_ = rhs.hazptr_;
rhs.domain_ = nullptr; rhs.domain_ = nullptr;
rhs.hazptr_ = nullptr; rhs.hazptr_ = nullptr;
} }
inline hazptr_holder& hazptr_holder::operator=(hazptr_holder&& rhs) noexcept { FOLLY_ALWAYS_INLINE
hazptr_holder& hazptr_holder::operator=(hazptr_holder&& rhs) noexcept {
/* Self-move is a no-op. */ /* Self-move is a no-op. */
if (this != &rhs) { if (LIKELY(this != &rhs)) {
this->~hazptr_holder(); this->~hazptr_holder();
new (this) hazptr_holder(std::move(rhs)); new (this) hazptr_holder(std::move(rhs));
} }
...@@ -243,7 +297,7 @@ FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect( ...@@ -243,7 +297,7 @@ FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect(
reset(f(ptr)); reset(f(ptr));
/*** Full fence ***/ hazptr_mb::light(); /*** Full fence ***/ hazptr_mb::light();
T* p = src.load(std::memory_order_acquire); T* p = src.load(std::memory_order_acquire);
if (p != ptr) { if (UNLIKELY(p != ptr)) {
ptr = p; ptr = p;
reset(); reset();
return false; return false;
...@@ -303,15 +357,14 @@ FOLLY_ALWAYS_INLINE void swap(hazptr_holder& lhs, hazptr_holder& rhs) noexcept { ...@@ -303,15 +357,14 @@ FOLLY_ALWAYS_INLINE void swap(hazptr_holder& lhs, hazptr_holder& rhs) noexcept {
/** Definition of default_hazptr_domain() */ /** Definition of default_hazptr_domain() */
inline hazptr_domain& default_hazptr_domain() { FOLLY_ALWAYS_INLINE hazptr_domain& default_hazptr_domain() {
static hazptr_domain d; DEBUG_PRINT(&default_domain_);
DEBUG_PRINT(&d); return default_domain_;
return d;
} }
/** hazptr_rec */ /** hazptr_rec */
inline void hazptr_rec::set(const void* p) noexcept { FOLLY_ALWAYS_INLINE void hazptr_rec::set(const void* p) noexcept {
DEBUG_PRINT(this << " " << p); DEBUG_PRINT(this << " " << p);
hazptr_.store(p, std::memory_order_release); hazptr_.store(p, std::memory_order_release);
} }
...@@ -322,7 +375,7 @@ inline const void* hazptr_rec::get() const noexcept { ...@@ -322,7 +375,7 @@ inline const void* hazptr_rec::get() const noexcept {
return p; return p;
} }
inline void hazptr_rec::clear() noexcept { FOLLY_ALWAYS_INLINE void hazptr_rec::clear() noexcept {
DEBUG_PRINT(this); DEBUG_PRINT(this);
hazptr_.store(nullptr, std::memory_order_release); hazptr_.store(nullptr, std::memory_order_release);
} }
...@@ -369,7 +422,11 @@ inline hazptr_domain::~hazptr_domain() { ...@@ -369,7 +422,11 @@ inline hazptr_domain::~hazptr_domain() {
retired = retired_.exchange(nullptr); retired = retired_.exchange(nullptr);
} }
} }
{ /* free all hazptr_rec-s */ /* Leak the data for the default domain to avoid destruction order
* issues with thread caches.
*/
if (this != &default_hazptr_domain()) {
/* free all hazptr_rec-s */
hazptr_rec* next; hazptr_rec* next;
for (auto p = hazptrs_.load(std::memory_order_acquire); p; p = next) { for (auto p = hazptrs_.load(std::memory_order_acquire); p; p = next) {
next = p->next_; next = p->next_;
...@@ -379,13 +436,7 @@ inline hazptr_domain::~hazptr_domain() { ...@@ -379,13 +436,7 @@ inline hazptr_domain::~hazptr_domain() {
} }
} }
FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() { inline hazptr_rec* hazptr_domain::hazptrAcquire() {
if (HAZPTR_TC && (HAZPTR_ONE_DOMAIN || this == &default_hazptr_domain())) {
auto hprec = hazptr_tc().get();
if (hprec) {
return hprec;
}
}
hazptr_rec* p; hazptr_rec* p;
hazptr_rec* next; hazptr_rec* next;
for (p = hazptrs_.load(std::memory_order_acquire); p; p = next) { for (p = hazptrs_.load(std::memory_order_acquire); p; p = next) {
...@@ -395,6 +446,7 @@ FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() { ...@@ -395,6 +446,7 @@ FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() {
} }
} }
p = static_cast<hazptr_rec*>(mr_->allocate(sizeof(hazptr_rec))); p = static_cast<hazptr_rec*>(mr_->allocate(sizeof(hazptr_rec)));
DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec));
if (p == nullptr) { if (p == nullptr) {
return nullptr; return nullptr;
} }
...@@ -408,11 +460,7 @@ FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() { ...@@ -408,11 +460,7 @@ FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() {
return p; return p;
} }
FOLLY_ALWAYS_INLINE void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept { inline void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept {
if (HAZPTR_TC && (HAZPTR_ONE_DOMAIN || this == &default_hazptr_domain()) &&
hazptr_tc().put(p)) {
return;
}
DEBUG_PRINT(this << " " << p); DEBUG_PRINT(this << " " << p);
p->release(); p->release();
} }
...@@ -506,6 +554,8 @@ class hazptr_stats { ...@@ -506,6 +554,8 @@ class hazptr_stats {
std::atomic<uint64_t> seq_cst_{0}; std::atomic<uint64_t> seq_cst_{0};
}; };
extern hazptr_stats hazptr_stats_;
inline hazptr_stats::~hazptr_stats() { inline hazptr_stats::~hazptr_stats() {
DEBUG_PRINT(this << " light " << light_.load()); DEBUG_PRINT(this << " light " << light_.load());
DEBUG_PRINT(this << " heavy " << heavy_.load()); DEBUG_PRINT(this << " heavy " << heavy_.load());
...@@ -530,15 +580,9 @@ inline void hazptr_stats::seq_cst() { ...@@ -530,15 +580,9 @@ inline void hazptr_stats::seq_cst() {
} }
} }
inline class hazptr_stats& hazptr_stats() {
static class hazptr_stats stats_;
DEBUG_PRINT(&stats_);
return stats_;
}
/** hazptr_mb */ /** hazptr_mb */
inline void hazptr_mb::light() { FOLLY_ALWAYS_INLINE void hazptr_mb::light() {
DEBUG_PRINT(""); DEBUG_PRINT("");
if (HAZPTR_AMB) { if (HAZPTR_AMB) {
folly::asymmetricLightBarrier(); folly::asymmetricLightBarrier();
...@@ -560,41 +604,38 @@ inline void hazptr_mb::heavy() { ...@@ -560,41 +604,38 @@ inline void hazptr_mb::heavy() {
} }
} }
/** hazptr_tc - functions */ /**
* TLS structures
*/
/**
* hazptr_tc structures
*/
/** hazptr_tc_entry */
inline void hazptr_tc_entry::fill(hazptr_rec* hprec) { FOLLY_ALWAYS_INLINE void hazptr_tc_entry::fill(hazptr_rec* hprec) {
hprec_ = hprec; hprec_ = hprec;
DEBUG_PRINT(this << " " << hprec); DEBUG_PRINT(this << " " << hprec);
} }
inline hazptr_rec* hazptr_tc_entry::get() { FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_entry::get() {
auto hprec = hprec_; auto hprec = hprec_;
hprec_ = nullptr;
DEBUG_PRINT(this << " " << hprec); DEBUG_PRINT(this << " " << hprec);
return hprec; return hprec;
} }
inline void hazptr_tc_entry::evict() { inline void hazptr_tc_entry::evict() {
auto hprec = hprec_; auto hprec = hprec_;
hprec_ = nullptr;
hprec->release(); hprec->release();
DEBUG_PRINT(this << " " << hprec); DEBUG_PRINT(this << " " << hprec);
} }
inline hazptr_tc::hazptr_tc() { /** hazptr_tc */
DEBUG_PRINT(this);
}
inline hazptr_tc::~hazptr_tc() {
DEBUG_PRINT(this);
for (int i = 0; i < count_; ++i) {
tc_[i].evict();
}
}
inline hazptr_rec* hazptr_tc::get() { FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc::get() {
if (count_ > 0) { if (LIKELY(count_ != 0)) {
auto hprec = tc_[--count_].get(); auto hprec = entry_[--count_].get();
DEBUG_PRINT(this << " " << hprec); DEBUG_PRINT(this << " " << hprec);
return hprec; return hprec;
} }
...@@ -602,66 +643,147 @@ inline hazptr_rec* hazptr_tc::get() { ...@@ -602,66 +643,147 @@ inline hazptr_rec* hazptr_tc::get() {
return nullptr; return nullptr;
} }
inline bool hazptr_tc::put(hazptr_rec* hprec) { FOLLY_ALWAYS_INLINE bool hazptr_tc::put(hazptr_rec* hprec) {
if (count_ < HAZPTR_TC_SIZE) { if (LIKELY(count_ < HAZPTR_TC_SIZE)) {
tc_[count_++].fill(hprec); entry_[count_++].fill(hprec);
DEBUG_PRINT(this << " " << count_ - 1); DEBUG_PRINT(this << " " << count_ - 1);
return true; return true;
} }
return false; return false;
} }
FOLLY_ALWAYS_INLINE class hazptr_tc& hazptr_tc() { /** hazptr_tc free functions */
static thread_local class hazptr_tc tc;
DEBUG_PRINT(&tc); FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_try_get() {
return tc; DEBUG_PRINT(TLS_UNINITIALIZED << TLS_ALIVE << TLS_DESTROYED);
DEBUG_PRINT(tls_state_);
if (LIKELY(tls_state_ == TLS_ALIVE)) {
DEBUG_PRINT(tls_state_);
return tls_tc_data_.get();
} else if (tls_state_ == TLS_UNINITIALIZED) {
tls_life_odr_use();
return tls_tc_data_.get();
}
return nullptr;
} }
/** hazptr_priv - functions */ FOLLY_ALWAYS_INLINE bool hazptr_tc_try_put(hazptr_rec* hprec) {
DEBUG_PRINT(tls_state_);
if (LIKELY(tls_state_ == TLS_ALIVE)) {
DEBUG_PRINT(tls_state_);
return tls_tc_data_.put(hprec);
}
return false;
}
inline hazptr_priv::hazptr_priv() { inline void hazptr_tc_init() {
DEBUG_PRINT(this); DEBUG_PRINT("");
auto& tc = tls_tc_data_;
DEBUG_PRINT(&tc);
tc.count_ = 0;
for (int i = 0; i < HAZPTR_TC_SIZE; ++i) {
tc.entry_[i].hprec_ = nullptr;
}
} }
inline hazptr_priv::~hazptr_priv() { inline void hazptr_tc_shutdown() {
DEBUG_PRINT(this); auto& tc = tls_tc_data_;
DCHECK(active_); DEBUG_PRINT(&tc);
active_ = false; for (int i = 0; i < tc.count_; ++i) {
if (tail_) { tc.entry_[i].evict();
pushAllToDomain();
} }
} }
/**
* hazptr_priv
*/
inline void hazptr_priv::push(hazptr_obj* obj) { inline void hazptr_priv::push(hazptr_obj* obj) {
auto& domain = default_hazptr_domain();
obj->next_ = nullptr; obj->next_ = nullptr;
if (tail_) { if (tail_) {
tail_->next_ = obj; tail_->next_ = obj;
} else { } else {
if (!active_) { if (!active_) {
default_hazptr_domain().objRetire(obj); domain.objRetire(obj);
return; return;
} }
head_ = obj; head_ = obj;
} }
tail_ = obj; tail_ = obj;
++rcount_; ++rcount_;
if (domain_->reachedThreshold(rcount_)) { if (domain.reachedThreshold(rcount_)) {
pushAllToDomain(); pushAllToDomain();
} }
} }
inline void hazptr_priv::pushAllToDomain() { inline void hazptr_priv::pushAllToDomain() {
domain_->pushRetired(head_, tail_, rcount_); auto& domain = default_hazptr_domain();
domain.pushRetired(head_, tail_, rcount_);
head_ = nullptr; head_ = nullptr;
tail_ = nullptr; tail_ = nullptr;
rcount_ = 0; rcount_ = 0;
domain_->tryBulkReclaim(); domain.tryBulkReclaim();
}
inline void hazptr_priv_init() {
auto& priv = tls_priv_data_;
DEBUG_PRINT(&priv);
priv.head_ = nullptr;
priv.tail_ = nullptr;
priv.rcount_ = 0;
priv.active_ = true;
} }
inline class hazptr_priv& hazptr_priv() { inline void hazptr_priv_shutdown() {
static thread_local class hazptr_priv priv; auto& priv = tls_priv_data_;
DEBUG_PRINT(&priv); DEBUG_PRINT(&priv);
return priv; DCHECK(priv.active_);
priv.active_ = false;
if (priv.tail_) {
priv.pushAllToDomain();
}
}
inline bool hazptr_priv_try_retire(hazptr_obj* obj) {
DEBUG_PRINT(tls_state_);
if (tls_state_ == TLS_ALIVE) {
DEBUG_PRINT(tls_state_);
tls_priv_data_.push(obj);
return true;
} else if (tls_state_ == TLS_UNINITIALIZED) {
DEBUG_PRINT(tls_state_);
tls_life_odr_use();
tls_priv_data_.push(obj);
return true;
}
return false;
}
/** hazptr_tls_life */
inline void tls_life_odr_use() {
DEBUG_PRINT(tls_state_);
CHECK(tls_state_ == TLS_UNINITIALIZED);
auto volatile tlsOdrUse = &tls_life_;
CHECK(tlsOdrUse != nullptr);
DEBUG_PRINT(tlsOdrUse);
}
inline hazptr_tls_life::hazptr_tls_life() {
DEBUG_PRINT(this);
CHECK(tls_state_ == TLS_UNINITIALIZED);
hazptr_tc_init();
hazptr_priv_init();
tls_state_ = TLS_ALIVE;
}
inline hazptr_tls_life::~hazptr_tls_life() {
DEBUG_PRINT(this);
CHECK(tls_state_ == TLS_ALIVE);
hazptr_tc_shutdown();
hazptr_priv_shutdown();
tls_state_ = TLS_DESTROYED;
} }
} // namespace folly } // namespace folly
......
/*
* Copyright 2017 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "hazptr.h"
namespace folly {
namespace hazptr {
FOLLY_STATIC_CTOR_PRIORITY_MAX hazptr_domain default_domain_;
hazptr_stats hazptr_stats_;
thread_local hazptr_tls_state tls_state_ = TLS_UNINITIALIZED;
thread_local hazptr_tc tls_tc_data_;
thread_local hazptr_priv tls_priv_data_;
thread_local hazptr_tls_life tls_life_; // last
} // namespace hazptr
} // namespace folly
...@@ -51,7 +51,7 @@ class hazptr_domain { ...@@ -51,7 +51,7 @@ class hazptr_domain {
friend class hazptr_holder; friend class hazptr_holder;
template <typename, typename> template <typename, typename>
friend class hazptr_obj_base; friend class hazptr_obj_base;
friend class hazptr_priv; friend struct hazptr_priv;
memory_resource* mr_; memory_resource* mr_;
std::atomic<hazptr_rec*> hazptrs_ = {nullptr}; std::atomic<hazptr_rec*> hazptrs_ = {nullptr};
...@@ -71,12 +71,14 @@ class hazptr_domain { ...@@ -71,12 +71,14 @@ class hazptr_domain {
/** Get the default hazptr_domain */ /** Get the default hazptr_domain */
hazptr_domain& default_hazptr_domain(); hazptr_domain& default_hazptr_domain();
extern hazptr_domain default_domain_;
/** Definition of hazptr_obj */ /** Definition of hazptr_obj */
class hazptr_obj { class hazptr_obj {
friend class hazptr_domain; friend class hazptr_domain;
template <typename, typename> template <typename, typename>
friend class hazptr_obj_base; friend class hazptr_obj_base;
friend class hazptr_priv; friend struct hazptr_priv;
void (*reclaim_)(hazptr_obj*); void (*reclaim_)(hazptr_obj*);
hazptr_obj* next_; hazptr_obj* next_;
......
/*
* Copyright 2017 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "memory_resource.h"
namespace folly {
namespace hazptr {
memory_resource** default_mr_ptr() {
/* library-local */ static memory_resource* default_mr =
new_delete_resource();
DEBUG_PRINT(&default_mr << " " << default_mr);
return &default_mr;
}
memory_resource* get_default_resource() {
DEBUG_PRINT("");
return *default_mr_ptr();
}
void set_default_resource(memory_resource* mr) {
DEBUG_PRINT("");
*default_mr_ptr() = mr;
}
memory_resource* new_delete_resource() {
class new_delete : public memory_resource {
public:
void* allocate(
const size_t bytes,
const size_t alignment = folly::max_align_v) override {
(void)alignment;
void* p = static_cast<void*>(new char[bytes]);
DEBUG_PRINT(this << " " << p << " " << bytes);
return p;
}
void deallocate(
void* p,
const size_t bytes,
const size_t alignment = folly::max_align_v) override {
(void)alignment;
(void)bytes;
DEBUG_PRINT(p << " " << bytes);
delete[] static_cast<char*>(p);
}
};
static new_delete mr;
return &mr;
}
} // namespace hazptr
} // namespace folly
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
/// std::pmr::memory_resource (C++17) as needed for developing a /// std::pmr::memory_resource (C++17) as needed for developing a
/// hazptr prototype. /// hazptr prototype.
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#include <folly/Portability.h> #include <folly/Portability.h>
#include <memory> #include <memory>
...@@ -44,51 +45,5 @@ memory_resource* get_default_resource(); ...@@ -44,51 +45,5 @@ memory_resource* get_default_resource();
void set_default_resource(memory_resource*); void set_default_resource(memory_resource*);
memory_resource* new_delete_resource(); memory_resource* new_delete_resource();
////////////////////////////////////////////////////////////////////////////////
/// Implementation
////////////////////////////////////////////////////////////////////////////////
inline memory_resource** default_mr_ptr() {
/* library-local */ static memory_resource* default_mr =
new_delete_resource();
DEBUG_PRINT(&default_mr << " " << default_mr);
return &default_mr;
}
inline memory_resource* get_default_resource() {
DEBUG_PRINT("");
return *default_mr_ptr();
}
inline void set_default_resource(memory_resource* mr) {
DEBUG_PRINT("");
*default_mr_ptr() = mr;
}
inline memory_resource* new_delete_resource() {
class new_delete : public memory_resource {
public:
void* allocate(
const size_t bytes,
const size_t alignment = folly::max_align_v) override {
(void)alignment;
void* p = static_cast<void*>(new char[bytes]);
DEBUG_PRINT(this << " " << p << " " << bytes);
return p;
}
void deallocate(
void* p,
const size_t bytes,
const size_t alignment = folly::max_align_v) override {
(void)alignment;
(void)bytes;
DEBUG_PRINT(p << " " << bytes);
delete[] static_cast<char*>(p);
}
};
static new_delete mr;
return &mr;
}
} // namespace folly } // namespace folly
} // namespace hazptr } // namespace hazptr
...@@ -198,12 +198,11 @@ TEST_F(HazptrTest, LIFO) { ...@@ -198,12 +198,11 @@ TEST_F(HazptrTest, LIFO) {
TEST_F(HazptrTest, SWMRLIST) { TEST_F(HazptrTest, SWMRLIST) {
using T = uint64_t; using T = uint64_t;
hazptr_domain custom_domain;
CHECK_GT(FLAGS_num_threads, 0); CHECK_GT(FLAGS_num_threads, 0);
for (int i = 0; i < FLAGS_num_reps; ++i) { for (int i = 0; i < FLAGS_num_reps; ++i) {
DEBUG_PRINT("========== start of rep scope"); DEBUG_PRINT("========== start of rep scope");
SWMRListSet<T> s(custom_domain); SWMRListSet<T> s;
std::vector<std::thread> threads(FLAGS_num_threads); std::vector<std::thread> threads(FLAGS_num_threads);
for (int tid = 0; tid < FLAGS_num_threads; ++tid) { for (int tid = 0; tid < FLAGS_num_threads; ++tid) {
threads[tid] = std::thread([&s, tid]() { threads[tid] = std::thread([&s, tid]() {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment