hazptr: Link counting. Protecting descendants of protected objects. Mutable...

hazptr: Link counting. Protecting descendants of protected objects. Mutable and immutable links. Automatic retirement.

Summary:
The class hazptr_obj_linked includes all the reference counting and automatic retirement mechanisms. It supports:
- Mutable and Immutable links.
- Certain and uncertain removal. With certain removal users call retire. With uncertain users call unlink when an object is unlinked. The library determines automatically when to retire such object.
- E.g., UnboundedQueue uses immutable links with certain removal.
- E.g., ConcurrentHashMap uses mutable links with uncertain removal.

Reviewed By: djwatson

Differential Revision: D7674658

fbshipit-source-id: 948f5c3690367deaa0e2023a2def3bed2c22b9f0

folly/Makefile.am

@@ -471,6 +471,7 @@ nobase_follyinclude_HEADERS = \
 	synchronization/HazptrDomain.h \
 	synchronization/HazptrHolder.h \
 	synchronization/HazptrObj.h \
+	synchronization/HazptrObjLinked.h \
 	synchronization/HazptrRec.h \
 	synchronization/HazptrThrLocal.h \
 	synchronization/LifoSem.h \

folly/synchronization/Hazptr-fwd.h

@@ -50,19 +50,40 @@ class hazptr_rec;
 template <template <typename> class Atom = std::atomic>
 class hazptr_obj;
 
+/** hazptr_obj_list */
+template <template <typename> class Atom = std::atomic>
+class hazptr_obj_list;
+
+/** hazptr_deleter */
+template <typename T, typename D>
+class hazptr_deleter;
+
 /** hazptr_obj_base */
 template <
     typename T,
     template <typename> class Atom = std::atomic,
-    typename Deleter = std::default_delete<T>>
+    typename D = std::default_delete<T>>
 class hazptr_obj_base;
 
-/** hazptr_obj_base_refcounted */
+///
+/// Classes related to link counted objects and automatic retirement.
+/// Defined in HazptrLinked.h
+///
+
+/** hazptr_root */
+template <typename T, template <typename> class Atom = std::atomic>
+class hazptr_root;
+
+/** hazptr_obj_linked */
+template <template <typename> class Atom = std::atomic>
+class hazptr_obj_linked;
+
+/** hazptr_obj_base_linked */
 template <
     typename T,
     template <typename> class Atom = std::atomic,
     typename Deleter = std::default_delete<T>>
-class hazptr_obj_base_refcounted;
+class hazptr_obj_base_linked;
 
 ///
 /// Classes and functions related to thread local structures.
@@ -105,9 +126,7 @@ hazptr_domain<Atom>& default_hazptr_domain();
 /** hazptr_domain_push_retired */
 template <template <typename> class Atom = std::atomic>
 void hazptr_domain_push_retired(
-    hazptr_obj<Atom>* head,
-    hazptr_obj<Atom>* tail,
-    int rcount,
+    hazptr_obj_list<Atom>& l,
     hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) noexcept;
 
 /** hazptr_retire */

folly/synchronization/Hazptr.h

@@ -19,6 +19,7 @@
 #include <folly/synchronization/HazptrDomain.h>
 #include <folly/synchronization/HazptrHolder.h>
 #include <folly/synchronization/HazptrObj.h>
+#include <folly/synchronization/HazptrObjLinked.h>
 #include <folly/synchronization/HazptrRec.h>
 #include <folly/synchronization/HazptrThrLocal.h>
 
@@ -125,6 +126,15 @@
 ///   typically is linear in the number of threads using hazard
 ///   pointers.
 ///
+/// Protecting Linked Structures and Automatic Retirement
+/// -----------------------------------------------------
+/// Hazard pointers provide link counting API to protect linked
+/// structures. It is capable of automatic retirement of objects even
+/// when the removal of objects is uncertain. It also supports
+/// optimizations when links are known to be immutable. All the link
+/// counting features incur no extra overhead for readers.
+/// See HazptrObjLinked.h for more details.
+///
 /// Alternative Safe Reclamation Methods
 /// ------------------------------------
 /// - Locking (exclusive or shared):

folly/synchronization/HazptrDomain.h

@@ -82,10 +82,11 @@ class hazptr_domain {
     };
 
     auto node = new hazptr_retire_node(obj, std::move(reclaim));
-    node->reclaim_ = [](hazptr_obj<Atom>* p) {
+    node->reclaim_ = [](hazptr_obj<Atom>* p, hazptr_obj_list<Atom>&) {
       delete static_cast<hazptr_retire_node*>(p);
     };
-    push_retired(node, node, 1);
+    hazptr_obj_list<Atom> l(node);
+    push_retired(l);
   }
 
   /** cleanup */
@@ -96,9 +97,7 @@ class hazptr_domain {
 
  private:
   friend void hazptr_domain_push_retired<Atom>(
-      hazptr_obj<Atom>*,
-      hazptr_obj<Atom>*,
-      int,
+      hazptr_obj_list<Atom>&,
       hazptr_domain<Atom>&) noexcept;
   friend class hazptr_holder<Atom>;
 #if FOLLY_HAZPTR_THR_LOCAL
@@ -117,22 +116,22 @@ class hazptr_domain {
   }
 
   /** push_retired */
-  void push_retired(hazptr_obj<Atom>* head, hazptr_obj<Atom>* tail, int count) {
+  void push_retired(hazptr_obj_list<Atom>& l, bool check = true) {
     /*** Full fence ***/ asymmetricLightBarrier();
     while (true) {
       auto r = retired();
-      tail->set_next(r);
+      l.tail()->set_next(r);
       if (retired_.compare_exchange_weak(
-              r, head, std::memory_order_release, std::memory_order_acquire)) {
+              r,
+              l.head(),
+              std::memory_order_release,
+              std::memory_order_acquire)) {
         break;
       }
     }
-    rcount_.fetch_add(count, std::memory_order_release);
-    if (try_timed_cleanup()) {
-      return;
-    }
-    if (reached_threshold(rcount(), hcount())) {
-      try_bulk_reclaim();
+    rcount_.fetch_add(l.count(), std::memory_order_release);
+    if (check) {
+      check_cleanup_and_reclaim();
     }
   }
 
@@ -160,12 +159,16 @@ class hazptr_domain {
     auto retired = retired_.exchange(nullptr);
     while (retired) {
       auto obj = retired;
+      hazptr_obj_list<Atom> l;
       while (obj) {
         auto next = obj->next();
         DCHECK(obj != next);
-        (*(obj->reclaim()))(obj);
+        (*(obj->reclaim()))(obj, l);
         obj = next;
       }
+      if (l.count()) {
+        push_retired(l);
+      }
       retired = retired_.exchange(nullptr);
     }
   }
@@ -185,6 +188,15 @@ class hazptr_domain {
     }
   }
 
+  void check_cleanup_and_reclaim() {
+    if (try_timed_cleanup()) {
+      return;
+    }
+    if (reached_threshold(rcount(), hcount())) {
+      try_bulk_reclaim();
+    }
+  }
+
   void relaxed_cleanup() noexcept {
 #if FOLLY_HAZPTR_THR_LOCAL
     hazptr_obj<Atom>* h = nullptr;
@@ -195,11 +207,12 @@ class hazptr_domain {
     }
     if (h) {
       DCHECK(t);
-      push_retired(h, t, 0);
+      hazptr_obj_list<Atom> l(h, t, 0);
+      push_retired(l);
     }
 #endif
     rcount_.store(0, std::memory_order_release);
-    bulk_reclaim();
+    bulk_reclaim(true);
   }
 
   void await_zero_bulk_reclaims() {
@@ -223,45 +236,46 @@ class hazptr_domain {
     bulk_reclaim();
   }
 
-  void bulk_reclaim() {
+  void bulk_reclaim(bool transitive = false) {
     num_bulk_reclaims_.fetch_add(1, std::memory_order_acquire);
-    auto obj = retired_.exchange(nullptr, std::memory_order_acquire);
-    /*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
-    auto rec = hazptrs_.load(std::memory_order_acquire);
-    /* Part 1 - read hazard pointer values into private search structure */
-    std::unordered_set<const void*> hashset; // TOTO: lock-free fixed hash set
-    for (; rec; rec = rec->next()) {
-      hashset.insert(rec->hazptr());
+    while (true) {
+      auto obj = retired_.exchange(nullptr, std::memory_order_acquire);
+      /*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
+      auto rec = hazptrs_.load(std::memory_order_acquire);
+      /* Part 1 - read hazard pointer values into private search structure */
+      std::unordered_set<const void*> hashset; // TOTO: lock-free fixed hash set
+      for (; rec; rec = rec->next()) {
+        hashset.insert(rec->hazptr());
+      }
+      /* Part 2 - for each retired object, reclaim if no match */
+      if (bulk_lookup_and_reclaim(obj, hashset) || !transitive) {
+        break;
+      }
     }
-    /* Part 2 - for each retired object, reclaim if no match */
-    bulk_lookup_and_reclaim(obj, hashset);
     num_bulk_reclaims_.fetch_sub(1, std::memory_order_release);
   }
 
-  void bulk_lookup_and_reclaim(
+  bool bulk_lookup_and_reclaim(
       hazptr_obj<Atom>* obj,
       const std::unordered_set<const void*>& hashset) {
-    int rcount = 0;
-    hazptr_obj<Atom>* head = nullptr;
-    hazptr_obj<Atom>* tail = nullptr;
+    hazptr_obj_list<Atom> children;
+    hazptr_obj_list<Atom> matched;
     while (obj) {
       auto next = obj->next();
       DCHECK_NE(obj, next);
       if (hashset.count(obj->raw_ptr()) == 0) {
-        (*(obj->reclaim()))(obj);
+        (*(obj->reclaim()))(obj, children);
       } else {
-        obj->set_next(head);
-        head = obj;
-        if (tail == nullptr) {
-          tail = obj;
-        }
-        ++rcount;
+        matched.push(obj);
       }
       obj = next;
     }
-    if (tail) {
-      push_retired(head, tail, rcount);
+    bool done = (children.count() == 0);
+    matched.splice(children);
+    if (matched.count() > 0) {
+      push_retired(matched, false /* don't call bulk_reclaim recursively */);
     }
+    return done;
   }
 
   bool try_timed_cleanup() {
@@ -335,13 +349,12 @@ FOLLY_ALWAYS_INLINE hazptr_domain<Atom>& default_hazptr_domain() {
 /** hazptr_domain_push_retired: push a list of retired objects into a domain */
 template <template <typename> class Atom>
 void hazptr_domain_push_retired(
-    hazptr_obj<Atom>* head,
-    hazptr_obj<Atom>* tail,
-    int rcount,
+    hazptr_obj_list<Atom>& l,
     hazptr_domain<Atom>& domain) noexcept {
-  domain.push_retired(head, tail, rcount);
+  domain.push_retired(l);
 }
 
+/** hazptr_retire */
 template <template <typename> class Atom, typename T, typename D>
 FOLLY_ALWAYS_INLINE void hazptr_retire(T* obj, D reclaim) {
   default_hazptr_domain<Atom>().retire(obj, std::move(reclaim));

folly/synchronization/HazptrObj.h

@@ -38,7 +38,18 @@ namespace folly {
  */
 template <template <typename> class Atom>
 class hazptr_obj {
-  using ReclaimFnPtr = void (*)(hazptr_obj*);
+  using ReclaimFnPtr = void (*)(hazptr_obj<Atom>*, hazptr_obj_list<Atom>&);
+
+  template <template <typename> class>
+  friend class hazptr_domain;
+  template <typename, template <typename> class, typename>
+  friend class hazptr_obj_base;
+  template <typename, template <typename> class, typename>
+  friend class hazptr_obj_base_linked;
+  template <template <typename> class>
+  friend class hazptr_obj_list;
+  template <template <typename> class>
+  friend class hazptr_priv;
 
   ReclaimFnPtr reclaim_;
   hazptr_obj<Atom>* next_;
@@ -95,14 +106,15 @@ class hazptr_obj {
     }
   }
 
-  void do_retire(hazptr_domain<Atom>& domain) {
+  void push_to_retired(hazptr_domain<Atom>& domain) {
 #if FOLLY_HAZPTR_THR_LOCAL
     if (&domain == &default_hazptr_domain<Atom>()) {
       hazptr_priv_tls<Atom>().push(this);
       return;
     }
 #endif
-    hazptr_domain_push_retired(this, this, 1, domain);
+    hazptr_obj_list<Atom> l(this);
+    hazptr_domain_push_retired(l, domain);
   }
 
   FOLLY_NOINLINE void pre_retire_check_fail() noexcept {
@@ -111,113 +123,134 @@ class hazptr_obj {
 }; // hazptr_obj
 
 /**
- *  hazptr_obj_base
+ *  hazptr_obj_list
  *
- *  Base template for objects protected by hazard pointers.
+ *  List of hazptr_obj-s.
  */
-template <typename T, template <typename> class Atom, typename D>
-class hazptr_obj_base : public hazptr_obj<Atom> {
-  D deleter_; // TODO: EBO
+template <template <typename> class Atom>
+class hazptr_obj_list {
+  hazptr_obj<Atom>* head_;
+  hazptr_obj<Atom>* tail_;
+  int count_;
 
  public:
-  /* Retire a removed object and pass the responsibility for
-   * reclaiming it to the hazptr library */
-  void retire(
-      D deleter = {},
-      hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) {
-    pre_retire(std::move(deleter));
-    set_reclaim();
-    this->do_retire(domain); // defined in hazptr_obj
+  hazptr_obj_list() noexcept : head_(nullptr), tail_(nullptr), count_(0) {}
+
+  explicit hazptr_obj_list(hazptr_obj<Atom>* obj) noexcept
+      : head_(obj), tail_(obj), count_(1) {}
+
+  explicit hazptr_obj_list(
+      hazptr_obj<Atom>* head,
+      hazptr_obj<Atom>* tail,
+      int count) noexcept
+      : head_(head), tail_(tail), count_(count) {}
+
+  hazptr_obj<Atom>* head() {
+    return head_;
   }
 
-  void retire(hazptr_domain<Atom>& domain) {
-    retire({}, domain);
+  hazptr_obj<Atom>* tail() {
+    return tail_;
   }
 
- private:
-  void pre_retire(D deleter) {
-    this->pre_retire_check(); // defined in hazptr_obj
-    deleter_ = std::move(deleter);
+  int count() {
+    return count_;
   }
 
-  void set_reclaim() {
-    this->reclaim_ = [](hazptr_obj<Atom>* p) {
-      auto hobp = static_cast<hazptr_obj_base<T, Atom, D>*>(p);
-      auto obj = static_cast<T*>(hobp);
-      hobp->deleter_(obj);
-    };
+  void push(hazptr_obj<Atom>* obj) {
+    obj->set_next(head_);
+    head_ = obj;
+    if (tail_ == nullptr) {
+      tail_ = obj;
+    }
+    ++count_;
   }
-}; // hazptr_obj_base
+
+  void splice(hazptr_obj_list<Atom>& l) {
+    if (l.count() == 0) {
+      return;
+    }
+    if (count() == 0) {
+      head_ = l.head();
+    } else {
+      tail_->set_next(l.head());
+    }
+    tail_ = l.tail();
+    count_ += l.count();
+    l.clear();
+  }
+
+  void clear() {
+    head_ = nullptr;
+    tail_ = nullptr;
+    count_ = 0;
+  }
+}; // hazptr_obj_list
 
 /**
- *  hazptr_obj_base_refcounted
+ *  hazptr_deleter
  *
- *  Base template for reference counted objects protected by hazard
- *  pointers.
+ *  For empty base optimization.
  */
-template <typename T, template <typename> class Atom, typename D>
-class hazptr_obj_base_refcounted : public hazptr_obj<Atom> {
-  Atom<uint32_t> refcount_{0};
+template <typename T, typename D>
+class hazptr_deleter {
   D deleter_;
 
+ public:
+  void set_deleter(D d = {}) {
+    deleter_ = std::move(d);
+  }
+
+  void delete_obj(T* p) {
+    deleter_(p);
+  }
+};
+
+template <typename T>
+class hazptr_deleter<T, std::default_delete<T>> {
+ public:
+  void set_deleter(std::default_delete<T> = {}) {}
+
+  void delete_obj(T* p) {
+    delete p;
+  }
+};
+
+/**
+ *  hazptr_obj_base
+ *
+ *  Base template for objects protected by hazard pointers.
+ */
+template <typename T, template <typename> class Atom, typename D>
+class hazptr_obj_base : public hazptr_obj<Atom>, public hazptr_deleter<T, D> {
  public:
   /* Retire a removed object and pass the responsibility for
    * reclaiming it to the hazptr library */
   void retire(
       D deleter = {},
       hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) {
-    this->pre_retire(std::move(deleter)); // defined in hazptr_obj
+    pre_retire(std::move(deleter));
     set_reclaim();
-    this->do_retire(domain); // defined in hazptr_obj
+    this->push_to_retired(domain); // defined in hazptr_obj
   }
 
   void retire(hazptr_domain<Atom>& domain) {
     retire({}, domain);
   }
 
-  /* Increments the reference count. */
-  void acquire_ref() noexcept {
-    refcount_.fetch_add(1u, std::memory_order_acq_rel);
-  }
-
-  /* The same as acquire_ref() except that in addition the caller
-   * guarantees that the call is made in a thread-safe context, e.g.,
-   * the object is not yet shared. This is just an optimization to
-   * save an atomic read-modify-write operation. */
-  void acquire_ref_safe() noexcept {
-    auto oldval = refcount_.load(std::memory_order_acquire);
-    refcount_.store(oldval + 1u, std::memory_order_release);
-  }
-
-  /* Decrements the reference count and returns true if the object is
-   * safe to reclaim. */
-  bool release_ref() noexcept {
-    auto oldval = refcount_.load(std::memory_order_acquire);
-    if (oldval > 0u) {
-      oldval = refcount_.fetch_sub(1u, std::memory_order_acq_rel);
-    } else {
-      if (kIsDebug) {
-        refcount_.store(~0u);
-      }
-    }
-    return oldval == 0;
-  }
-
  private:
   void pre_retire(D deleter) {
     this->pre_retire_check(); // defined in hazptr_obj
-    deleter_ = std::move(deleter);
+    this->set_deleter(std::move(deleter));
   }
 
   void set_reclaim() {
-    this->reclaim_ = [](hazptr_obj<Atom>* p) {
-      auto hrobp = static_cast<hazptr_obj_base_refcounted<T, Atom, D>*>(p);
-      if (hrobp->release_ref()) {
-        auto obj = static_cast<T*>(hrobp);
-        hrobp->deleter_(obj);
-      }
+    this->reclaim_ = [](hazptr_obj<Atom>* p, hazptr_obj_list<Atom>&) {
+      auto hobp = static_cast<hazptr_obj_base<T, Atom, D>*>(p);
+      auto obj = static_cast<T*>(hobp);
+      hobp->delete_obj(obj);
     };
   }
-}; // hazptr_obj_base_refcounted
+}; // hazptr_obj_base
 
 } // namespace folly

folly/synchronization/HazptrObjLinked.h

+/*
+ * Copyright 2018-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <folly/synchronization/Hazptr-fwd.h>
+#include <folly/synchronization/HazptrObj.h>
+
+#include <glog/logging.h>
+
+#include <atomic>
+#include <stack>
+
+///
+/// Classes related to link counted objects and automatic retirement.
+///
+
+namespace folly {
+
+/**
+ *  hazptr_root
+ *
+ *  Link to counted objects. When destroyed unlinks the linked object
+ *  if any.
+ *
+ *  Template parameter T must support a member function unlink(),
+ *  inherited from hazptr_obj_base_linked.
+ *
+ *  Use example: Bucket heads in ConcurrentHashMap.
+ */
+template <typename T, template <typename> class Atom>
+class hazptr_root {
+  Atom<T*> link_;
+
+ public:
+  explicit hazptr_root(T* p = nullptr) noexcept : link_(p) {}
+
+  ~hazptr_root() {
+    auto p = link_.load(std::memory_order_relaxed);
+    if (p) {
+      p->unlink();
+    }
+  }
+
+  const Atom<T*>& operator()() const noexcept {
+    return link_;
+  }
+
+  Atom<T*>& operator()() noexcept {
+    return link_;
+  }
+}; // hazptr_root
+
+/**
+ *  hazptr_obj_linked
+ *
+ *  Base class template for link counted objects.
+ *  Supports:
+ *  - Protecting descendants of protected objects.
+ *  - One-pass reclamation of long immutable chains of objects.
+ *  - Automatic reclamation of acyclic structures.
+ *
+ *  Two inbound link counts are maintained per object:
+ *  - Link count: Represents the number of links from mutable paths.
+ *  - Ref count: Represents the number of links from immutable paths.
+ *     [Note: The ref count is one less than such links plus one if
+ *     the object hasn't gone through matching with hazard pointers
+ *     without finding a match. That is, a new object without inbound
+ *     links has a ref count of 0 and an about-to-be-reclaimed object
+ *     can be viewed to have a ref count of -1.]
+ *
+ *  User code can increment the link and ref counts by calling
+ *  acquire_link and acquire_ref or their variants that require the
+ *  user to guarantee thread safety. There are no public functions to
+ *  decrement the counts explicitly. Counts are decremented implicitly
+ *  as described in hazptr_obj_base_linked.
+ */
+template <template <typename> class Atom>
+class hazptr_obj_linked : public hazptr_obj<Atom> {
+  using Count = uint32_t;
+
+  static constexpr Count kRef = 1u;
+  static constexpr Count kLink = 1u << 16;
+  static constexpr Count kRefMask = kLink - 1u;
+  static constexpr Count kLinkMask = ~kRefMask;
+
+  Atom<Count> count_{0};
+
+ public:
+  void acquire_link() noexcept {
+    count_inc(kLink);
+  }
+
+  void acquire_link_safe() noexcept {
+    count_inc_safe(kLink);
+  }
+
+  void acquire_ref() noexcept {
+    count_inc(kRef);
+  }
+
+  void acquire_ref_safe() noexcept {
+    count_inc_safe(kRef);
+  }
+
+ private:
+  template <typename, template <typename> class, typename>
+  friend class hazptr_obj_base_linked;
+
+  void count_inc(Count add) {
+    auto oldval = count_.fetch_add(add, std::memory_order_acq_rel);
+    DCHECK_LT(oldval & kLinkMask, kLinkMask);
+    DCHECK_LT(oldval & kRefMask, kRefMask);
+  }
+
+  void count_inc_safe(Count add) {
+    auto oldval = count_.load(std::memory_order_relaxed);
+    count_.store(oldval + add, std::memory_order_release);
+    DCHECK_LT(oldval & kLinkMask, kLinkMask);
+    DCHECK_LT(oldval & kRefMask, kRefMask);
+  }
+
+  bool count_cas(Count& oldval, Count newval) {
+    return count_.compare_exchange_weak(
+        oldval, newval, std::memory_order_acq_rel, std::memory_order_acquire);
+  }
+
+  bool release_link() noexcept {
+    auto sub = kLink;
+    auto oldval = count_.load(std::memory_order_acquire);
+    while (true) {
+      DCHECK_GT(oldval & kLinkMask, 0u);
+      if (oldval == kLink) {
+        count_.store(0u, std::memory_order_release);
+        return true;
+      }
+      if (count_cas(oldval, oldval - sub)) {
+        return false;
+      }
+    }
+  }
+
+  bool release_ref() noexcept {
+    auto sub = kRef;
+    auto oldval = count_.load(std::memory_order_acquire);
+    while (true) {
+      if (oldval == 0u) {
+        if (kIsDebug) {
+          count_.store(kRefMask);
+        }
+        return true;
+      }
+      DCHECK_GT(oldval & kRefMask, 0u);
+      if (count_cas(oldval, oldval - sub)) {
+        return false;
+      }
+    }
+  }
+
+  bool downgrade_link() noexcept {
+    auto oldval = count_.load(std::memory_order_acquire);
+    auto sub = kLink - kRef;
+    while (true) {
+      if (oldval == kLink) {
+        count_.store(kRef, std::memory_order_release);
+        return true;
+      }
+      if (count_cas(oldval, oldval - sub)) {
+        return (oldval & kLinkMask) == kLink;
+      }
+    }
+  }
+}; // hazptr_obj_linked
+
+/**
+ *  hazptr_obj_base_linked
+ *
+ *  Base class template for link counted objects.
+ *
+ *  Supports both *explicit* and *implicit* object retirement, depending
+ *  on whether object removal is *certain* or *uncertain*.
+ *
+ *  A derived object's removal is certain when it is always possible
+ *  to reason based only on the local state of user code when an
+ *  object is removed, i.e., becomes unreachable from static
+ *  roots. Otherwise, removal is uncertain.
+ *
+ *  For example, Removal in UnboundedQueue is certain, whereas removal
+ *  is ConcurrentHashMap is uncertain.
+ *
+ *  If removal is certain, user code can call retire() explicitly.
+ *  Otherwise, user code should call unlink() whenever an an inbound
+ *  link to the object is changed. Calls to unlink() automatically
+ *  retire the object when the link count is decremented to 0. [Note:
+ *  A ref count greater than 0 does not delay retiring an object.]
+ *
+ *  Derived type T must define a member function template
+ *    template <typename S>
+ *    void push_links(bool m, S& s) {
+ *      if (m) { // m stands mutable links
+ *        // for each outbound mutable pointer p call
+ *        //   s.push(p);
+ *      } else {
+ *        // for each outbound immutable pointer p call
+ *        //   s.push(p);
+ *      }
+ *   }
+ *
+ *   T may have both, either, or none of the tow types of outbound
+ *   links. For example, UnboundedQueue Segment has an immutable
+ *   link, and ConcurrentHashMap NodeT has a mutable link.
+ */
+template <typename T, template <typename> class Atom, typename D>
+class hazptr_obj_base_linked : public hazptr_obj_linked<Atom>,
+                               public hazptr_deleter<T, D> {
+  using Stack = std::stack<hazptr_obj_base_linked<T, Atom, D>*>;
+
+ public:
+  void retire() {
+    this->pre_retire_check(); // defined in hazptr_obj
+    set_reclaim();
+    this->push_to_retired(
+        default_hazptr_domain<Atom>()); // defined in hazptr_obj
+  }
+
+  /* unlink: Retire object if last link is released. */
+  void unlink() {
+    if (this->release_link()) { // defined in hazptr_obj_linked
+      downgrade_retire_immutable_descendants();
+      retire();
+    }
+  }
+
+ private:
+  void set_reclaim() noexcept {
+    this->reclaim_ = [](hazptr_obj<Atom>* p, hazptr_obj_list<Atom>& l) {
+      auto obj = static_cast<hazptr_obj_base_linked<T, Atom, D>*>(p);
+      if (obj->release_ref()) { // defined in hazptr_obj_linked
+        obj->release_delete_immutable_descendants();
+        obj->release_retire_mutable_children(l);
+        obj->delete_self();
+      }
+    };
+  }
+
+  void downgrade_retire_immutable_descendants() {
+    Stack s;
+    call_push_links(false, s);
+    while (!s.empty()) {
+      auto p = s.top();
+      s.pop();
+      if (p && p->downgrade_link()) {
+        p->call_push_links(false, s);
+        p->retire();
+      }
+    }
+  }
+
+  void release_delete_immutable_descendants() {
+    Stack s;
+    call_push_links(false, s);
+    while (!s.empty()) {
+      auto p = s.top();
+      s.pop();
+      if (p && p->release_ref()) {
+        p->call_push_links(false, s);
+        p->delete_self();
+      }
+    }
+  }
+
+  void release_retire_mutable_children(hazptr_obj_list<Atom>& l) {
+    Stack s;
+    call_push_links(true, s);
+    while (!s.empty()) {
+      auto p = s.top();
+      s.pop();
+      if (p->release_link()) {
+        p->pre_retire_check(); // defined in hazptr_obj
+        p->set_reclaim();
+        l.push(p); // treated as if retired immediately
+      }
+    }
+  }
+
+  void call_push_links(bool m, Stack& s) {
+    static_cast<T*>(this)->push_links(m, s); // to be defined in T
+  }
+
+  void delete_self() {
+    this->delete_obj(static_cast<T*>(this)); // defined in hazptr_deleter
+  }
+}; // hazptr_obj_base_linked
+
+} // namespace folly

folly/synchronization/HazptrThrLocal.h

@@ -207,7 +207,8 @@ class hazptr_priv {
     collect(h, t);
     if (h) {
       DCHECK(t);
-      hazptr_domain_push_retired<Atom>(h, t, rcount_);
+      hazptr_obj_list<Atom> l(h, t, rcount_);
+      hazptr_domain_push_retired<Atom>(l);
       rcount_ = 0;
     }
   }

folly/synchronization/test/HazptrTest.cpp

@@ -38,8 +38,9 @@ using folly::hazptr_domain;
 using folly::hazptr_holder;
 using folly::hazptr_local;
 using folly::hazptr_obj_base;
-using folly::hazptr_obj_base_refcounted;
+using folly::hazptr_obj_base_linked;
 using folly::hazptr_retire;
+using folly::hazptr_root;
 using folly::hazptr_tc;
 using folly::HazptrLockFreeLIFO;
 using folly::HazptrSWMRSet;
@@ -50,6 +51,7 @@ using DSched = folly::test::DeterministicSchedule;
 
 // Structures
 
+/** Count */
 class Count {
   std::atomic<int> ctors_{0};
   std::atomic<int> dtors_{0};
@@ -85,17 +87,19 @@ class Count {
   void inc_retires() noexcept {
     retires_.fetch_add(1);
   }
-};
+}; // Count
 
 static Count c_;
 
+/** Node */
 template <template <typename> class Atom = std::atomic>
 class Node : public hazptr_obj_base<Node<Atom>, Atom> {
   int val_;
   Atom<Node<Atom>*> next_;
 
  public:
-  explicit Node(int v = 0, Node* n = nullptr) noexcept : val_(v), next_(n) {
+  explicit Node(int v = 0, Node* n = nullptr, bool = false) noexcept
+      : val_(v), next_(n) {
     c_.inc_ctors();
   }
 
@@ -114,46 +118,52 @@ class Node : public hazptr_obj_base<Node<Atom>, Atom> {
   Atom<Node<Atom>*>* ptr_next() noexcept {
     return &next_;
   }
-};
+}; // Node
 
-template <template <typename> class Atom = std::atomic>
-class NodeRC : public hazptr_obj_base_refcounted<NodeRC<Atom>, Atom> {
+/** NodeRC */
+template <bool Mutable, template <typename> class Atom = std::atomic>
+class NodeRC : public hazptr_obj_base_linked<NodeRC<Mutable, Atom>, Atom> {
+  Atom<NodeRC<Mutable, Atom>*> next_;
   int val_;
-  Atom<NodeRC<Atom>*> next_;
-  bool marked_;
 
  public:
-  explicit NodeRC(int v = 0, NodeRC* n = nullptr) noexcept
-      : val_(v), next_(n), marked_(false) {
+  explicit NodeRC(int v = 0, NodeRC* n = nullptr, bool acq = false) noexcept
+      : next_(n), val_(v) {
+    this->set_deleter();
     c_.inc_ctors();
-    this->acquire_ref_safe();
+    if (acq) {
+      if (Mutable) {
+        this->acquire_link_safe();
+      } else {
+        this->acquire_ref_safe();
+      }
+    }
   }
 
   ~NodeRC() {
     c_.inc_dtors();
-    if (!marked_) {
-      auto n = next();
-      while (n) {
-        if (!n->release_ref()) {
-          return;
-        }
-        auto p = n;
-        n = p->next();
-        p->marked_ = true;
-        delete p;
-      }
-    }
   }
 
   int value() const noexcept {
     return val_;
   }
 
-  NodeRC<Atom>* next() const noexcept {
+  NodeRC<Mutable, Atom>* next() const noexcept {
     return next_.load(std::memory_order_acquire);
   }
-};
 
+  template <typename S>
+  void push_links(bool m, S& s) {
+    if (Mutable == m) {
+      auto p = next();
+      if (p) {
+        s.push(p);
+      }
+    }
+  }
+}; // NodeRC
+
+/** List */
 template <typename T, template <typename> class Atom = std::atomic>
 struct List {
   Atom<T*> head_{nullptr};
@@ -161,9 +171,10 @@ struct List {
  public:
   explicit List(int size) {
     auto p = head_.load(std::memory_order_relaxed);
-    for (int i = 0; i < size; ++i) {
-      p = new T(i + 10000, p);
+    for (int i = 0; i < size - 1; ++i) {
+      p = new T(i + 10000, p, true);
     }
+    p = new T(size + 9999, p);
     head_.store(p, std::memory_order_relaxed);
   }
 
@@ -225,7 +236,41 @@ struct List {
     hazptr_local<1, Atom> hptr;
     return protect_all(val, hptr[0]);
   }
-};
+}; // NodeRC
+
+/** NodeAuto */
+template <template <typename> class Atom = std::atomic>
+class NodeAuto : public hazptr_obj_base_linked<NodeAuto<Atom>, Atom> {
+  Atom<NodeAuto<Atom>*> link_[2];
+
+ public:
+  explicit NodeAuto(NodeAuto* l1 = nullptr, NodeAuto* l2 = nullptr) noexcept {
+    this->set_deleter();
+    link_[0].store(l1, std::memory_order_relaxed);
+    link_[1].store(l2, std::memory_order_relaxed);
+    c_.inc_ctors();
+  }
+
+  ~NodeAuto() {
+    c_.inc_dtors();
+  }
+
+  NodeAuto<Atom>* link(size_t i) {
+    return link_[i].load(std::memory_order_acquire);
+  }
+
+  template <typename S>
+  void push_links(bool m, S& s) {
+    if (m == false) { // Immutable
+      for (int i = 0; i < 2; ++i) {
+        auto p = link(i);
+        if (p) {
+          s.push(p);
+        }
+      }
+    }
+  }
+}; // NodeAuto
 
 // Test Functions
 
@@ -240,19 +285,20 @@ void basic_objects_test() {
   }
   {
     ++num;
-    auto obj = new NodeRC<Atom>;
-    obj->release_ref();
+    auto obj = new NodeRC<false, Atom>(0, nullptr);
     obj->retire();
   }
   {
     ++num;
-    auto obj = new NodeRC<Atom>;
-    obj->acquire_ref();
-    obj->acquire_ref_safe();
-    obj->release_ref();
-    obj->retire();
-    obj->release_ref();
-    obj->release_ref();
+    auto obj = new NodeRC<false, Atom>(0, nullptr);
+    obj->acquire_link_safe();
+    obj->unlink();
+  }
+  {
+    ++num;
+    auto obj = new NodeRC<false, Atom>(0, nullptr);
+    obj->acquire_link_safe();
+    hazptr_root<NodeRC<false, Atom>> root(obj);
   }
   ASSERT_EQ(c_.ctors(), num);
   hazptr_cleanup<Atom>();
@@ -442,18 +488,21 @@ void local_test() {
   hazptr_cleanup<Atom>();
 }
 
-template <template <typename> class Atom = std::atomic>
-void refcount_test() {
+template <bool Mutable, template <typename> class Atom = std::atomic>
+void linked_test() {
   c_.clear();
-  NodeRC<Atom>* p = nullptr;
+  NodeRC<Mutable, Atom>* p = nullptr;
   int num = 193;
-  for (int i = 0; i < num; ++i) {
-    p = new NodeRC<Atom>(i, p);
+  for (int i = 0; i < num - 1; ++i) {
+    p = new NodeRC<Mutable, Atom>(i, p, true);
   }
+  p = new NodeRC<Mutable, Atom>(num - 1, p, Mutable);
   hazptr_holder<Atom> hptr;
   hptr.reset(p);
-  for (auto q = p->next(); q; q = q->next()) {
-    q->retire();
+  if (!Mutable) {
+    for (auto q = p->next(); q; q = q->next()) {
+      q->retire();
+    }
   }
   int v = num;
   for (auto q = p; q; q = q->next()) {
@@ -462,12 +511,15 @@ void refcount_test() {
     ASSERT_EQ(q->value(), v);
   }
 
-  ASSERT_TRUE(!p->release_ref());
   hazptr_cleanup<Atom>();
   ASSERT_EQ(c_.ctors(), num);
   ASSERT_EQ(c_.dtors(), 0);
 
-  p->retire();
+  if (Mutable) {
+    hazptr_root<NodeRC<Mutable, Atom>, Atom> root(p);
+  } else {
+    p->retire();
+  }
   hazptr_cleanup<Atom>();
   ASSERT_EQ(c_.dtors(), 0);
 
@@ -476,16 +528,16 @@ void refcount_test() {
   ASSERT_EQ(c_.dtors(), num);
 }
 
-template <template <typename> class Atom = std::atomic>
-void mt_refcount_test() {
+template <bool Mutable, template <typename> class Atom = std::atomic>
+void mt_linked_test() {
   c_.clear();
 
   Atom<bool> ready(false);
+  Atom<bool> done(false);
   Atom<int> setHazptrs(0);
-  Atom<NodeRC<Atom>*> head;
+  hazptr_root<NodeRC<Mutable, Atom>, Atom> head;
 
   int num = FLAGS_num_ops;
-  ;
   int nthr = FLAGS_num_threads;
   ASSERT_GT(FLAGS_num_threads, 0);
   std::vector<std::thread> thr(nthr);
@@ -495,7 +547,7 @@ void mt_refcount_test() {
         /* spin */
       }
       hazptr_holder<Atom> hptr;
-      auto p = hptr.get_protected(head);
+      auto p = hptr.get_protected(head());
       ++setHazptrs;
       /* Concurrent with removal */
       int v = num;
@@ -505,30 +557,35 @@ void mt_refcount_test() {
         ASSERT_EQ(q->value(), v);
       }
       ASSERT_EQ(v, 0);
+      while (!done.load()) {
+        /* spin */
+      }
     });
   }
 
-  NodeRC<Atom>* p = nullptr;
-  for (int i = 0; i < num; ++i) {
-    p = new NodeRC<Atom>(i, p);
+  NodeRC<Mutable, Atom>* p = nullptr;
+  for (int i = 0; i < num - 1; ++i) {
+    p = new NodeRC<Mutable, Atom>(i, p, true);
   }
+  p = new NodeRC<Mutable, Atom>(num - 1, p, Mutable);
   ASSERT_EQ(c_.ctors(), num);
-  head.store(p);
+  head().store(p);
   ready.store(true);
   while (setHazptrs.load() < nthr) {
     /* spin */
   }
 
-  /* this is concurrent with traversal by reader */
-  head.store(nullptr);
-  for (auto q = p; q; q = q->next()) {
-    q->retire();
+  /* this is concurrent with traversal by readers */
+  head().store(nullptr);
+  if (Mutable) {
+    p->unlink();
+  } else {
+    for (auto q = p; q; q = q->next()) {
+      q->retire();
+    }
   }
   ASSERT_EQ(c_.dtors(), 0);
-
-  if (p->release_ref()) {
-    delete p;
-  }
+  done.store(true);
 
   for (auto& t : thr) {
     DSched::join(t);
@@ -538,6 +595,96 @@ void mt_refcount_test() {
   ASSERT_EQ(c_.dtors(), num);
 }
 
+template <template <typename> class Atom = std::atomic>
+void auto_retire_test() {
+  c_.clear();
+  auto d = new NodeAuto<Atom>;
+  d->acquire_link_safe();
+  auto c = new NodeAuto<Atom>(d);
+  d->acquire_link_safe();
+  auto b = new NodeAuto<Atom>(d);
+  c->acquire_link_safe();
+  b->acquire_link_safe();
+  auto a = new NodeAuto<Atom>(b, c);
+  hazptr_holder<Atom> h;
+  {
+    hazptr_root<NodeAuto<Atom>> root;
+    a->acquire_link_safe();
+    root().store(a);
+    ASSERT_EQ(c_.ctors(), 4);
+    /* So far the links and link counts are:
+           root-->a  a-->b  a-->c  b-->d  c-->d
+           a(1,0) b(1,0) c(1,0) d(2,0)
+    */
+    h.reset(c); /* h protects c */
+    hazptr_cleanup<Atom>();
+    ASSERT_EQ(c_.dtors(), 0);
+    /* Nothing is retired or reclaimed yet */
+  }
+  /* root dtor calls a->unlink, which calls a->release_link, which
+     changes a's link counts from (1,0) to (0,0), which triggers calls
+     to c->downgrade_link, b->downgrade_link, and a->retire.
+
+     c->downgrade_link changes c's link counts from (1,0) to (0,1),
+     which triggers calls to d->downgrade_link and c->retire.
+
+     d->downgrade_link changes d's link counts from (2,0) to (1,1).
+
+     b->downgrade_link changes b's link counts from (1,0) to (0,1),
+     which triggers calls to d->downgrade_link and b->retire.
+
+     d->downgrade_link changes d's link counts from (1,1) to (0,2),
+     which triggers a call to d->retire.
+
+     So far (assuming retire-s did not trigger bulk_reclaim):
+           a-->b  a-->c  b-->d  c-->d
+           a(0,0) b(0,1) c(0,1) d(0,2)
+           Retired: a b c d
+           Protected: c
+  */
+  hazptr_cleanup<Atom>();
+  /* hazptr_cleanup calls bulk_reclaim which finds a, b, and d
+     unprotected, which triggers calls to a->release_ref,
+     b->release_ref, and d->release_ref (not necessarily in that
+     order).
+
+     a->release_ref finds a's link counts to be (0,0), which triggers
+     calls to c->release_ref, b->release_ref and delete a.
+
+     The call to c->release_ref changes its link counts from (0,1) to
+     (0,0).
+
+     The first call to b->release_ref changes b's link counts to
+     (0,0). The second call finds the link counts to be (0,0), which
+     triggers a call to d->release_ref and delete b.
+
+     The first call to d->release_ref changes its link counts to
+     (0,1), and the second call changes them to (0,0);
+
+     So far:
+           c-->d
+           a(deleted) b(deleted) c(0,0) d(0,0)
+           Retired and protected: c
+           bulk_reclamed-ed (i.e, found not protected): d
+  */
+  ASSERT_EQ(c_.dtors(), 2);
+  h.reset(); /* c is now no longer protected */
+  hazptr_cleanup<Atom>();
+  /* hazptr_cleanup calls bulk_reclaim which finds c unprotected,
+     which triggers a call to c->release_ref.
+
+     c->release_ref finds c's link counts to be (0,0), which
+     triggers calls to d->release_ref and delete c.
+
+     d->release_ref finds d's link counts to be (0,0), which triggers
+     a call to delete d.
+
+     Finally:
+           a(deleted) b(deleted) c(deleted) d(deleted)
+  */
+  ASSERT_EQ(c_.dtors(), 4);
+}
+
 template <template <typename> class Atom = std::atomic>
 void free_function_retire_test() {
   auto foo = new int;
@@ -807,22 +954,49 @@ TEST(HazptrTest, dsched_local) {
   local_test<DeterministicAtomic>();
 }
 
-TEST(HazptrTest, refcount) {
-  refcount_test();
+TEST(HazptrTest, linked_mutable) {
+  linked_test<true>();
+}
+
+TEST(HazptrTest, dsched_linked_mutable) {
+  DSched sched(DSched::uniform(0));
+  linked_test<true, DeterministicAtomic>();
+}
+
+TEST(HazptrTest, linked_immutable) {
+  linked_test<false>();
+}
+
+TEST(HazptrTest, dsched_linked_immutable) {
+  DSched sched(DSched::uniform(0));
+  linked_test<false, DeterministicAtomic>();
+}
+
+TEST(HazptrTest, mt_linked_mutable) {
+  mt_linked_test<true>();
+}
+
+TEST(HazptrTest, dsched_mt_linked_mutable) {
+  DSched sched(DSched::uniform(0));
+  mt_linked_test<true, DeterministicAtomic>();
+}
+
+TEST(HazptrTest, mt_linked_immutable) {
+  mt_linked_test<false>();
 }
 
-TEST(HazptrTest, dsched_refcount) {
+TEST(HazptrTest, dsched_mt_linked_immutable) {
   DSched sched(DSched::uniform(0));
-  refcount_test<DeterministicAtomic>();
+  mt_linked_test<false, DeterministicAtomic>();
 }
 
-TEST(HazptrTest, mt_refcount) {
-  mt_refcount_test();
+TEST(HazptrTest, auto_retire) {
+  auto_retire_test();
 }
 
-TEST(HazptrTest, dsched_mt_refcount) {
+TEST(HazptrTest, dsched_auto_retire) {
   DSched sched(DSched::uniform(0));
-  mt_refcount_test<DeterministicAtomic>();
+  auto_retire_test<DeterministicAtomic>();
 }
 
 TEST(HazptrTest, free_function_retire) {
@@ -1036,7 +1210,7 @@ uint64_t list_protect_all_bench(
     int size,
     bool provided = false) {
   auto repFn = [&] {
-    List<NodeRC<>> l(size);
+    List<NodeRC<true>> l(size);
     auto init = [] {};
     auto fn = [&](int tid) {
       if (provided) {