Commit 13e9943e authored by Maged Michael's avatar Maged Michael Committed by Facebook GitHub Bot

ConcurrentHashMap: Optimize iteration, empty, size, destruction.

Summary:
## Optimizations:
- Avoid unnecessary creation of empty segments when iterating.
- Keep track of the lowest and highest nonempty segments.

## Improvements:
- Construction/destruction:
```
CHM ctor/dtor -- empty              213 ns    200 ns    196 ns
CHM ctor/dtor -- 1 item             395 ns    388 ns    373 ns
```
```
CHM ctor/dtor -- empty               99 ns     96 ns     94 ns
CHM ctor/dtor -- 1 item             218 ns    217 ns    216 ns
```
- begin() before and after:
```
CHM begin() -- empty                705 ns    689 ns    660 ns
```
```
CHM begin() -- empty                  6 ns      6 ns      6 ns
```
- Iteration before and after:
```
CHM iterate -- empty                722 ns    714 ns    707 ns
CHM iterate -- 1 item               621 ns    601 ns    587 ns
CHM iterate -- 10 items              89 ns     88 ns     84 ns
CHM iterate -- 100 items             55 ns     51 ns     48 ns
CHM iterate -- 1K items              15 ns     14 ns     13 ns
CHM iterate -- 10K items             13 ns     13 ns     12 ns
CHM iterate -- 100K items            13 ns     12 ns     11 ns
CHM iterate -- 1M items              36 ns     29 ns     27 ns
CHM iterate -- 10M items             36 ns     35 ns     34 ns
```
```
CHM iterate -- empty                  7 ns      6 ns      6 ns
CHM iterate -- 1 item                33 ns     32 ns     31 ns
CHM iterate -- 10 items              27 ns     26 ns     26 ns
CHM iterate -- 100 items             34 ns     33 ns     33 ns
CHM iterate -- 1K items              10 ns     10 ns     10 ns
CHM iterate -- 10K items             10 ns      9 ns      8 ns
CHM iterate -- 100K items             9 ns      7 ns      7 ns
CHM iterate -- 1M items              24 ns     19 ns     17 ns
CHM iterate -- 10M items             27 ns     24 ns     23 ns
```
- empty() before and after:
```
CHM empty() -- empty                106 ns    104 ns    102 ns
```
```
CHM empty() -- empty                  1 ns      1 ns      1 ns
```
- size() before and after:
```
CHM size() -- empty                 104 ns    101 ns     99 ns
CHM size() -- 1 item                103 ns    101 ns     96 ns
CHM size() -- 10 items              107 ns    106 ns    104 ns
CHM size() -- 100 items             111 ns    100 ns     95 ns
```
```
CHM size() -- empty                   1 ns      1 ns      1 ns
CHM size() -- 1 item                  2 ns      2 ns      2 ns
CHM size() -- 10 items                8 ns      8 ns      8 ns
CHM size() -- 100 items              88 ns     71 ns     68 ns
```

Reviewed By: Gownta

Differential Revision: D32684489

fbshipit-source-id: fb58277d6e58de0d95cca3c4274e1b08d182a45b
parent d3aff664
...@@ -222,6 +222,12 @@ class ConcurrentHashMap { ...@@ -222,6 +222,12 @@ class ConcurrentHashMap {
} }
cohort_.store(o.cohort(), std::memory_order_relaxed); cohort_.store(o.cohort(), std::memory_order_relaxed);
o.cohort_.store(nullptr, std::memory_order_relaxed); o.cohort_.store(nullptr, std::memory_order_relaxed);
beginSeg_.store(
o.beginSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
o.beginSeg_.store(NumShards, std::memory_order_relaxed);
endSeg_.store(
o.endSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
o.endSeg_.store(0, std::memory_order_relaxed);
} }
ConcurrentHashMap& operator=(ConcurrentHashMap&& o) { ConcurrentHashMap& operator=(ConcurrentHashMap&& o) {
...@@ -241,11 +247,19 @@ class ConcurrentHashMap { ...@@ -241,11 +247,19 @@ class ConcurrentHashMap {
cohort_shutdown_cleanup(); cohort_shutdown_cleanup();
cohort_.store(o.cohort(), std::memory_order_relaxed); cohort_.store(o.cohort(), std::memory_order_relaxed);
o.cohort_.store(nullptr, std::memory_order_relaxed); o.cohort_.store(nullptr, std::memory_order_relaxed);
beginSeg_.store(
o.beginSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
o.beginSeg_.store(NumShards, std::memory_order_relaxed);
endSeg_.store(
o.endSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
o.endSeg_.store(0, std::memory_order_relaxed);
return *this; return *this;
} }
~ConcurrentHashMap() { ~ConcurrentHashMap() {
for (uint64_t i = 0; i < NumShards; i++) { uint64_t begin = beginSeg_.load(std::memory_order_acquire);
uint64_t end = endSeg_.load(std::memory_order_acquire);
for (uint64_t i = begin; i < end; ++i) {
auto seg = segments_[i].load(std::memory_order_relaxed); auto seg = segments_[i].load(std::memory_order_relaxed);
if (seg) { if (seg) {
seg->~SegmentT(); seg->~SegmentT();
...@@ -256,7 +270,12 @@ class ConcurrentHashMap { ...@@ -256,7 +270,12 @@ class ConcurrentHashMap {
} }
bool empty() const noexcept { bool empty() const noexcept {
for (uint64_t i = 0; i < NumShards; i++) { uint64_t begin = beginSeg_.load(std::memory_order_acquire);
uint64_t end = endSeg_.load(std::memory_order_acquire);
// Note: beginSeg_ and endSeg_ are only conservative hints of the
// range of non-empty segments. This function cannot conclude that
// a map is nonempty merely because beginSeg_ < endSeg_.
for (uint64_t i = begin; i < end; ++i) {
auto seg = segments_[i].load(std::memory_order_acquire); auto seg = segments_[i].load(std::memory_order_acquire);
if (seg) { if (seg) {
if (!seg->empty()) { if (!seg->empty()) {
...@@ -425,7 +444,7 @@ class ConcurrentHashMap { ...@@ -425,7 +444,7 @@ class ConcurrentHashMap {
auto segment = pickSegment(pos->first); auto segment = pickSegment(pos->first);
ConstIterator res(this, segment); ConstIterator res(this, segment);
ensureSegment(segment)->erase(res.it_, pos.it_); ensureSegment(segment)->erase(res.it_, pos.it_);
res.next(); // May point to segment end, and need to advance. res.advanceIfAtSegmentEnd();
return res; return res;
} }
...@@ -457,7 +476,9 @@ class ConcurrentHashMap { ...@@ -457,7 +476,9 @@ class ConcurrentHashMap {
// NOT noexcept, initializes new shard segments vs. // NOT noexcept, initializes new shard segments vs.
void clear() { void clear() {
for (uint64_t i = 0; i < NumShards; i++) { uint64_t begin = beginSeg_.load(std::memory_order_acquire);
uint64_t end = endSeg_.load(std::memory_order_acquire);
for (uint64_t i = begin; i < end; ++i) {
auto seg = segments_[i].load(std::memory_order_acquire); auto seg = segments_[i].load(std::memory_order_acquire);
if (seg) { if (seg) {
seg->clear(); seg->clear();
...@@ -469,7 +490,9 @@ class ConcurrentHashMap { ...@@ -469,7 +490,9 @@ class ConcurrentHashMap {
count = count >> ShardBits; count = count >> ShardBits;
if (!count) if (!count)
return; return;
for (uint64_t i = 0; i < NumShards; i++) { uint64_t begin = beginSeg_.load(std::memory_order_acquire);
uint64_t end = endSeg_.load(std::memory_order_acquire);
for (uint64_t i = begin; i < end; ++i) {
auto seg = segments_[i].load(std::memory_order_acquire); auto seg = segments_[i].load(std::memory_order_acquire);
if (seg) { if (seg) {
seg->rehash(count); seg->rehash(count);
...@@ -480,7 +503,9 @@ class ConcurrentHashMap { ...@@ -480,7 +503,9 @@ class ConcurrentHashMap {
// This is a rolling size, and is not exact at any moment in time. // This is a rolling size, and is not exact at any moment in time.
size_t size() const noexcept { size_t size() const noexcept {
size_t res = 0; size_t res = 0;
for (uint64_t i = 0; i < NumShards; i++) { uint64_t begin = beginSeg_.load(std::memory_order_acquire);
uint64_t end = endSeg_.load(std::memory_order_acquire);
for (uint64_t i = begin; i < end; ++i) {
auto seg = segments_[i].load(std::memory_order_acquire); auto seg = segments_[i].load(std::memory_order_acquire);
if (seg) { if (seg) {
res += seg->size(); res += seg->size();
...@@ -492,7 +517,9 @@ class ConcurrentHashMap { ...@@ -492,7 +517,9 @@ class ConcurrentHashMap {
float max_load_factor() const { return load_factor_; } float max_load_factor() const { return load_factor_; }
void max_load_factor(float factor) { void max_load_factor(float factor) {
for (uint64_t i = 0; i < NumShards; i++) { uint64_t begin = beginSeg_.load(std::memory_order_acquire);
uint64_t end = endSeg_.load(std::memory_order_acquire);
for (uint64_t i = begin; i < end; ++i) {
auto seg = segments_[i].load(std::memory_order_acquire); auto seg = segments_[i].load(std::memory_order_acquire);
if (seg) { if (seg) {
seg->max_load_factor(factor); seg->max_load_factor(factor);
...@@ -510,7 +537,7 @@ class ConcurrentHashMap { ...@@ -510,7 +537,7 @@ class ConcurrentHashMap {
ConstIterator& operator++() { ConstIterator& operator++() {
++it_; ++it_;
next(); advanceIfAtSegmentEnd();
return *this; return *this;
} }
...@@ -544,32 +571,43 @@ class ConcurrentHashMap { ...@@ -544,32 +571,43 @@ class ConcurrentHashMap {
private: private:
// cbegin iterator // cbegin iterator
explicit ConstIterator(const ConcurrentHashMap* parent) explicit ConstIterator(const ConcurrentHashMap* parent)
: it_(parent->ensureSegment(0)->cbegin()), : it_(nullptr),
segment_(0), segment_(parent->beginSeg_.load(std::memory_order_acquire)),
parent_(parent) { parent_(parent) {
// Always iterate to the first element, could be in any shard. advanceToSegmentBegin();
next();
} }
// cend iterator // cend iterator
explicit ConstIterator(uint64_t shards) : it_(nullptr), segment_(shards) {} explicit ConstIterator(uint64_t shards) : it_(nullptr), segment_(shards) {}
void next() { void advanceIfAtSegmentEnd() {
while (segment_ < parent_->NumShards && DCHECK_LT(segment_, parent_->NumShards);
it_ == parent_->ensureSegment(segment_)->cend()) { SegmentT* seg =
SegmentT* seg{nullptr}; parent_->segments_[segment_].load(std::memory_order_acquire);
while (!seg) { DCHECK(seg);
segment_++; if (it_ == seg->cend()) {
if (segment_ < parent_->NumShards) { ++segment_;
seg = parent_->segments_[segment_].load(std::memory_order_acquire); advanceToSegmentBegin();
if (!seg) {
continue;
} }
}
FOLLY_ALWAYS_INLINE void advanceToSegmentBegin() {
// Advance to the beginning of the next nonempty segment
// starting from segment_.
uint64_t end = parent_->endSeg_.load(std::memory_order_acquire);
while (segment_ < end) {
SegmentT* seg =
parent_->segments_[segment_].load(std::memory_order_acquire);
if (seg) {
it_ = seg->cbegin(); it_ = seg->cbegin();
if (it_ != seg->cend()) {
return;
} }
break;
} }
++segment_;
} }
// All segments are empty. Advance to end.
segment_ = parent_->NumShards;
} }
typename SegmentT::Iterator it_; typename SegmentT::Iterator it_;
...@@ -658,11 +696,26 @@ class ConcurrentHashMap { ...@@ -658,11 +696,26 @@ class ConcurrentHashMap {
Allocator().deallocate((uint8_t*)newseg, sizeof(SegmentT)); Allocator().deallocate((uint8_t*)newseg, sizeof(SegmentT));
} else { } else {
seg = newseg; seg = newseg;
updateBeginAndEndSegments(i);
} }
} }
return seg; return seg;
} }
void updateBeginAndEndSegments(uint64_t i) const {
uint64_t val = beginSeg_.load(std::memory_order_acquire);
while (i < val && !casSeg(beginSeg_, val, i)) {
}
val = endSeg_.load(std::memory_order_acquire);
while (i + 1 > val && !casSeg(endSeg_, val, i + 1)) {
}
}
bool casSeg(Atom<uint64_t>& seg, uint64_t& expval, uint64_t newval) const {
return seg.compare_exchange_weak(
expval, newval, std::memory_order_acq_rel, std::memory_order_acquire);
}
hazptr_obj_cohort<Atom>* cohort() const noexcept { hazptr_obj_cohort<Atom>* cohort() const noexcept {
return cohort_.load(std::memory_order_acquire); return cohort_.load(std::memory_order_acquire);
} }
...@@ -694,6 +747,8 @@ class ConcurrentHashMap { ...@@ -694,6 +747,8 @@ class ConcurrentHashMap {
size_t size_{0}; size_t size_{0};
size_t max_size_{0}; size_t max_size_{0};
mutable Atom<hazptr_obj_cohort<Atom>*> cohort_{nullptr}; mutable Atom<hazptr_obj_cohort<Atom>*> cohort_{nullptr};
mutable Atom<uint64_t> beginSeg_{NumShards};
mutable Atom<uint64_t> endSeg_{0};
}; };
#if FOLLY_SSE_PREREQ(4, 2) && !FOLLY_MOBILE #if FOLLY_SSE_PREREQ(4, 2) && !FOLLY_MOBILE
......
...@@ -112,6 +112,10 @@ TYPED_TEST_P(ConcurrentHashMapTest, MapTest) { ...@@ -112,6 +112,10 @@ TYPED_TEST_P(ConcurrentHashMapTest, MapTest) {
EXPECT_FALSE(foomap.empty()); EXPECT_FALSE(foomap.empty());
foomap.clear(); foomap.clear();
EXPECT_TRUE(foomap.empty()); EXPECT_TRUE(foomap.empty());
EXPECT_TRUE(foomap.insert(3, 0).second);
EXPECT_FALSE(foomap.empty());
foomap.erase(3);
EXPECT_TRUE(foomap.empty());
} }
TYPED_TEST_P(ConcurrentHashMapTest, MaxSizeTest) { TYPED_TEST_P(ConcurrentHashMapTest, MaxSizeTest) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment