Commit 69c7f1b7 authored by David Goldblatt's avatar David Goldblatt Committed by Facebook Github Bot

Add AtomicReadMostlyMainPtr

Summary:
This is a ReadMostlyMainPtr variant that allows racy accesses. By making the
write-side slower, the read side can avoid any contended shared accesses or
RMWs.

Reviewed By: djwatson

Differential Revision: D13413105

fbshipit-source-id: f03c7ad58be72b63549b145ed6f41c51563831d1
parent cc381bd4
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/experimental/AtomicReadMostlyMainPtr.h>
#include <folly/executors/InlineExecutor.h>
namespace folly {
namespace detail {
namespace {
struct FailingExecutor : folly::Executor {
// We shouldn't be invoking any callbacks.
void add(Func func) override {
LOG(DFATAL)
<< "Added an RCU callback to the AtomicReadMostlyMainPtr executor.";
InlineExecutor::instance().add(std::move(func));
}
};
} // namespace
// *All* modifications of *all* AtomicReadMostlyMainPtrs use the same mutex and
// domain. The first of these just shrinks the size of the individual objects a
// little, but the second is necessary for correctness; all rcu_domains need
// their own tag (as an rcu_domain API constraint), but we want to support
// arbitrarily many AtomicReadMostlyMainPtrs.
Indestructible<std::mutex> atomicReadMostlyMu;
Indestructible<folly::rcu_domain<detail::AtomicReadMostlyTag>>
atomicReadMostlyDomain(new FailingExecutor);
} // namespace detail
} // namespace folly
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <atomic>
#include <cstdint>
#include <memory>
#include <mutex>
#include <folly/Indestructible.h>
#include <folly/experimental/ReadMostlySharedPtr.h>
#include <folly/synchronization/Rcu.h>
namespace folly {
namespace detail {
struct AtomicReadMostlyTag;
extern Indestructible<std::mutex> atomicReadMostlyMu;
extern Indestructible<rcu_domain<AtomicReadMostlyTag>> atomicReadMostlyDomain;
} // namespace detail
/*
* What atomic_shared_ptr is to shared_ptr, AtomicReadMostlyMainPtr is to
* ReadMostlyMainPtr; it allows racy conflicting accesses to one. This gives
* true shared_ptr-like semantics, including reclamation at the point where the
* last pointer to an object goes away.
*
* It's about the same speed (slightly slower) as ReadMostlyMainPtr. The most
* significant feature they share is avoiding reader-reader contention and
* atomic RMWs in the absence of writes.
*/
template <typename T>
class AtomicReadMostlyMainPtr {
public:
AtomicReadMostlyMainPtr() : curMainPtrIndex_(0) {}
explicit AtomicReadMostlyMainPtr(std::shared_ptr<T> ptr)
: curMainPtrIndex_(0) {
mainPtrs_[0] = ReadMostlyMainPtr<T>{std::move(ptr)};
}
void operator=(std::shared_ptr<T> desired) {
store(std::move(desired));
}
bool is_lock_free() const {
return false;
}
ReadMostlySharedPtr<T> load(
std::memory_order order = std::memory_order_seq_cst) const {
rcu_token token = detail::atomicReadMostlyDomain->lock_shared();
// Synchronization point with the store in storeLocked().
auto index = curMainPtrIndex_.load(order);
auto result = mainPtrs_[index].getShared();
detail::atomicReadMostlyDomain->unlock_shared(std::move(token));
return result;
}
void store(
std::shared_ptr<T> ptr,
std::memory_order order = std::memory_order_seq_cst) {
std::shared_ptr<T> old;
{
std::lock_guard<std::mutex> lg(*detail::atomicReadMostlyMu);
old = exchangeLocked(std::move(ptr), order);
}
// If ~T() runs (triggered by the shared_ptr refcount decrement), it's here,
// after dropping the lock. This avoids a possible (albeit esoteric)
// deadlock if ~T() modifies the AtomicReadMostlyMainPtr that used to point
// to it.
}
std::shared_ptr<T> exchange(
std::shared_ptr<T> ptr,
std::memory_order order = std::memory_order_seq_cst) {
std::lock_guard<std::mutex> lg(*detail::atomicReadMostlyMu);
return exchangeLocked(std::move(ptr), order);
}
bool compare_exchange_weak(
std::shared_ptr<T>& expected,
const std::shared_ptr<T>& desired,
std::memory_order successOrder = std::memory_order_seq_cst,
std::memory_order failureOrder = std::memory_order_seq_cst) {
return compare_exchange_strong(
expected, desired, successOrder, failureOrder);
}
bool compare_exchange_strong(
std::shared_ptr<T>& expected,
const std::shared_ptr<T>& desired,
std::memory_order successOrder = std::memory_order_seq_cst,
std::memory_order failureOrder = std::memory_order_seq_cst) {
// See the note at the end of store; we need to defer any destruction we
// might trigger until after the lock is released.
// This is not actually needed down the success path (the reference passed
// in as expected is another pointer to the same object, so we won't
// decrement the refcount to 0), but "never decrement a refcount while
// holding a lock" is an easier rule to keep in our heads, and costs us
// nothing.
std::shared_ptr<T> prev;
std::shared_ptr<T> expectedDup;
{
std::lock_guard<std::mutex> lg(*detail::atomicReadMostlyMu);
auto index = curMainPtrIndex_.load(failureOrder);
ReadMostlyMainPtr<T>& oldMain = mainPtrs_[index];
if (oldMain.get() != expected.get()) {
expectedDup = std::move(expected);
expected = oldMain.getStdShared();
return false;
}
prev = exchangeLocked(desired, successOrder);
}
return true;
}
private:
// Must hold the global mutex.
std::shared_ptr<T> exchangeLocked(
std::shared_ptr<T> ptr,
std::memory_order order = std::memory_order_seq_cst) {
// This is where the tricky bits happen; all modifications of the mainPtrs_
// and index happen here. We maintain the invariant that, on entry to this
// method, all read-side critical sections in progress are using the version
// indicated by curMainPtrIndex_, and the other version is nulled out.
// (Readers can still hold a ReadMostlySharedPtr to the thing the old
// version used to point to; they just can't access the old version to get
// that handle any more).
auto index = curMainPtrIndex_.load(std::memory_order_relaxed);
ReadMostlyMainPtr<T>& oldMain = mainPtrs_[index];
ReadMostlyMainPtr<T>& newMain = mainPtrs_[1 - index];
// From the entry invariant, there's no readers accessing newMain right now.
newMain.reset(std::move(ptr));
// If order is acq_rel, it should degrade to just release, since this is a
// store rather than an RMW. (Of course, this is such a slow method that we
// don't really care, but precision is its own reward. If TSAN one day
// understands asymmetric barriers, this will also improve its error
// detection here). We get our "acquire-y-ness" from the mutex.
auto realOrder =
(order == std::memory_order_acq_rel ? std::memory_order_release
: order);
// After this, read-side critical sections can access both versions, but
// new ones will use newMain.
// This is also synchronization point with loads.
curMainPtrIndex_.store(1 - index, realOrder);
// Wait for all read-side critical sections using oldMain to finish.
detail::atomicReadMostlyDomain->synchronize();
// We've reestablished the first half of the invariant (all readers are
// using newMain), now let's establish the other one (that the other pointer
// is null).
auto result = oldMain.getStdShared();
oldMain.reset();
return result;
}
// The right way to think of this implementation is as an
// std::atomic<ReadMostlyMainPtr<T>*>, protected by RCU. There's only two
// tricky parts:
// 1. We give ourselves our own RCU domain, and synchronize on modification,
// so that we don't do any batching of deallocations. This gives
// shared_ptr-like eager reclamation semantics.
// 2. Instead of putting the ReadMostlyMainPtrs on the heap, we keep them as
// part of the same object to improve locality.
// Really, just a 0/1 index. This is also the synchronization point for memory
// orders.
std::atomic<uint8_t> curMainPtrIndex_;
// Both the ReadMostlyMainPtrs themselves and the domain have nontrivial
// indirections even on the read path, and asymmetric barriers on the write
// path. Some of these could be fused as a later optimization, at the cost of
// having to put more tricky threading primitives in this class that are
// currently abstracted out by those.
ReadMostlyMainPtr<T> mainPtrs_[2];
};
} // namespace folly
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/experimental/AtomicReadMostlyMainPtr.h>
#include <folly/Benchmark.h>
#include <folly/portability/GFlags.h>
using folly::AtomicReadMostlyMainPtr;
using folly::ReadMostlySharedPtr;
BENCHMARK(SingleThreadedLoads, n) {
auto sharedInt = std::make_shared<int>(123);
AtomicReadMostlyMainPtr<int> data(sharedInt);
for (unsigned i = 0; i < n; ++i) {
ReadMostlySharedPtr<int> ptr = data.load();
}
}
BENCHMARK(SingleThreadedStores, n) {
auto sharedInt = std::make_shared<int>(123);
AtomicReadMostlyMainPtr<int> data(sharedInt);
for (unsigned i = 0; i < n; ++i) {
data.store(sharedInt);
ReadMostlySharedPtr<int> ptr = data.load();
}
}
int main(int argc, char** argv) {
gflags::ParseCommandLineFlags(&argc, &argv, true);
folly::runBenchmarks();
return 0;
}
/*
* Output:
============================================================================
folly/experimental/test/AtomicReadMostlyMainPtrBenchmark.cpprelative time/iter iters/s
============================================================================
SingleThreadedLoads 14.36ns 69.65M
SingleThreadedStores 5.88us 170.15K
============================================================================
*/
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment