Commit 88382fbf authored by Pranjal Raihan's avatar Pranjal Raihan Committed by Facebook GitHub Bot

compact_once_flag

Summary: `folly::compact_once_flag` is implemented using 1 byte. `folly::MicroLock` slot0 is used as the mutex for the slow path and slot1 is used for the stored boolean.

Reviewed By: andriigrynenko

Differential Revision: D25797066

fbshipit-source-id: 529de84b50e61523318a19606bc24c54f1ffa3a4
parent 4731bb0c
...@@ -17,19 +17,32 @@ ...@@ -17,19 +17,32 @@
#pragma once #pragma once
#include <atomic> #include <atomic>
#include <mutex> #include <limits>
#include <utility> #include <utility>
#include <folly/Likely.h> #include <folly/Likely.h>
#include <folly/MicroLock.h>
#include <folly/Portability.h> #include <folly/Portability.h>
#include <folly/SharedMutex.h> #include <folly/SharedMutex.h>
#include <folly/functional/Invoke.h> #include <folly/functional/Invoke.h>
namespace folly { namespace folly {
// basic_once_flag
//
// The flag template to be used with call_once. Parameterizable by the mutex
// type and atomic template. The mutex type is required to mimic std::mutex and
// the atomic type is required to mimic std::atomic.
template <typename Mutex, template <typename> class Atom = std::atomic> template <typename Mutex, template <typename> class Atom = std::atomic>
class basic_once_flag; class basic_once_flag;
// compact_once_flag
//
// An alternative flag template that can be used with call_once that uses only
// 1 byte. Internally, compact_once_flag uses folly::MicroLock and its data
// slots.
class compact_once_flag;
// call_once // call_once
// //
// Drop-in replacement for std::call_once. // Drop-in replacement for std::call_once.
...@@ -42,20 +55,18 @@ class basic_once_flag; ...@@ -42,20 +55,18 @@ class basic_once_flag;
// This implementation corrects both flaws. // This implementation corrects both flaws.
// //
// The tradeoff is a slightly larger once_flag struct at 8 bytes, vs 4 bytes // The tradeoff is a slightly larger once_flag struct at 8 bytes, vs 4 bytes
// with libstdc++ on Linux/x64. // with libstdc++ on Linux/x64. However, you may use folly::compact_once_flag
// which is 1 byte.
// //
// Does not work with std::once_flag. // Does not work with std::once_flag.
// //
// mimic: std::call_once // mimic: std::call_once
template < template <typename OnceFlag, typename F, typename... Args>
typename Mutex, FOLLY_ALWAYS_INLINE void call_once(OnceFlag& flag, F&& f, Args&&... args) {
template <typename> if (LIKELY(flag.test_once())) {
class Atom, return;
typename F, }
typename... Args> flag.call_once_slow(std::forward<F>(f), std::forward<Args>(args)...);
FOLLY_ALWAYS_INLINE void
call_once(basic_once_flag<Mutex, Atom>& flag, F&& f, Args&&... args) {
flag.call_once(std::forward<F>(f), std::forward<Args>(args)...);
} }
// try_call_once // try_call_once
...@@ -69,18 +80,15 @@ call_once(basic_once_flag<Mutex, Atom>& flag, F&& f, Args&&... args) { ...@@ -69,18 +80,15 @@ call_once(basic_once_flag<Mutex, Atom>& flag, F&& f, Args&&... args) {
// passed as an argument returns true; otherwise returns false. // passed as an argument returns true; otherwise returns false.
// //
// Note: This has no parallel in the std::once_flag interface. // Note: This has no parallel in the std::once_flag interface.
template < template <typename OnceFlag, typename F, typename... Args>
typename Mutex, FOLLY_NODISCARD FOLLY_ALWAYS_INLINE bool
template <typename> try_call_once(OnceFlag& flag, F&& f, Args&&... args) noexcept {
class Atom,
typename F,
typename... Args>
FOLLY_NODISCARD FOLLY_ALWAYS_INLINE bool try_call_once(
basic_once_flag<Mutex, Atom>& flag,
F&& f,
Args&&... args) noexcept {
static_assert(is_nothrow_invocable_v<F, Args...>, "must be noexcept"); static_assert(is_nothrow_invocable_v<F, Args...>, "must be noexcept");
return flag.try_call_once(std::forward<F>(f), std::forward<Args>(args)...); if (LIKELY(flag.test_once())) {
return true;
}
return flag.try_call_once_slow(
std::forward<F>(f), std::forward<Args>(args)...);
} }
// test_once // test_once
...@@ -91,17 +99,11 @@ FOLLY_NODISCARD FOLLY_ALWAYS_INLINE bool try_call_once( ...@@ -91,17 +99,11 @@ FOLLY_NODISCARD FOLLY_ALWAYS_INLINE bool try_call_once(
// code tracking a separate and possibly-padded bool. // code tracking a separate and possibly-padded bool.
// //
// Note: This has no parallel in the std::once_flag interface. // Note: This has no parallel in the std::once_flag interface.
template <typename Mutex, template <typename> class Atom> template <typename OnceFlag>
FOLLY_ALWAYS_INLINE bool test_once( FOLLY_ALWAYS_INLINE bool test_once(OnceFlag const& flag) noexcept {
basic_once_flag<Mutex, Atom> const& flag) noexcept { return flag.test_once();
return flag.called_.load(std::memory_order_acquire);
} }
// basic_once_flag
//
// The flag template to be used with call_once. Parameterizable by the mutex
// type and atomic template. The mutex type is required to mimic std::mutex and
// the atomic type is required to mimic std::atomic.
template <typename Mutex, template <typename> class Atom> template <typename Mutex, template <typename> class Atom>
class basic_once_flag { class basic_once_flag {
public: public:
...@@ -110,24 +112,11 @@ class basic_once_flag { ...@@ -110,24 +112,11 @@ class basic_once_flag {
basic_once_flag& operator=(const basic_once_flag&) = delete; basic_once_flag& operator=(const basic_once_flag&) = delete;
private: private:
template < template <typename OnceFlag, typename F, typename... Args>
typename Mutex_, friend void call_once(OnceFlag&, F&&, Args&&...);
template <typename>
class Atom_,
typename F,
typename... Args>
friend void call_once(basic_once_flag<Mutex_, Atom_>&, F&&, Args&&...);
template <typename Mutex_, template <typename> class Atom_> template <typename OnceFlag>
friend bool test_once(basic_once_flag<Mutex_, Atom_> const& flag) noexcept; friend bool test_once(OnceFlag const& flag) noexcept;
template <typename F, typename... Args>
FOLLY_ALWAYS_INLINE void call_once(F&& f, Args&&... args) {
if (LIKELY(called_.load(std::memory_order_acquire))) {
return;
}
call_once_slow(std::forward<F>(f), std::forward<Args>(args)...);
}
template <typename F, typename... Args> template <typename F, typename... Args>
FOLLY_NOINLINE void call_once_slow(F&& f, Args&&... args) { FOLLY_NOINLINE void call_once_slow(F&& f, Args&&... args) {
...@@ -139,22 +128,8 @@ class basic_once_flag { ...@@ -139,22 +128,8 @@ class basic_once_flag {
called_.store(true, std::memory_order_release); called_.store(true, std::memory_order_release);
} }
template < template <typename OnceFlag, typename F, typename... Args>
typename Mutex_, friend bool try_call_once(OnceFlag&, F&&, Args&&...) noexcept;
template <typename>
class Atom_,
typename F,
typename... Args>
friend bool
try_call_once(basic_once_flag<Mutex_, Atom_>&, F&&, Args&&...) noexcept;
template <typename F, typename... Args>
FOLLY_ALWAYS_INLINE bool try_call_once(F&& f, Args&&... args) noexcept {
if (LIKELY(called_.load(std::memory_order_acquire))) {
return true;
}
return try_call_once_slow(std::forward<F>(f), std::forward<Args>(args)...);
}
template <typename F, typename... Args> template <typename F, typename... Args>
FOLLY_NOINLINE bool try_call_once_slow(F&& f, Args&&... args) noexcept { FOLLY_NOINLINE bool try_call_once_slow(F&& f, Args&&... args) noexcept {
...@@ -167,15 +142,72 @@ class basic_once_flag { ...@@ -167,15 +142,72 @@ class basic_once_flag {
return pass; return pass;
} }
FOLLY_ALWAYS_INLINE bool test_once() const noexcept {
return called_.load(std::memory_order_acquire);
}
Atom<bool> called_{false}; Atom<bool> called_{false};
Mutex mutex_; Mutex mutex_;
}; };
class compact_once_flag {
public:
compact_once_flag() noexcept { mutex_.init(); }
compact_once_flag(const compact_once_flag&) = delete;
compact_once_flag& operator=(const compact_once_flag&) = delete;
private:
template <typename OnceFlag, typename F, typename... Args>
friend void call_once(OnceFlag&, F&&, Args&&...);
template <typename OnceFlag>
friend bool test_once(OnceFlag const& flag) noexcept;
template <typename F, typename... Args>
FOLLY_NOINLINE void call_once_slow(F&& f, Args&&... args) {
folly::MicroLock::LockGuardWithDataSlots<1> guard(mutex_, 0);
if (guard.loadedValue() != 0) {
return;
}
invoke(std::forward<F>(f), std::forward<Args>(args)...);
guard.storeValue(std::numeric_limits<uint8_t>::max());
}
template <typename OnceFlag, typename F, typename... Args>
friend bool try_call_once(OnceFlag&, F&&, Args&&...) noexcept;
template <typename F, typename... Args>
FOLLY_NOINLINE bool try_call_once_slow(F&& f, Args&&... args) noexcept {
folly::MicroLock::LockGuardWithDataSlots<1> guard(mutex_);
if (guard.loadedValue() != 0) {
return true;
}
const auto pass = static_cast<bool>(
invoke(std::forward<F>(f), std::forward<Args>(args)...));
guard.storeValue(pass ? std::numeric_limits<uint8_t>::max() : 0);
return pass;
}
FOLLY_ALWAYS_INLINE bool test_once() const noexcept {
const uint8_t storedValue = mutex_.load(std::memory_order_acquire) &
folly::MicroLock::slotMask<1>();
return (storedValue != 0);
}
// We're only using slot0 of the MicroLock as a mutex, slot1 stores a bool
// indicating whether this flag has been already called.
folly::MicroLock mutex_;
};
static_assert(
sizeof(compact_once_flag) == 1,
"compact_once_flag should be 1 byte");
// once_flag // once_flag
// //
// The flag type to be used with call_once. An instance of basic_once_flag. // The flag type to be used with call_once. An instance of basic_once_flag.
// //
// Does not work with sd::call_once. // Does not work with std::call_once.
// //
// mimic: std::once_flag // mimic: std::once_flag
using once_flag = basic_once_flag<SharedMutex>; using once_flag = basic_once_flag<SharedMutex>;
......
...@@ -41,8 +41,18 @@ void bm_impl(CallOnceFunc&& fn, size_t iters) { ...@@ -41,8 +41,18 @@ void bm_impl(CallOnceFunc&& fn, size_t iters) {
} }
} }
TEST(FollyCallOnce, Simple) { template <typename T>
folly::once_flag flag; class FollyCallOnce : public testing::Test {
public:
using OnceFlag = T;
};
using OnceFlagTypes =
testing::Types<folly::once_flag, folly::compact_once_flag>;
TYPED_TEST_CASE(FollyCallOnce, OnceFlagTypes);
TYPED_TEST(FollyCallOnce, Simple) {
typename TestFixture::OnceFlag flag;
auto fn = [&](int* outp) { ++*outp; }; auto fn = [&](int* outp) { ++*outp; };
int out = 0; int out = 0;
ASSERT_FALSE(folly::test_once(folly::as_const(flag))); ASSERT_FALSE(folly::test_once(folly::as_const(flag)));
...@@ -54,9 +64,9 @@ TEST(FollyCallOnce, Simple) { ...@@ -54,9 +64,9 @@ TEST(FollyCallOnce, Simple) {
ASSERT_EQ(1, out); ASSERT_EQ(1, out);
} }
TEST(FollyCallOnce, Exception) { TYPED_TEST(FollyCallOnce, Exception) {
struct ExpectedException {}; struct ExpectedException {};
folly::once_flag flag; typename TestFixture::OnceFlag flag;
size_t numCalls = 0; size_t numCalls = 0;
EXPECT_THROW( EXPECT_THROW(
folly::call_once( folly::call_once(
...@@ -73,9 +83,9 @@ TEST(FollyCallOnce, Exception) { ...@@ -73,9 +83,9 @@ TEST(FollyCallOnce, Exception) {
EXPECT_EQ(2, numCalls); EXPECT_EQ(2, numCalls);
} }
TEST(FollyCallOnce, Stress) { TYPED_TEST(FollyCallOnce, Stress) {
for (int i = 0; i < 100; ++i) { for (int i = 0; i < 100; ++i) {
folly::once_flag flag; typename TestFixture::OnceFlag flag;
int out = 0; int out = 0;
bm_impl([&] { folly::call_once(flag, [&] { ++out; }); }, 100); bm_impl([&] { folly::call_once(flag, [&] { ++out; }); }, 100);
ASSERT_EQ(1, out); ASSERT_EQ(1, out);
...@@ -83,10 +93,10 @@ TEST(FollyCallOnce, Stress) { ...@@ -83,10 +93,10 @@ TEST(FollyCallOnce, Stress) {
} }
namespace { namespace {
template <typename T> template <typename OnceFlag, typename T>
struct Lazy { struct Lazy {
folly::aligned_storage_for_t<T> storage; folly::aligned_storage_for_t<T> storage;
folly::once_flag once; OnceFlag once;
~Lazy() { ~Lazy() {
if (folly::test_once(once)) { if (folly::test_once(once)) {
...@@ -111,15 +121,15 @@ struct MaybeRaise { ...@@ -111,15 +121,15 @@ struct MaybeRaise {
}; };
} // namespace } // namespace
TEST(FollyCallOnce, Lazy) { TYPED_TEST(FollyCallOnce, Lazy) {
Lazy<MaybeRaise> lazy; Lazy<typename TestFixture::OnceFlag, MaybeRaise> lazy;
EXPECT_THROW(lazy.construct_or_fetch(true), std::runtime_error); EXPECT_THROW(lazy.construct_or_fetch(true), std::runtime_error);
auto& num = *lazy.construct_or_fetch(false).check; auto& num = *lazy.construct_or_fetch(false).check;
EXPECT_EQ(7, num); EXPECT_EQ(7, num);
} }
TEST(FollyTryCallOnce, example) { TYPED_TEST(FollyCallOnce, TryCallOnce) {
folly::once_flag once; typename TestFixture::OnceFlag once;
EXPECT_FALSE(folly::try_call_once(once, []() noexcept { return false; })); EXPECT_FALSE(folly::try_call_once(once, []() noexcept { return false; }));
EXPECT_FALSE(folly::test_once(once)); EXPECT_FALSE(folly::test_once(once));
EXPECT_TRUE(folly::try_call_once(once, []() noexcept { return true; })); EXPECT_TRUE(folly::try_call_once(once, []() noexcept { return true; }));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment