Commit 4d234b99 authored by Aaryaman Sagar's avatar Aaryaman Sagar Committed by Facebook Github Bot

Make futex functions free functions instead of members

Summary:
The current futex API required a reference to a futex object in order to
invoke `futexWake()`, although this is not buggy by itself, it is technically
UB and nothing is stopping ASAN from catching this sort of use-after-free and
reporting it as an error.  Especially when the futex is represented as a
pointer, requiring a dereference to reach a member function

The bug can come up when you call `futexWake()` on a futex that has been
destroyed, for example

```
auto&& futex_ptr = std::atomic<Futex<>*>{nullptr};
auto&& thread = std::thread{[&]() {
  auto&& futex = Futex<>{0};
  futex_ptr.store(&futex);

  while (futex.load(std::memory_order_relaxed) != 1) {
    futex.futexWait(0);
  }
}};

while (!futex_ptr.load()) {}
futex_ptr.load()->store(1);
futex_ptr.load()->futexWake(1);
thread.join();
```

Here immediately after the `store(1)`, our thread could have loaded the value,
seen that it had changed, and never went to sleep.  Or it could have seen the
value as 0, went to sleep and immediately returned when it saw that the value
in the futex word was not what was expected.

In the scenario described above calling `futexWake()` is done on a "dangling"
pointer.  To avoid this, we just never dereference the pointer, and pass the
pointer to the futex syscall, where it will do the right things

A side benefit to the refactor is that adding specializations is very easy.
And we don't have to mess with member function specializations anymore, which
are inherently hard to work with (eg. cannot partially specialize)

The ADL extension points (currently implemented for `Futex<std::atomic>`,
`Futex<DeterministicAtomic>` and `Futex<EmulatedFutexAtomic>`) are

```
int futexWakeImpl(FutexType* futex, int count, uint32_t wakeMask);

FutexResult futexWaitImpl(
    FutexType* futex,
    uint32_t expected,
    std::chrono::system_clock::time_point const* absSystemTime,
    std::chrono::steady_clock::time_point const* absSteadyTime,
    uint32_t waitMask);
```

Reviewed By: yfeldblum

Differential Revision: D9376527

fbshipit-source-id: bb2b54e511fdf1da992c630a9bc7dc37f76da641
parent 2f002209
...@@ -47,7 +47,7 @@ retry: ...@@ -47,7 +47,7 @@ retry:
goto retry; goto retry;
} }
} }
(void)wordPtr->futexWait(newWord, slotHeldBit); detail::futexWait(wordPtr, newWord, slotHeldBit);
needWaitBit = slotWaitBit; needWaitBit = slotWaitBit;
} else if (spins > maxSpins) { } else if (spins > maxSpins) {
// sched_yield(), but more portable // sched_yield(), but more portable
......
...@@ -157,7 +157,7 @@ void MicroLockCore::unlock(unsigned slot) { ...@@ -157,7 +157,7 @@ void MicroLockCore::unlock(unsigned slot) {
oldWord, newWord, std::memory_order_release, std::memory_order_relaxed)); oldWord, newWord, std::memory_order_release, std::memory_order_relaxed));
if (oldWord & waitBit(slot)) { if (oldWord & waitBit(slot)) {
(void)wordPtr->futexWake(1, heldBit(slot)); detail::futexWake(wordPtr, 1, heldBit(slot));
} }
} }
......
...@@ -513,7 +513,7 @@ class SharedMutexImpl { ...@@ -513,7 +513,7 @@ class SharedMutexImpl {
} }
bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) { bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) {
futex.futexWait(expected, waitMask); detail::futexWait(&futex, expected, waitMask);
return true; return true;
} }
}; };
...@@ -566,7 +566,8 @@ class SharedMutexImpl { ...@@ -566,7 +566,8 @@ class SharedMutexImpl {
} }
bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) { bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) {
auto result = futex.futexWaitUntil(expected, deadline(), waitMask); auto result =
detail::futexWaitUntil(&futex, expected, deadline(), waitMask);
return result != folly::detail::FutexResult::TIMEDOUT; return result != folly::detail::FutexResult::TIMEDOUT;
} }
}; };
...@@ -586,7 +587,8 @@ class SharedMutexImpl { ...@@ -586,7 +587,8 @@ class SharedMutexImpl {
} }
bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) { bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) {
auto result = futex.futexWaitUntil(expected, absDeadline_, waitMask); auto result =
detail::futexWaitUntil(&futex, expected, absDeadline_, waitMask);
return result != folly::detail::FutexResult::TIMEDOUT; return result != folly::detail::FutexResult::TIMEDOUT;
} }
}; };
...@@ -979,7 +981,8 @@ class SharedMutexImpl { ...@@ -979,7 +981,8 @@ class SharedMutexImpl {
// wakeup, we just disable the optimization in the case that there // wakeup, we just disable the optimization in the case that there
// are waiting U or S that we are eligible to wake. // are waiting U or S that we are eligible to wake.
if ((wakeMask & kWaitingE) == kWaitingE && if ((wakeMask & kWaitingE) == kWaitingE &&
(state & wakeMask) == kWaitingE && state_.futexWake(1, kWaitingE) > 0) { (state & wakeMask) == kWaitingE &&
detail::futexWake(&state_, 1, kWaitingE) > 0) {
// somebody woke up, so leave state_ as is and clear it later // somebody woke up, so leave state_ as is and clear it later
return; return;
} }
...@@ -994,7 +997,7 @@ class SharedMutexImpl { ...@@ -994,7 +997,7 @@ class SharedMutexImpl {
} }
void futexWakeAll(uint32_t wakeMask) { void futexWakeAll(uint32_t wakeMask) {
state_.futexWake(std::numeric_limits<int>::max(), wakeMask); detail::futexWake(&state_, std::numeric_limits<int>::max(), wakeMask);
} }
DeferredReaderSlot* deferredReader(uint32_t slot) { DeferredReaderSlot* deferredReader(uint32_t slot) {
......
...@@ -601,7 +601,7 @@ class DynamicBoundedQueue { ...@@ -601,7 +601,7 @@ class DynamicBoundedQueue {
} }
if (MayBlock) { if (MayBlock) {
if (canBlock(weight, capacity)) { if (canBlock(weight, capacity)) {
waiting_.futexWaitUntil(WAITING, deadline); detail::futexWaitUntil(&waiting_, WAITING, deadline);
} }
} else { } else {
asm_volatile_pause(); asm_volatile_pause();
...@@ -645,7 +645,7 @@ class DynamicBoundedQueue { ...@@ -645,7 +645,7 @@ class DynamicBoundedQueue {
if (MayBlock) { if (MayBlock) {
std::atomic_thread_fence(std::memory_order_seq_cst); std::atomic_thread_fence(std::memory_order_seq_cst);
waiting_.store(NOTWAITING, std::memory_order_relaxed); waiting_.store(NOTWAITING, std::memory_order_relaxed);
waiting_.futexWake(); detail::futexWake(&waiting_);
} }
} }
......
/*
* Copyright 2013-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <folly/detail/Futex.h>
namespace folly {
namespace detail {
/** Optimal when TargetClock is the same type as Clock.
*
* Otherwise, both Clock::now() and TargetClock::now() must be invoked. */
template <typename TargetClock, typename Clock, typename Duration>
typename TargetClock::time_point time_point_conv(
std::chrono::time_point<Clock, Duration> const& time) {
using std::chrono::duration_cast;
using TimePoint = std::chrono::time_point<Clock, Duration>;
using TargetDuration = typename TargetClock::duration;
using TargetTimePoint = typename TargetClock::time_point;
if (time == TimePoint::max()) {
return TargetTimePoint::max();
} else if (std::is_same<Clock, TargetClock>::value) {
// in place of time_point_cast, which cannot compile without if-constexpr
auto const delta = time.time_since_epoch();
return TargetTimePoint(duration_cast<TargetDuration>(delta));
} else {
// different clocks with different epochs, so non-optimal case
auto const delta = time - Clock::now();
return TargetClock::now() + duration_cast<TargetDuration>(delta);
}
}
/**
* Available overloads, with definitions elsewhere
*
* These functions are treated as ADL-extension points, the templates above
* call these functions without them having being pre-declared. This works
* because ADL lookup finds the definitions of these functions when you pass
* the relevant arguments
*/
int futexWakeImpl(Futex<std::atomic>* futex, int count, uint32_t wakeMask);
FutexResult futexWaitImpl(
Futex<std::atomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
int futexWakeImpl(
Futex<EmulatedFutexAtomic>* futex,
int count,
uint32_t wakeMask);
FutexResult futexWaitImpl(
Futex<EmulatedFutexAtomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
template <typename Futex, typename Deadline>
typename std::enable_if<Deadline::clock::is_steady, FutexResult>::type
futexWaitImpl(
Futex* futex,
uint32_t expected,
Deadline const& deadline,
uint32_t waitMask) {
return futexWaitImpl(futex, expected, nullptr, &deadline, waitMask);
}
template <typename Futex, typename Deadline>
typename std::enable_if<!Deadline::clock::is_steady, FutexResult>::type
futexWaitImpl(
Futex* futex,
uint32_t expected,
Deadline const& deadline,
uint32_t waitMask) {
return futexWaitImpl(futex, expected, &deadline, nullptr, waitMask);
}
template <typename Futex>
FutexResult futexWait(Futex* futex, uint32_t expected, uint32_t waitMask) {
auto rv = futexWaitImpl(futex, expected, nullptr, nullptr, waitMask);
assert(rv != FutexResult::TIMEDOUT);
return rv;
}
template <typename Futex>
int futexWake(Futex* futex, int count, uint32_t wakeMask) {
return futexWakeImpl(futex, count, wakeMask);
}
template <typename Futex, class Clock, class Duration>
FutexResult futexWaitUntil(
Futex* futex,
uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask) {
using Target = typename std::conditional<
Clock::is_steady,
std::chrono::steady_clock,
std::chrono::system_clock>::type;
auto const converted = time_point_conv<Target>(deadline);
return converted == Target::time_point::max()
? futexWaitImpl(futex, expected, nullptr, nullptr, waitMask)
: futexWaitImpl(futex, expected, converted, waitMask);
}
} // namespace detail
} // namespace folly
...@@ -222,47 +222,46 @@ FutexResult emulatedFutexWaitImpl( ...@@ -222,47 +222,46 @@ FutexResult emulatedFutexWaitImpl(
} // namespace } // namespace
///////////////////////////////// /////////////////////////////////
// Futex<> specializations // Futex<> overloads
template <> int futexWakeImpl(Futex<std::atomic>* futex, int count, uint32_t wakeMask) {
int
Futex<std::atomic>::futexWake(int count, uint32_t wakeMask) {
#ifdef __linux__ #ifdef __linux__
return nativeFutexWake(this, count, wakeMask); return nativeFutexWake(futex, count, wakeMask);
#else #else
return emulatedFutexWake(this, count, wakeMask); return emulatedFutexWake(futex, count, wakeMask);
#endif #endif
} }
template <> int futexWakeImpl(
int Futex<EmulatedFutexAtomic>* futex,
Futex<EmulatedFutexAtomic>::futexWake(int count, uint32_t wakeMask) { int count,
return emulatedFutexWake(this, count, wakeMask); uint32_t wakeMask) {
return emulatedFutexWake(futex, count, wakeMask);
} }
template <> FutexResult futexWaitImpl(
FutexResult Futex<std::atomic>::futexWaitImpl( Futex<std::atomic>* futex,
uint32_t expected, uint32_t expected,
system_clock::time_point const* absSystemTime, system_clock::time_point const* absSystemTime,
steady_clock::time_point const* absSteadyTime, steady_clock::time_point const* absSteadyTime,
uint32_t waitMask) { uint32_t waitMask) {
#ifdef __linux__ #ifdef __linux__
return nativeFutexWaitImpl( return nativeFutexWaitImpl(
this, expected, absSystemTime, absSteadyTime, waitMask); futex, expected, absSystemTime, absSteadyTime, waitMask);
#else #else
return emulatedFutexWaitImpl( return emulatedFutexWaitImpl(
this, expected, absSystemTime, absSteadyTime, waitMask); futex, expected, absSystemTime, absSteadyTime, waitMask);
#endif #endif
} }
template <> FutexResult futexWaitImpl(
FutexResult Futex<EmulatedFutexAtomic>::futexWaitImpl( Futex<EmulatedFutexAtomic>* futex,
uint32_t expected, uint32_t expected,
system_clock::time_point const* absSystemTime, system_clock::time_point const* absSystemTime,
steady_clock::time_point const* absSteadyTime, steady_clock::time_point const* absSteadyTime,
uint32_t waitMask) { uint32_t waitMask) {
return emulatedFutexWaitImpl( return emulatedFutexWaitImpl(
this, expected, absSystemTime, absSteadyTime, waitMask); futex, expected, absSystemTime, absSteadyTime, waitMask);
} }
} // namespace detail } // namespace detail
......
...@@ -19,12 +19,14 @@ ...@@ -19,12 +19,14 @@
#include <atomic> #include <atomic>
#include <cassert> #include <cassert>
#include <chrono> #include <chrono>
#include <cstdint>
#include <limits> #include <limits>
#include <type_traits> #include <type_traits>
#include <folly/portability/Unistd.h> #include <folly/portability/Unistd.h>
namespace folly { namespace detail { namespace folly {
namespace detail {
enum class FutexResult { enum class FutexResult {
VALUE_CHANGED, /* futex value didn't match expected */ VALUE_CHANGED, /* futex value didn't match expected */
...@@ -41,110 +43,55 @@ enum class FutexResult { ...@@ -41,110 +43,55 @@ enum class FutexResult {
* If you don't know how to use futex(), you probably shouldn't be using * If you don't know how to use futex(), you probably shouldn't be using
* this class. Even if you do know how, you should have a good reason * this class. Even if you do know how, you should have a good reason
* (and benchmarks to back you up). * (and benchmarks to back you up).
*
* Because of the semantics of the futex syscall, the futex family of
* functions are available as free functions rather than member functions
*/ */
template <template <typename> class Atom = std::atomic> template <template <typename> class Atom = std::atomic>
struct Futex : Atom<uint32_t> { using Futex = Atom<std::uint32_t>;
Futex() : Atom<uint32_t>() {}
explicit constexpr Futex(uint32_t init) : Atom<uint32_t>(init) {}
/** Puts the thread to sleep if this->load() == expected. Returns true when
* it is returning because it has consumed a wake() event, false for any
* other return (signal, this->load() != expected, or spurious wakeup). */
FutexResult futexWait(uint32_t expected, uint32_t waitMask = -1) {
auto rv = futexWaitImpl(expected, nullptr, nullptr, waitMask);
assert(rv != FutexResult::TIMEDOUT);
return rv;
}
/** Similar to futexWait but also accepts a deadline until when the wait call
* may block.
*
* Optimal clock types: std::chrono::system_clock, std::chrono::steady_clock.
* NOTE: On some systems steady_clock is just an alias for system_clock,
* and is not actually steady.
*
* For any other clock type, now() will be invoked twice. */
template <class Clock, class Duration = typename Clock::duration>
FutexResult futexWaitUntil(
uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask = -1) {
using Target = typename std::conditional<
Clock::is_steady,
std::chrono::steady_clock,
std::chrono::system_clock>::type;
auto const converted = time_point_conv<Target>(deadline);
return converted == Target::time_point::max()
? futexWaitImpl(expected, nullptr, nullptr, waitMask)
: futexWaitImpl(expected, converted, waitMask);
}
/** Wakens up to count waiters where (waitMask & wakeMask) !=
* 0, returning the number of awoken threads, or -1 if an error
* occurred. Note that when constructing a concurrency primitive
* that can guard its own destruction, it is likely that you will
* want to ignore EINVAL here (as well as making sure that you
* never touch the object after performing the memory store that
* is the linearization point for unlock or control handoff).
* See https://sourceware.org/bugzilla/show_bug.cgi?id=13690 */
int futexWake(int count = std::numeric_limits<int>::max(),
uint32_t wakeMask = -1);
private:
/** Optimal when TargetClock is the same type as Clock.
*
* Otherwise, both Clock::now() and TargetClock::now() must be invoked. */
template <typename TargetClock, typename Clock, typename Duration>
static typename TargetClock::time_point time_point_conv(
std::chrono::time_point<Clock, Duration> const& time) {
using std::chrono::duration_cast;
using TimePoint = std::chrono::time_point<Clock, Duration>;
using TargetDuration = typename TargetClock::duration;
using TargetTimePoint = typename TargetClock::time_point;
if (time == TimePoint::max()) {
return TargetTimePoint::max();
} else if (std::is_same<Clock, TargetClock>::value) {
// in place of time_point_cast, which cannot compile without if-constexpr
auto const delta = time.time_since_epoch();
return TargetTimePoint(duration_cast<TargetDuration>(delta));
} else {
// different clocks with different epochs, so non-optimal case
auto const delta = time - Clock::now();
return TargetClock::now() + duration_cast<TargetDuration>(delta);
}
}
template <typename Deadline> /**
typename std::enable_if<Deadline::clock::is_steady, FutexResult>::type * Puts the thread to sleep if this->load() == expected. Returns true when
futexWaitImpl( * it is returning because it has consumed a wake() event, false for any
uint32_t expected, * other return (signal, this->load() != expected, or spurious wakeup).
Deadline const& deadline, */
uint32_t waitMask) { template <typename Futex>
return futexWaitImpl(expected, nullptr, &deadline, waitMask); FutexResult futexWait(Futex* futex, uint32_t expected, uint32_t waitMask = -1);
}
template <typename Deadline> /**
typename std::enable_if<!Deadline::clock::is_steady, FutexResult>::type * Similar to futexWait but also accepts a deadline until when the wait call
futexWaitImpl( * may block.
uint32_t expected, *
Deadline const& deadline, * Optimal clock types: std::chrono::system_clock, std::chrono::steady_clock.
uint32_t waitMask) { * NOTE: On some systems steady_clock is just an alias for system_clock,
return futexWaitImpl(expected, &deadline, nullptr, waitMask); * and is not actually steady.
} *
* For any other clock type, now() will be invoked twice.
*/
template <
typename Futex,
class Clock,
class Duration = typename Clock::duration>
FutexResult futexWaitUntil(
Futex* futex,
uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask = -1);
/** Underlying implementation of futexWait and futexWaitUntil. /**
* At most one of absSystemTime and absSteadyTime should be non-null. * Wakes up to count waiters where (waitMask & wakeMask) != 0, returning the
* Timeouts are separated into separate parameters to allow the * number of awoken threads, or -1 if an error occurred. Note that when
* implementations to be elsewhere without templating on the clock * constructing a concurrency primitive that can guard its own destruction, it
* type, which is otherwise complicated by the fact that steady_clock * is likely that you will want to ignore EINVAL here (as well as making sure
* is the same as system_clock on some platforms. */ * that you never touch the object after performing the memory store that is
FutexResult futexWaitImpl( * the linearization point for unlock or control handoff). See
uint32_t expected, * https://sourceware.org/bugzilla/show_bug.cgi?id=13690
std::chrono::system_clock::time_point const* absSystemTime, */
std::chrono::steady_clock::time_point const* absSteadyTime, template <typename Futex>
uint32_t waitMask); int futexWake(
}; Futex* futex,
int count = std::numeric_limits<int>::max(),
uint32_t wakeMask = -1);
/** A std::atomic subclass that can be used to force Futex to emulate /** A std::atomic subclass that can be used to force Futex to emulate
* the underlying futex() syscall. This is primarily useful to test or * the underlying futex() syscall. This is primarily useful to test or
...@@ -158,27 +105,7 @@ struct EmulatedFutexAtomic : public std::atomic<T> { ...@@ -158,27 +105,7 @@ struct EmulatedFutexAtomic : public std::atomic<T> {
EmulatedFutexAtomic(EmulatedFutexAtomic&& rhs) = delete; EmulatedFutexAtomic(EmulatedFutexAtomic&& rhs) = delete;
}; };
/* Available specializations, with definitions elsewhere */
template <>
int Futex<std::atomic>::futexWake(int count, uint32_t wakeMask);
template <>
FutexResult Futex<std::atomic>::futexWaitImpl(
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
template <>
int Futex<EmulatedFutexAtomic>::futexWake(int count, uint32_t wakeMask);
template <>
FutexResult Futex<EmulatedFutexAtomic>::futexWaitImpl(
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
} // namespace detail } // namespace detail
} // namespace folly } // namespace folly
#include <folly/detail/Futex-inl.h>
...@@ -99,10 +99,10 @@ struct MemoryIdler { ...@@ -99,10 +99,10 @@ struct MemoryIdler {
/// system with bursty requests. The default is to wait up to 50% /// system with bursty requests. The default is to wait up to 50%
/// extra, so on average 25% extra. /// extra, so on average 25% extra.
template < template <
template <typename> class Atom, typename Futex,
typename IdleTime = std::chrono::steady_clock::duration> typename IdleTime = std::chrono::steady_clock::duration>
static FutexResult futexWait( static FutexResult futexWait(
Futex<Atom>& fut, Futex& fut,
uint32_t expected, uint32_t expected,
uint32_t waitMask = -1, uint32_t waitMask = -1,
IdleTime const& idleTimeout = IdleTime const& idleTimeout =
...@@ -121,7 +121,9 @@ struct MemoryIdler { ...@@ -121,7 +121,9 @@ struct MemoryIdler {
timeoutVariationFrac)) { timeoutVariationFrac)) {
return pre; return pre;
} }
return fut.futexWait(expected, waitMask);
using folly::detail::futexWait;
return futexWait(&fut, expected, waitMask);
} }
/// Equivalent to fut.futexWaitUntil(expected, deadline, waitMask), but /// Equivalent to fut.futexWaitUntil(expected, deadline, waitMask), but
...@@ -133,11 +135,11 @@ struct MemoryIdler { ...@@ -133,11 +135,11 @@ struct MemoryIdler {
/// system with bursty requests. The default is to wait up to 50% /// system with bursty requests. The default is to wait up to 50%
/// extra, so on average 25% extra. /// extra, so on average 25% extra.
template < template <
template <typename> class Atom, typename Futex,
typename Deadline, typename Deadline,
typename IdleTime = std::chrono::steady_clock::duration> typename IdleTime = std::chrono::steady_clock::duration>
static FutexResult futexWaitUntil( static FutexResult futexWaitUntil(
Futex<Atom>& fut, Futex& fut,
uint32_t expected, uint32_t expected,
Deadline const& deadline, Deadline const& deadline,
uint32_t waitMask = -1, uint32_t waitMask = -1,
...@@ -157,17 +159,16 @@ struct MemoryIdler { ...@@ -157,17 +159,16 @@ struct MemoryIdler {
timeoutVariationFrac)) { timeoutVariationFrac)) {
return pre; return pre;
} }
return fut.futexWaitUntil(expected, deadline, waitMask);
using folly::detail::futexWaitUntil;
return futexWaitUntil(&fut, expected, deadline, waitMask);
} }
private: private:
template < template <typename Futex, typename Deadline, typename IdleTime>
template <typename> class Atom,
typename Deadline,
typename IdleTime>
static bool futexWaitPreIdle( static bool futexWaitPreIdle(
FutexResult& _ret, FutexResult& _ret,
Futex<Atom>& fut, Futex& fut,
uint32_t expected, uint32_t expected,
Deadline const& deadline, Deadline const& deadline,
uint32_t waitMask, uint32_t waitMask,
...@@ -189,7 +190,8 @@ struct MemoryIdler { ...@@ -189,7 +190,8 @@ struct MemoryIdler {
if (idleTimeout > IdleTime::zero()) { if (idleTimeout > IdleTime::zero()) {
auto idleDeadline = Deadline::clock::now() + idleTimeout; auto idleDeadline = Deadline::clock::now() + idleTimeout;
if (idleDeadline < deadline) { if (idleDeadline < deadline) {
auto rv = fut.futexWaitUntil(expected, idleDeadline, waitMask); using folly::detail::futexWaitUntil;
auto rv = futexWaitUntil(&fut, expected, idleDeadline, waitMask);
if (rv != FutexResult::TIMEDOUT) { if (rv != FutexResult::TIMEDOUT) {
// finished before timeout hit, no flush // finished before timeout hit, no flush
_ret = rv; _ret = rv;
......
...@@ -153,13 +153,13 @@ struct TurnSequencer { ...@@ -153,13 +153,13 @@ struct TurnSequencer {
} }
} }
if (absTime) { if (absTime) {
auto futexResult = auto futexResult = detail::futexWaitUntil(
state_.futexWaitUntil(new_state, *absTime, futexChannel(turn)); &state_, new_state, *absTime, futexChannel(turn));
if (futexResult == FutexResult::TIMEDOUT) { if (futexResult == FutexResult::TIMEDOUT) {
return TryWaitResult::TIMEDOUT; return TryWaitResult::TIMEDOUT;
} }
} else { } else {
state_.futexWait(new_state, futexChannel(turn)); detail::futexWait(&state_, new_state, futexChannel(turn));
} }
} }
...@@ -202,8 +202,8 @@ struct TurnSequencer { ...@@ -202,8 +202,8 @@ struct TurnSequencer {
max_waiter_delta == 0 ? 0 : max_waiter_delta - 1); max_waiter_delta == 0 ? 0 : max_waiter_delta - 1);
if (state_.compare_exchange_strong(state, new_state)) { if (state_.compare_exchange_strong(state, new_state)) {
if (max_waiter_delta != 0) { if (max_waiter_delta != 0) {
state_.futexWake(std::numeric_limits<int>::max(), detail::futexWake(
futexChannel(turn + 1)); &state_, std::numeric_limits<int>::max(), futexChannel(turn + 1));
} }
break; break;
} }
......
...@@ -142,8 +142,8 @@ inline void EventCount::notifyAll() noexcept { ...@@ -142,8 +142,8 @@ inline void EventCount::notifyAll() noexcept {
inline void EventCount::doNotify(int n) noexcept { inline void EventCount::doNotify(int n) noexcept {
uint64_t prev = val_.fetch_add(kAddEpoch, std::memory_order_acq_rel); uint64_t prev = val_.fetch_add(kAddEpoch, std::memory_order_acq_rel);
if (UNLIKELY(prev & kWaiterMask)) { if (UNLIKELY(prev & kWaiterMask)) {
(reinterpret_cast<detail::Futex<std::atomic>*>(&val_) + kEpochOffset) detail::futexWake(
->futexWake(n); reinterpret_cast<detail::Futex<std::atomic>*>(&val_) + kEpochOffset, n);
} }
} }
...@@ -162,8 +162,9 @@ inline void EventCount::cancelWait() noexcept { ...@@ -162,8 +162,9 @@ inline void EventCount::cancelWait() noexcept {
inline void EventCount::wait(Key key) noexcept { inline void EventCount::wait(Key key) noexcept {
while ((val_.load(std::memory_order_acquire) >> kEpochShift) == key.epoch_) { while ((val_.load(std::memory_order_acquire) >> kEpochShift) == key.epoch_) {
(reinterpret_cast<detail::Futex<std::atomic>*>(&val_) + kEpochOffset) detail::futexWait(
->futexWait(key.epoch_); reinterpret_cast<detail::Futex<std::atomic>*>(&val_) + kEpochOffset,
key.epoch_);
} }
// memory_order_relaxed would suffice for correctness, but the faster // memory_order_relaxed would suffice for correctness, but the faster
// #waiters gets to 0, the less likely it is that we'll do spurious wakeups // #waiters gets to 0, the less likely it is that we'll do spurious wakeups
......
...@@ -320,7 +320,7 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_push_impl( ...@@ -320,7 +320,7 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_push_impl(
if (res) { if (res) {
if (wake) { if (wake) {
empty_.futexWake(); detail::futexWake(&empty_);
} }
return true; return true;
} }
...@@ -329,12 +329,12 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_push_impl( ...@@ -329,12 +329,12 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_push_impl(
} }
while (isTrue(full_)) { while (isTrue(full_)) {
if (when == std::chrono::time_point<Clock>::max()) { if (when == std::chrono::time_point<Clock>::max()) {
full_.futexWait(1); detail::futexWait(&full_, 1);
} else { } else {
if (Clock::now() > when) { if (Clock::now() > when) {
return false; return false;
} else { } else {
full_.futexWaitUntil(1, when); detail::futexWaitUntil(&full_, 1, when);
} }
} }
} // inner while loop } // inner while loop
...@@ -369,18 +369,18 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_pop_impl( ...@@ -369,18 +369,18 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_pop_impl(
if (res) { if (res) {
if (wake) { if (wake) {
full_.futexWake(); detail::futexWake(&full_);
} }
return true; return true;
} }
while (isTrue(empty_)) { while (isTrue(empty_)) {
if (when == std::chrono::time_point<Clock>::max()) { if (when == std::chrono::time_point<Clock>::max()) {
empty_.futexWait(1); detail::futexWait(&empty_, 1);
} else { } else {
if (Clock::now() > when) { if (Clock::now() > when) {
return false; return false;
} else { } else {
empty_.futexWaitUntil(1, when); detail::futexWaitUntil(&empty_, 1, when);
} }
} }
} // inner while loop } // inner while loop
...@@ -415,12 +415,12 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_peek_impl( ...@@ -415,12 +415,12 @@ FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_peek_impl(
} }
while (isTrue(empty_)) { while (isTrue(empty_)) {
if (when == std::chrono::time_point<Clock>::max()) { if (when == std::chrono::time_point<Clock>::max()) {
empty_.futexWait(1); detail::futexWait(&empty_, 1);
} else { } else {
if (Clock::now() > when) { if (Clock::now() > when) {
return false; return false;
} else { } else {
empty_.futexWaitUntil(1, when); detail::futexWaitUntil(&empty_, 1, when);
} }
} }
} // inner while loop } // inner while loop
......
...@@ -1042,7 +1042,7 @@ class RelaxedConcurrentPriorityQueue { ...@@ -1042,7 +1042,7 @@ class RelaxedConcurrentPriorityQueue {
if (futex_array_[loc].compare_exchange_strong(curfutex, ready)) { if (futex_array_[loc].compare_exchange_strong(curfutex, ready)) {
if (curfutex & if (curfutex &
1) { // One or more consumers may be blocked on this futex 1) { // One or more consumers may be blocked on this futex
futex_array_[loc].futexWake(); detail::futexWake(&futex_array_[loc]);
} }
return; return;
} else { } else {
...@@ -1095,7 +1095,7 @@ class RelaxedConcurrentPriorityQueue { ...@@ -1095,7 +1095,7 @@ class RelaxedConcurrentPriorityQueue {
auto curfutex = futex_array_[loc].load(std::memory_order_acquire); auto curfutex = futex_array_[loc].load(std::memory_order_acquire);
if (curfutex & if (curfutex &
1) { /// The last round consumers are still waiting, go to sleep 1) { /// The last round consumers are still waiting, go to sleep
futex_array_[loc].futexWait(curfutex); detail::futexWait(&futex_array_[loc], curfutex);
} }
if (trySpinBeforeBlock( if (trySpinBeforeBlock(
curticket, curticket,
...@@ -1106,12 +1106,12 @@ class RelaxedConcurrentPriorityQueue { ...@@ -1106,12 +1106,12 @@ class RelaxedConcurrentPriorityQueue {
curfutex = futex_array_[loc].load(std::memory_order_acquire); curfutex = futex_array_[loc].load(std::memory_order_acquire);
if (curfutex & if (curfutex &
1) { /// The last round consumers are still waiting, go to sleep 1) { /// The last round consumers are still waiting, go to sleep
futex_array_[loc].futexWait(curfutex); detail::futexWait(&futex_array_[loc], curfutex);
} else if (!futexIsReady(curticket)) { // current ticket < pop ticket } else if (!futexIsReady(curticket)) { // current ticket < pop ticket
uint32_t blocking_futex = curfutex + 1; uint32_t blocking_futex = curfutex + 1;
if (futex_array_[loc].compare_exchange_strong( if (futex_array_[loc].compare_exchange_strong(
curfutex, blocking_futex)) { curfutex, blocking_futex)) {
futex_array_[loc].futexWait(blocking_futex); detail::futexWait(&futex_array_[loc], blocking_futex);
} }
} else { } else {
return; return;
......
...@@ -24,6 +24,9 @@ ...@@ -24,6 +24,9 @@
namespace folly { namespace folly {
namespace fibers { namespace fibers {
using folly::detail::futexWaitUntil;
using folly::detail::futexWake;
void Baton::setWaiter(Waiter& waiter) { void Baton::setWaiter(Waiter& waiter) {
auto curr_waiter = waiter_.load(); auto curr_waiter = waiter_.load();
do { do {
...@@ -120,8 +123,9 @@ bool Baton::timedWaitThread(TimeoutController::Duration timeout) { ...@@ -120,8 +123,9 @@ bool Baton::timedWaitThread(TimeoutController::Duration timeout) {
waiter_.compare_exchange_strong(waiter, THREAD_WAITING))) { waiter_.compare_exchange_strong(waiter, THREAD_WAITING))) {
auto deadline = TimeoutController::Clock::now() + timeout; auto deadline = TimeoutController::Clock::now() + timeout;
do { do {
auto* futex = &futex_.futex;
const auto wait_rv = const auto wait_rv =
futex_.futex.futexWaitUntil(uint32_t(THREAD_WAITING), deadline); futexWaitUntil(futex, uint32_t(THREAD_WAITING), deadline);
if (wait_rv == folly::detail::FutexResult::TIMEDOUT) { if (wait_rv == folly::detail::FutexResult::TIMEDOUT) {
return false; return false;
} }
...@@ -174,11 +178,11 @@ bool Baton::try_wait() { ...@@ -174,11 +178,11 @@ bool Baton::try_wait() {
void Baton::postThread() { void Baton::postThread() {
auto expected = THREAD_WAITING; auto expected = THREAD_WAITING;
auto* futex = &futex_.futex;
if (!waiter_.compare_exchange_strong(expected, POSTED)) { if (!waiter_.compare_exchange_strong(expected, POSTED)) {
return; return;
} }
futexWake(futex, 1);
futex_.futex.futexWake(1);
} }
void Baton::reset() { void Baton::reset() {
......
...@@ -24,6 +24,7 @@ struct MockClock { ...@@ -24,6 +24,7 @@ struct MockClock {
public: public:
using duration = std::chrono::steady_clock::duration; using duration = std::chrono::steady_clock::duration;
using time_point = std::chrono::steady_clock::time_point; using time_point = std::chrono::steady_clock::time_point;
static constexpr auto is_steady = true;
static time_point now() { static time_point now() {
return Now; return Now;
......
...@@ -154,7 +154,7 @@ class Baton { ...@@ -154,7 +154,7 @@ class Baton {
assert(before == WAITING); assert(before == WAITING);
state_.store(LATE_DELIVERY, std::memory_order_release); state_.store(LATE_DELIVERY, std::memory_order_release);
state_.futexWake(1); detail::futexWake(&state_, 1);
} }
/// Waits until post() has been called in the current Baton lifetime. /// Waits until post() has been called in the current Baton lifetime.
......
...@@ -268,7 +268,7 @@ FOLLY_NOINLINE void SaturatingSemaphore<MayBlock, Atom>::postSlowWaiterMayBlock( ...@@ -268,7 +268,7 @@ FOLLY_NOINLINE void SaturatingSemaphore<MayBlock, Atom>::postSlowWaiterMayBlock(
READY, READY,
std::memory_order_release, std::memory_order_release,
std::memory_order_relaxed)) { std::memory_order_relaxed)) {
state_.futexWake(); detail::futexWake(&state_);
return; return;
} }
} }
......
...@@ -60,7 +60,7 @@ class ThreadCachedInts { ...@@ -60,7 +60,7 @@ class ThreadCachedInts {
ints_->orphan_dec_[1].fetch_add( ints_->orphan_dec_[1].fetch_add(
dec_[1].load(std::memory_order_relaxed), std::memory_order_relaxed); dec_[1].load(std::memory_order_relaxed), std::memory_order_relaxed);
ints_->waiting_.store(0, std::memory_order_release); ints_->waiting_.store(0, std::memory_order_release);
ints_->waiting_.futexWake(); detail::futexWake(&ints_->waiting_);
// reset the cache_ on destructor so we can handle the delete/recreate // reset the cache_ on destructor so we can handle the delete/recreate
cache_ = nullptr; cache_ = nullptr;
} }
...@@ -102,7 +102,7 @@ class ThreadCachedInts { ...@@ -102,7 +102,7 @@ class ThreadCachedInts {
folly::asymmetricLightBarrier(); // C folly::asymmetricLightBarrier(); // C
if (waiting_.load(std::memory_order_acquire)) { if (waiting_.load(std::memory_order_acquire)) {
waiting_.store(0, std::memory_order_release); waiting_.store(0, std::memory_order_release);
waiting_.futexWake(); detail::futexWake(&waiting_);
} }
} }
...@@ -150,7 +150,7 @@ class ThreadCachedInts { ...@@ -150,7 +150,7 @@ class ThreadCachedInts {
if (readFull(phase) == 0) { if (readFull(phase) == 0) {
break; break;
} }
waiting_.futexWait(1); detail::futexWait(&waiting_, 1);
} }
waiting_.store(0, std::memory_order_relaxed); waiting_.store(0, std::memory_order_relaxed);
} }
......
...@@ -39,7 +39,7 @@ BENCHMARK(FutexNoWaitersWake, iters) { ...@@ -39,7 +39,7 @@ BENCHMARK(FutexNoWaitersWake, iters) {
t = std::thread([&]() { t = std::thread([&]() {
b.wait(); b.wait();
for (auto i = 0u; i < iters; i++) { for (auto i = 0u; i < iters; i++) {
fu.futexWake(1); detail::futexWake(&fu, 1);
} }
}); });
} }
...@@ -82,7 +82,7 @@ BENCHMARK(FutexWakeOne, iters) { ...@@ -82,7 +82,7 @@ BENCHMARK(FutexWakeOne, iters) {
t = std::thread([&]() { t = std::thread([&]() {
b.wait(); b.wait();
while (true) { while (true) {
fu.futexWait(0); detail::futexWait(&fu, 0);
if (fu.load(std::memory_order_relaxed)) { if (fu.load(std::memory_order_relaxed)) {
return; return;
} }
...@@ -92,10 +92,10 @@ BENCHMARK(FutexWakeOne, iters) { ...@@ -92,10 +92,10 @@ BENCHMARK(FutexWakeOne, iters) {
susp.dismiss(); susp.dismiss();
b.wait(); b.wait();
for (auto i = 0u; i < iters; i++) { for (auto i = 0u; i < iters; i++) {
fu.futexWake(1); detail::futexWake(&fu, 1);
} }
fu.store(1); fu.store(1);
fu.futexWake(threads.size()); detail::futexWake(&fu, threads.size());
for (auto& t : threads) { for (auto& t : threads) {
t.join(); t.join();
...@@ -148,7 +148,7 @@ BENCHMARK(FutexWakeAll, iters) { ...@@ -148,7 +148,7 @@ BENCHMARK(FutexWakeAll, iters) {
t = std::thread([&]() { t = std::thread([&]() {
b.wait(); b.wait();
while (true) { while (true) {
fu.futexWait(0); detail::futexWait(&fu, 0);
if (done.load(std::memory_order_relaxed)) { if (done.load(std::memory_order_relaxed)) {
return; return;
} }
...@@ -158,11 +158,11 @@ BENCHMARK(FutexWakeAll, iters) { ...@@ -158,11 +158,11 @@ BENCHMARK(FutexWakeAll, iters) {
susp.dismiss(); susp.dismiss();
b.wait(); b.wait();
for (auto i = 0u; i < iters; i++) { for (auto i = 0u; i < iters; i++) {
fu.futexWake(threads.size()); detail::futexWake(&fu, threads.size());
} }
fu.store(1); fu.store(1);
done = true; done = true;
fu.futexWake(threads.size()); detail::futexWake(&fu, threads.size());
for (auto& t : threads) { for (auto& t : threads) {
t.join(); t.join();
......
...@@ -301,32 +301,28 @@ void DeterministicSchedule::wait(sem_t* sem) { ...@@ -301,32 +301,28 @@ void DeterministicSchedule::wait(sem_t* sem) {
// we're not busy waiting because this is a deterministic schedule // we're not busy waiting because this is a deterministic schedule
} }
} }
} // namespace test
} // namespace folly
namespace folly {
namespace detail {
using namespace test; detail::FutexResult futexWaitImpl(
using namespace std::chrono; detail::Futex<DeterministicAtomic>* futex,
template <>
FutexResult Futex<DeterministicAtomic>::futexWaitImpl(
uint32_t expected, uint32_t expected,
system_clock::time_point const* absSystemTimeout, std::chrono::system_clock::time_point const* absSystemTimeout,
steady_clock::time_point const* absSteadyTimeout, std::chrono::steady_clock::time_point const* absSteadyTimeout,
uint32_t waitMask) { uint32_t waitMask) {
using namespace test;
using namespace std::chrono;
using namespace folly::detail;
bool hasTimeout = absSystemTimeout != nullptr || absSteadyTimeout != nullptr; bool hasTimeout = absSystemTimeout != nullptr || absSteadyTimeout != nullptr;
bool awoken = false; bool awoken = false;
FutexResult result = FutexResult::AWOKEN; FutexResult result = FutexResult::AWOKEN;
DeterministicSchedule::beforeSharedAccess(); DeterministicSchedule::beforeSharedAccess();
FOLLY_TEST_DSCHED_VLOG(this << ".futexWait(" << std::hex << expected FOLLY_TEST_DSCHED_VLOG(
<< ", .., " << std::hex << waitMask "futexWait(" << futex << ", " << std::hex << expected << ", .., "
<< ") beginning.."); << std::hex << waitMask << ") beginning..");
futexLock.lock(); futexLock.lock();
if (this->data == expected) { if (futex->data == expected) {
auto& queue = futexQueues[this]; auto& queue = futexQueues[futex];
queue.emplace_back(waitMask, &awoken); queue.emplace_back(waitMask, &awoken);
auto ours = queue.end(); auto ours = queue.end();
ours--; ours--;
...@@ -340,10 +336,10 @@ FutexResult Futex<DeterministicAtomic>::futexWaitImpl( ...@@ -340,10 +336,10 @@ FutexResult Futex<DeterministicAtomic>::futexWaitImpl(
// a 10% probability if we haven't been woken up already // a 10% probability if we haven't been woken up already
if (!awoken && hasTimeout && if (!awoken && hasTimeout &&
DeterministicSchedule::getRandNumber(100) < 10) { DeterministicSchedule::getRandNumber(100) < 10) {
assert(futexQueues.count(this) != 0 && &futexQueues[this] == &queue); assert(futexQueues.count(futex) != 0 && &futexQueues[futex] == &queue);
queue.erase(ours); queue.erase(ours);
if (queue.empty()) { if (queue.empty()) {
futexQueues.erase(this); futexQueues.erase(futex);
} }
// Simulate ETIMEDOUT 90% of the time and other failures // Simulate ETIMEDOUT 90% of the time and other failures
// remaining time // remaining time
...@@ -373,20 +369,25 @@ FutexResult Futex<DeterministicAtomic>::futexWaitImpl( ...@@ -373,20 +369,25 @@ FutexResult Futex<DeterministicAtomic>::futexWaitImpl(
resultStr = "VALUE_CHANGED"; resultStr = "VALUE_CHANGED";
break; break;
} }
FOLLY_TEST_DSCHED_VLOG(this << ".futexWait(" << std::hex << expected FOLLY_TEST_DSCHED_VLOG(
<< ", .., " << std::hex << waitMask << ") -> " "futexWait(" << futex << ", " << std::hex << expected << ", .., "
<< resultStr); << std::hex << waitMask << ") -> " << resultStr);
DeterministicSchedule::afterSharedAccess(); DeterministicSchedule::afterSharedAccess();
return result; return result;
} }
template <> int futexWakeImpl(
int Futex<DeterministicAtomic>::futexWake(int count, uint32_t wakeMask) { detail::Futex<test::DeterministicAtomic>* futex,
int count,
uint32_t wakeMask) {
using namespace test;
using namespace std::chrono;
int rv = 0; int rv = 0;
DeterministicSchedule::beforeSharedAccess(); DeterministicSchedule::beforeSharedAccess();
futexLock.lock(); futexLock.lock();
if (futexQueues.count(this) > 0) { if (futexQueues.count(futex) > 0) {
auto& queue = futexQueues[this]; auto& queue = futexQueues[futex];
auto iter = queue.begin(); auto iter = queue.begin();
while (iter != queue.end() && rv < count) { while (iter != queue.end() && rv < count) {
auto cur = iter++; auto cur = iter++;
...@@ -397,16 +398,21 @@ int Futex<DeterministicAtomic>::futexWake(int count, uint32_t wakeMask) { ...@@ -397,16 +398,21 @@ int Futex<DeterministicAtomic>::futexWake(int count, uint32_t wakeMask) {
} }
} }
if (queue.empty()) { if (queue.empty()) {
futexQueues.erase(this); futexQueues.erase(futex);
} }
} }
futexLock.unlock(); futexLock.unlock();
FOLLY_TEST_DSCHED_VLOG(this << ".futexWake(" << count << ", " << std::hex FOLLY_TEST_DSCHED_VLOG(
<< wakeMask << ") -> " << rv); "futexWake(" << futex << ", " << count << ", " << std::hex << wakeMask
<< ") -> " << rv);
DeterministicSchedule::afterSharedAccess(); DeterministicSchedule::afterSharedAccess();
return rv; return rv;
} }
} // namespace detail
} // namespace test
} // namespace folly
namespace folly {
template <> template <>
CacheLocality const& CacheLocality::system<test::DeterministicAtomic>() { CacheLocality const& CacheLocality::system<test::DeterministicAtomic>() {
...@@ -416,6 +422,6 @@ CacheLocality const& CacheLocality::system<test::DeterministicAtomic>() { ...@@ -416,6 +422,6 @@ CacheLocality const& CacheLocality::system<test::DeterministicAtomic>() {
template <> template <>
Getcpu::Func AccessSpreader<test::DeterministicAtomic>::pickGetcpuFunc() { Getcpu::Func AccessSpreader<test::DeterministicAtomic>::pickGetcpuFunc() {
return &detail::DeterministicSchedule::getcpu; return &test::DeterministicSchedule::getcpu;
} }
} // namespace folly } // namespace folly
...@@ -455,6 +455,18 @@ struct DeterministicAtomic { ...@@ -455,6 +455,18 @@ struct DeterministicAtomic {
} }
}; };
/* Futex extensions for DeterministicSchedule based Futexes */
int futexWakeImpl(
detail::Futex<test::DeterministicAtomic>* futex,
int count,
uint32_t wakeMask);
detail::FutexResult futexWaitImpl(
detail::Futex<test::DeterministicAtomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
/** /**
* DeterministicMutex is a drop-in replacement of std::mutex that * DeterministicMutex is a drop-in replacement of std::mutex that
* cooperates with DeterministicSchedule. * cooperates with DeterministicSchedule.
...@@ -504,23 +516,6 @@ struct DeterministicMutex { ...@@ -504,23 +516,6 @@ struct DeterministicMutex {
} }
}; };
} // namespace test } // namespace test
} // namespace folly
/* Specialization declarations */
namespace folly {
namespace detail {
template <>
int Futex<test::DeterministicAtomic>::futexWake(int count, uint32_t wakeMask);
template <>
FutexResult Futex<test::DeterministicAtomic>::futexWaitImpl(
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
} // namespace detail
template <> template <>
Getcpu::Func AccessSpreader<test::DeterministicAtomic>::pickGetcpuFunc(); Getcpu::Func AccessSpreader<test::DeterministicAtomic>::pickGetcpuFunc();
......
...@@ -39,19 +39,19 @@ typedef DeterministicSchedule DSched; ...@@ -39,19 +39,19 @@ typedef DeterministicSchedule DSched;
template <template <typename> class Atom> template <template <typename> class Atom>
void run_basic_thread( void run_basic_thread(
Futex<Atom>& f) { Futex<Atom>& f) {
EXPECT_EQ(FutexResult::AWOKEN, f.futexWait(0)); EXPECT_EQ(FutexResult::AWOKEN, futexWait(&f, 0));
} }
template <template <typename> class Atom> template <template <typename> class Atom>
void run_basic_tests() { void run_basic_tests() {
Futex<Atom> f(0); Futex<Atom> f(0);
EXPECT_EQ(FutexResult::VALUE_CHANGED, f.futexWait(1)); EXPECT_EQ(FutexResult::VALUE_CHANGED, futexWait(&f, 1));
EXPECT_EQ(f.futexWake(), 0); EXPECT_EQ(futexWake(&f), 0);
auto thr = DSched::thread(std::bind(run_basic_thread<Atom>, std::ref(f))); auto thr = DSched::thread(std::bind(run_basic_thread<Atom>, std::ref(f)));
while (f.futexWake() != 1) { while (futexWake(&f) != 1) {
std::this_thread::yield(); std::this_thread::yield();
} }
...@@ -68,7 +68,7 @@ void liveClockWaitUntilTests() { ...@@ -68,7 +68,7 @@ void liveClockWaitUntilTests() {
while (true) { while (true) {
const auto deadline = time_point_cast<Duration>( const auto deadline = time_point_cast<Duration>(
Clock::now() + microseconds(1 << (stress % 20))); Clock::now() + microseconds(1 << (stress % 20)));
const auto res = fp->futexWaitUntil(0, deadline); const auto res = futexWaitUntil(fp, 0, deadline);
EXPECT_TRUE(res == FutexResult::TIMEDOUT || res == FutexResult::AWOKEN); EXPECT_TRUE(res == FutexResult::TIMEDOUT || res == FutexResult::AWOKEN);
if (res == FutexResult::AWOKEN) { if (res == FutexResult::AWOKEN) {
break; break;
...@@ -76,7 +76,7 @@ void liveClockWaitUntilTests() { ...@@ -76,7 +76,7 @@ void liveClockWaitUntilTests() {
} }
}); });
while (f.futexWake() != 1) { while (futexWake(&f) != 1) {
std::this_thread::yield(); std::this_thread::yield();
} }
...@@ -86,7 +86,7 @@ void liveClockWaitUntilTests() { ...@@ -86,7 +86,7 @@ void liveClockWaitUntilTests() {
{ {
const auto start = Clock::now(); const auto start = Clock::now();
const auto deadline = time_point_cast<Duration>(start + milliseconds(100)); const auto deadline = time_point_cast<Duration>(start + milliseconds(100));
EXPECT_EQ(f.futexWaitUntil(0, deadline), FutexResult::TIMEDOUT); EXPECT_EQ(futexWaitUntil(&f, 0, deadline), FutexResult::TIMEDOUT);
LOG(INFO) << "Futex wait timed out after waiting for " LOG(INFO) << "Futex wait timed out after waiting for "
<< duration_cast<milliseconds>(Clock::now() - start).count() << duration_cast<milliseconds>(Clock::now() - start).count()
<< "ms using clock with " << Duration::period::den << "ms using clock with " << Duration::period::den
...@@ -97,7 +97,7 @@ void liveClockWaitUntilTests() { ...@@ -97,7 +97,7 @@ void liveClockWaitUntilTests() {
const auto start = Clock::now(); const auto start = Clock::now();
const auto deadline = time_point_cast<Duration>( const auto deadline = time_point_cast<Duration>(
start - 2 * start.time_since_epoch()); start - 2 * start.time_since_epoch());
EXPECT_EQ(f.futexWaitUntil(0, deadline), FutexResult::TIMEDOUT); EXPECT_EQ(futexWaitUntil(&f, 0, deadline), FutexResult::TIMEDOUT);
LOG(INFO) << "Futex wait with invalid deadline timed out after waiting for " LOG(INFO) << "Futex wait with invalid deadline timed out after waiting for "
<< duration_cast<milliseconds>(Clock::now() - start).count() << duration_cast<milliseconds>(Clock::now() - start).count()
<< "ms using clock with " << Duration::period::den << "ms using clock with " << Duration::period::den
...@@ -111,7 +111,7 @@ void deterministicAtomicWaitUntilTests() { ...@@ -111,7 +111,7 @@ void deterministicAtomicWaitUntilTests() {
// Futex wait must eventually fail with either FutexResult::TIMEDOUT or // Futex wait must eventually fail with either FutexResult::TIMEDOUT or
// FutexResult::INTERRUPTED // FutexResult::INTERRUPTED
const auto res = f.futexWaitUntil(0, Clock::now() + milliseconds(100)); const auto res = futexWaitUntil(&f, 0, Clock::now() + milliseconds(100));
EXPECT_TRUE(res == FutexResult::TIMEDOUT || res == FutexResult::INTERRUPTED); EXPECT_TRUE(res == FutexResult::TIMEDOUT || res == FutexResult::INTERRUPTED);
} }
...@@ -192,10 +192,10 @@ void run_wake_blocked_test() { ...@@ -192,10 +192,10 @@ void run_wake_blocked_test() {
bool success = false; bool success = false;
Futex<Atom> f(0); Futex<Atom> f(0);
auto thr = DSched::thread( auto thr = DSched::thread(
[&] { success = FutexResult::AWOKEN == f.futexWait(0); }); [&] { success = FutexResult::AWOKEN == futexWait(&f, 0); });
/* sleep override */ std::this_thread::sleep_for(delay); /* sleep override */ std::this_thread::sleep_for(delay);
f.store(1); f.store(1);
f.futexWake(1); futexWake(&f, 1);
DSched::join(thr); DSched::join(thr);
LOG(INFO) << "delay=" << delay.count() << "_ms, success=" << success; LOG(INFO) << "delay=" << delay.count() << "_ms, success=" << success;
if (success) { if (success) {
......
...@@ -49,16 +49,6 @@ TEST(MemoryIdler, releaseMallocTLS) { ...@@ -49,16 +49,6 @@ TEST(MemoryIdler, releaseMallocTLS) {
delete[] p; delete[] p;
} }
/// MockedAtom gives us a way to select a mocked Futex implementation
/// inside Baton, even though the atom itself isn't exercised by the
/// mocked futex
template <typename T>
struct MockAtom : public std::atomic<T> {
explicit MockAtom(T init = 0) : std::atomic<T>(init) {}
};
/// MockClock is a bit tricky because we are mocking a static function /// MockClock is a bit tricky because we are mocking a static function
/// (now()), so we need to find the corresponding mock instance without /// (now()), so we need to find the corresponding mock instance without
/// extending its scope beyond that of the test. I generally avoid /// extending its scope beyond that of the test. I generally avoid
...@@ -84,29 +74,39 @@ struct MockClock { ...@@ -84,29 +74,39 @@ struct MockClock {
}; };
std::weak_ptr<StrictMock<MockClock>> MockClock::s_mockClockInstance; std::weak_ptr<StrictMock<MockClock>> MockClock::s_mockClockInstance;
static auto const forever = MockClock::time_point::max();
/// MockedAtom gives us a way to select a mocked Futex implementation
/// inside Baton, even though the atom itself isn't exercised by the
/// mocked futex
///
/// Futex<MockAtom> is our mocked futex implementation. Note that the method
/// signatures differ from the real Futex because we have elided unused default
/// params and collapsed templated methods into the used type
template <typename T>
struct MockAtom : public std::atomic<T> {
explicit MockAtom(T init = 0) : std::atomic<T>(init) {}
namespace folly { namespace detail {
/// Futex<MockAtom> is our mocked futex implementation. Note that the
/// method signatures differ from the real Futex because we have elided
/// unused default params and collapsed templated methods into the
/// used type
template <>
struct Futex<MockAtom> {
MOCK_METHOD2(futexWait, FutexResult(uint32_t, uint32_t)); MOCK_METHOD2(futexWait, FutexResult(uint32_t, uint32_t));
MOCK_METHOD3(futexWaitUntil, MOCK_METHOD3(futexWaitUntil,
FutexResult(uint32_t, const MockClock::time_point&, uint32_t)); FutexResult(uint32_t, const MockClock::time_point&, uint32_t));
}; };
} // namespace detail FutexResult
} // namespace folly futexWait(Futex<MockAtom>* futex, uint32_t expected, uint32_t waitMask) {
return futex->futexWait(expected, waitMask);
static auto const forever = MockClock::time_point::max(); }
template <typename Clock, typename Duration>
FutexResult futexWaitUntil(
Futex<MockAtom>* futex,
std::uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask) {
return futex->futexWaitUntil(expected, deadline, waitMask);
}
TEST(MemoryIdler, futexWaitValueChangedEarly) { TEST(MemoryIdler, futexWaitValueChangedEarly) {
StrictMock<Futex<MockAtom>> fut; Futex<MockAtom> fut;
auto clock = MockClock::setup(); auto clock = MockClock::setup();
auto begin = MockClock::time_point(std::chrono::seconds(100)); auto begin = MockClock::time_point(std::chrono::seconds(100));
auto idleTimeout = MemoryIdler::defaultIdleTimeout.load(); auto idleTimeout = MemoryIdler::defaultIdleTimeout.load();
...@@ -121,7 +121,7 @@ TEST(MemoryIdler, futexWaitValueChangedEarly) { ...@@ -121,7 +121,7 @@ TEST(MemoryIdler, futexWaitValueChangedEarly) {
} }
TEST(MemoryIdler, futexWaitValueChangedLate) { TEST(MemoryIdler, futexWaitValueChangedLate) {
StrictMock<Futex<MockAtom>> fut; Futex<MockAtom> fut;
auto clock = MockClock::setup(); auto clock = MockClock::setup();
auto begin = MockClock::time_point(std::chrono::seconds(100)); auto begin = MockClock::time_point(std::chrono::seconds(100));
auto idleTimeout = MemoryIdler::defaultIdleTimeout.load(); auto idleTimeout = MemoryIdler::defaultIdleTimeout.load();
...@@ -138,7 +138,7 @@ TEST(MemoryIdler, futexWaitValueChangedLate) { ...@@ -138,7 +138,7 @@ TEST(MemoryIdler, futexWaitValueChangedLate) {
} }
TEST(MemoryIdler, futexWaitAwokenEarly) { TEST(MemoryIdler, futexWaitAwokenEarly) {
StrictMock<Futex<MockAtom>> fut; Futex<MockAtom> fut;
auto clock = MockClock::setup(); auto clock = MockClock::setup();
auto begin = MockClock::time_point(std::chrono::seconds(100)); auto begin = MockClock::time_point(std::chrono::seconds(100));
auto idleTimeout = MemoryIdler::defaultIdleTimeout.load(); auto idleTimeout = MemoryIdler::defaultIdleTimeout.load();
...@@ -151,7 +151,7 @@ TEST(MemoryIdler, futexWaitAwokenEarly) { ...@@ -151,7 +151,7 @@ TEST(MemoryIdler, futexWaitAwokenEarly) {
} }
TEST(MemoryIdler, futexWaitAwokenLate) { TEST(MemoryIdler, futexWaitAwokenLate) {
StrictMock<Futex<MockAtom>> fut; Futex<MockAtom> fut;
auto clock = MockClock::setup(); auto clock = MockClock::setup();
auto begin = MockClock::time_point(std::chrono::seconds(100)); auto begin = MockClock::time_point(std::chrono::seconds(100));
auto idleTimeout = MemoryIdler::defaultIdleTimeout.load(); auto idleTimeout = MemoryIdler::defaultIdleTimeout.load();
...@@ -168,7 +168,7 @@ TEST(MemoryIdler, futexWaitAwokenLate) { ...@@ -168,7 +168,7 @@ TEST(MemoryIdler, futexWaitAwokenLate) {
} }
TEST(MemoryIdler, futexWaitImmediateFlush) { TEST(MemoryIdler, futexWaitImmediateFlush) {
StrictMock<Futex<MockAtom>> fut; Futex<MockAtom> fut;
auto clock = MockClock::setup(); auto clock = MockClock::setup();
EXPECT_CALL(fut, futexWaitUntil(2, forever, 0xff)) EXPECT_CALL(fut, futexWaitUntil(2, forever, 0xff))
...@@ -180,7 +180,7 @@ TEST(MemoryIdler, futexWaitImmediateFlush) { ...@@ -180,7 +180,7 @@ TEST(MemoryIdler, futexWaitImmediateFlush) {
} }
TEST(MemoryIdler, futexWaitNeverFlush) { TEST(MemoryIdler, futexWaitNeverFlush) {
StrictMock<Futex<MockAtom>> fut; Futex<MockAtom> fut;
auto clock = MockClock::setup(); auto clock = MockClock::setup();
EXPECT_CALL(fut, futexWaitUntil(1, forever, -1)) EXPECT_CALL(fut, futexWaitUntil(1, forever, -1))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment