Commit 95934611 authored by Aaryaman Sagar's avatar Aaryaman Sagar Committed by Facebook Github Bot

Remove inline asm from PicoSpinLock

Summary:
Removes the inline asm from PicoSpinLock and uses the fetch_set and
fetch_reset utilities instead

Reviewed By: yfeldblum

Differential Revision: D9697136

fbshipit-source-id: 022bb2d2d9d7cd6ac8a0011bff4c8995db989950
parent f98c3cdc
...@@ -48,12 +48,9 @@ ...@@ -48,12 +48,9 @@
#include <glog/logging.h> #include <glog/logging.h>
#include <folly/Portability.h> #include <folly/Portability.h>
#include <folly/synchronization/AtomicUtil.h>
#include <folly/synchronization/detail/Sleeper.h> #include <folly/synchronization/detail/Sleeper.h>
#if !FOLLY_X64 && !FOLLY_AARCH64 && !FOLLY_PPC64
#error "PicoSpinLock.h is currently x64, aarch64 and ppc64 only."
#endif
namespace folly { namespace folly {
/* /*
...@@ -134,93 +131,11 @@ struct PicoSpinLock { ...@@ -134,93 +131,11 @@ struct PicoSpinLock {
* got it. * got it.
*/ */
bool try_lock() const { bool try_lock() const {
bool ret = false; auto previous = atomic_fetch_set(
*reinterpret_cast<std::atomic<UIntType>*>(&lock_),
#if defined(FOLLY_SANITIZE_THREAD) Bit,
// TODO: Might be able to fully move to std::atomic when gcc emits lock btr: std::memory_order_acquire);
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=49244 return !previous;
ret =
!(reinterpret_cast<std::atomic<UIntType>*>(&lock_)->fetch_or(
kLockBitMask_, std::memory_order_acquire) &
kLockBitMask_);
#elif _MSC_VER
switch (sizeof(IntType)) {
case 2:
// There is no _interlockedbittestandset16 for some reason :(
ret = _InterlockedOr16((volatile short*)&lock_, (short)kLockBitMask_) &
kLockBitMask_;
break;
case 4:
ret = _interlockedbittestandset((volatile long*)&lock_, Bit);
break;
case 8:
ret = _interlockedbittestandset64((volatile long long*)&lock_, Bit);
break;
}
#elif FOLLY_X64
#define FB_DOBTS(size) \
asm volatile("lock; bts" #size " %1, (%2); setnc %0" \
: "=r"(ret) \
: "i"(Bit), "r"(&lock_) \
: "memory", "flags")
switch (sizeof(IntType)) {
case 2:
FB_DOBTS(w);
break;
case 4:
FB_DOBTS(l);
break;
case 8:
FB_DOBTS(q);
break;
}
#undef FB_DOBTS
#elif FOLLY_AARCH64
using SIntType = typename std::make_signed<UIntType>::type;
auto const lock = reinterpret_cast<SIntType*>(&lock_);
auto const mask = static_cast<SIntType>(kLockBitMask_);
return !(mask & __atomic_fetch_or(lock, mask, __ATOMIC_ACQUIRE));
#elif FOLLY_PPC64
#define FB_DOBTS(size) \
asm volatile( \
"\teieio\n" \
"\tl" #size \
"arx 14,0,%[lockPtr]\n" \
"\tli 15,1\n" \
"\tsldi 15,15,%[bit]\n" \
"\tand. 16,15,14\n" \
"\tbne 0f\n" \
"\tor 14,14,15\n" \
"\tst" #size \
"cx. 14,0,%[lockPtr]\n" \
"\tbne 0f\n" \
"\tori %[output],%[output],1\n" \
"\tisync\n" \
"0:\n" \
: [output] "+r"(ret) \
: [lockPtr] "r"(&lock_), [bit] "i"(Bit) \
: "cr0", "memory", "r14", "r15", "r16")
switch (sizeof(IntType)) {
case 2:
FB_DOBTS(h);
break;
case 4:
FB_DOBTS(w);
break;
case 8:
FB_DOBTS(d);
break;
}
#undef FB_DOBTS
#else
#error "x86 aarch64 ppc64 only"
#endif
return ret;
} }
/* /*
...@@ -238,79 +153,11 @@ struct PicoSpinLock { ...@@ -238,79 +153,11 @@ struct PicoSpinLock {
* integer. * integer.
*/ */
void unlock() const { void unlock() const {
#ifdef _MSC_VER auto previous = atomic_fetch_reset(
switch (sizeof(IntType)) { *reinterpret_cast<std::atomic<UIntType>*>(&lock_),
case 2: Bit,
// There is no _interlockedbittestandreset16 for some reason :( std::memory_order_release);
_InterlockedAnd16((volatile short*)&lock_, (short)~kLockBitMask_); DCHECK(previous);
break;
case 4:
_interlockedbittestandreset((volatile long*)&lock_, Bit);
break;
case 8:
_interlockedbittestandreset64((volatile long long*)&lock_, Bit);
break;
}
#elif FOLLY_X64
#define FB_DOBTR(size) \
asm volatile("lock; btr" #size " %0, (%1)" \
: \
: "i"(Bit), "r"(&lock_) \
: "memory", "flags")
// Reads and writes can not be reordered wrt locked instructions,
// so we don't need a memory fence here.
switch (sizeof(IntType)) {
case 2:
FB_DOBTR(w);
break;
case 4:
FB_DOBTR(l);
break;
case 8:
FB_DOBTR(q);
break;
}
#undef FB_DOBTR
#elif FOLLY_AARCH64
using SIntType = typename std::make_signed<UIntType>::type;
auto const lock = reinterpret_cast<SIntType*>(&lock_);
auto const mask = static_cast<SIntType>(kLockBitMask_);
__atomic_fetch_and(lock, ~mask, __ATOMIC_RELEASE);
#elif FOLLY_PPC64
#define FB_DOBTR(size) \
asm volatile( \
"\teieio\n" \
"0: l" #size \
"arx 14,0,%[lockPtr]\n" \
"\tli 15,1\n" \
"\tsldi 15,15,%[bit]\n" \
"\txor 14,14,15\n" \
"\tst" #size \
"cx. 14,0,%[lockPtr]\n" \
"\tbne 0b\n" \
"\tisync\n" \
: \
: [lockPtr] "r"(&lock_), [bit] "i"(Bit) \
: "cr0", "memory", "r14", "r15")
switch (sizeof(IntType)) {
case 2:
FB_DOBTR(h);
break;
case 4:
FB_DOBTR(w);
break;
case 8:
FB_DOBTR(d);
break;
}
#undef FB_DOBTR
#else
#error "x64 aarch64 ppc64 only"
#endif
} }
}; };
......
...@@ -41,6 +41,11 @@ using std::string; ...@@ -41,6 +41,11 @@ using std::string;
using folly::PicoSpinLock; using folly::PicoSpinLock;
#endif #endif
DEFINE_int64(
stress_test_seconds,
2,
"Number of seconds for which to run stress tests");
namespace { namespace {
struct LockedVal { struct LockedVal {
...@@ -306,3 +311,94 @@ TEST(SmallLocks, MicroLockTryLock) { ...@@ -306,3 +311,94 @@ TEST(SmallLocks, MicroLockTryLock) {
EXPECT_FALSE(lock.try_lock()); EXPECT_FALSE(lock.try_lock());
lock.unlock(); lock.unlock();
} }
namespace {
template <typename Mutex, typename Duration>
void simpleStressTest(Duration duration, int numThreads) {
auto&& mutex = Mutex{};
auto&& data = std::atomic<std::uint64_t>{0};
auto&& threads = std::vector<std::thread>{};
auto&& stop = std::atomic<bool>{true};
for (auto i = 0; i < numThreads; ++i) {
threads.emplace_back([&] {
while (!stop.load(std::memory_order_relaxed)) {
auto lck = std::unique_lock<Mutex>{mutex};
EXPECT_EQ(data.fetch_add(1, std::memory_order_relaxed), 0);
EXPECT_EQ(data.fetch_sub(1, std::memory_order_relaxed), 1);
}
});
}
std::this_thread::sleep_for(duration);
stop.store(true);
for (auto& thread : threads) {
thread.join();
}
}
} // namespace
TEST(SmallLocks, MicroSpinLockStressTestLockTwoThreads) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
simpleStressTest<MicroSpinLock>(duration, 2);
}
TEST(SmallLocks, MicroSpinLockStressTestLockHardwareConcurrency) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
auto threads = std::thread::hardware_concurrency();
simpleStressTest<MicroSpinLock>(duration, threads);
}
TEST(SmallLocks, PicoSpinLockStressTestLockTwoThreads) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
simpleStressTest<PicoSpinLock<std::uint16_t>>(duration, 2);
}
TEST(SmallLocks, PicoSpinLockStressTestLockHardwareConcurrency) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
auto threads = std::thread::hardware_concurrency();
simpleStressTest<PicoSpinLock<std::uint16_t>>(duration, threads);
}
namespace {
template <typename Mutex>
class MutexWrapper {
public:
void lock() {
while (!mutex_.try_lock()) {
}
}
void unlock() {
mutex_.unlock();
}
Mutex mutex_;
};
template <typename Mutex, typename Duration>
void simpleStressTestTryLock(Duration duration, int numThreads) {
simpleStressTest<MutexWrapper<Mutex>>(duration, numThreads);
}
} // namespace
TEST(SmallLocks, MicroSpinLockStressTestTryLockTwoThreads) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
simpleStressTestTryLock<MicroSpinLock>(duration, 2);
}
TEST(SmallLocks, MicroSpinLockStressTestTryLockHardwareConcurrency) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
auto threads = std::thread::hardware_concurrency();
simpleStressTestTryLock<MicroSpinLock>(duration, threads);
}
TEST(SmallLocks, PicoSpinLockStressTestTryLockTwoThreads) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
simpleStressTestTryLock<PicoSpinLock<std::uint16_t>>(duration, 2);
}
TEST(SmallLocks, PicoSpinLockStressTestTryLockHardwareConcurrency) {
auto duration = std::chrono::seconds{FLAGS_stress_test_seconds};
auto threads = std::thread::hardware_concurrency();
simpleStressTestTryLock<PicoSpinLock<std::uint16_t>>(duration, threads);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment