Commit 3f0064f6 authored by Andrii Grynenko's avatar Andrii Grynenko Committed by Facebook Github Bot 5

Use membarrier in TLRefCount

Summary: membarrier guarantees that there's at most one update to thread-local counter, which collecting thread may not see.

Reviewed By: djwatson

Differential Revision: D3532952

fbshipit-source-id: 6106bfe87c70c5f864573a424662778e20423bbb
parent cf21fddd
...@@ -89,6 +89,7 @@ nobase_follyinclude_HEADERS = \ ...@@ -89,6 +89,7 @@ nobase_follyinclude_HEADERS = \
ExceptionWrapper.h \ ExceptionWrapper.h \
Executor.h \ Executor.h \
EvictingCacheMap.h \ EvictingCacheMap.h \
experimental/AsymmetricMemoryBarrier.h \
experimental/AutoTimer.h \ experimental/AutoTimer.h \
experimental/Bits.h \ experimental/Bits.h \
experimental/BitVectorCoding.h \ experimental/BitVectorCoding.h \
...@@ -273,6 +274,7 @@ nobase_follyinclude_HEADERS = \ ...@@ -273,6 +274,7 @@ nobase_follyinclude_HEADERS = \
portability/String.h \ portability/String.h \
portability/Syslog.h \ portability/Syslog.h \
portability/SysFile.h \ portability/SysFile.h \
portability/SysMembarrier.h \
portability/SysMman.h \ portability/SysMman.h \
portability/SysResource.h \ portability/SysResource.h \
portability/SysStat.h \ portability/SysStat.h \
...@@ -428,6 +430,7 @@ libfolly_la_SOURCES = \ ...@@ -428,6 +430,7 @@ libfolly_la_SOURCES = \
portability/Stdlib.cpp \ portability/Stdlib.cpp \
portability/String.cpp \ portability/String.cpp \
portability/SysFile.cpp \ portability/SysFile.cpp \
portability/SysMembarrier.cpp \
portability/SysMman.cpp \ portability/SysMman.cpp \
portability/SysResource.cpp \ portability/SysResource.cpp \
portability/SysStat.cpp \ portability/SysStat.cpp \
...@@ -450,6 +453,7 @@ libfolly_la_SOURCES = \ ...@@ -450,6 +453,7 @@ libfolly_la_SOURCES = \
TimeoutQueue.cpp \ TimeoutQueue.cpp \
Uri.cpp \ Uri.cpp \
Version.cpp \ Version.cpp \
experimental/AsymmetricMemoryBarrier.cpp \
experimental/bser/Dump.cpp \ experimental/bser/Dump.cpp \
experimental/bser/Load.cpp \ experimental/bser/Load.cpp \
experimental/DynamicParser.cpp \ experimental/DynamicParser.cpp \
......
...@@ -354,3 +354,12 @@ using namespace FOLLY_GFLAGS_NAMESPACE; ...@@ -354,3 +354,12 @@ using namespace FOLLY_GFLAGS_NAMESPACE;
// we will take the next one. // we will take the next one.
#define FOLLY_STATIC_CTOR_PRIORITY_MAX __attribute__((__init_priority__(102))) #define FOLLY_STATIC_CTOR_PRIORITY_MAX __attribute__((__init_priority__(102)))
#endif #endif
namespace folly {
#ifdef __linux__
constexpr auto kIsLinux = true;
#else
constexpr auto kIsLinux = false;
#endif
}
/*
* Copyright 2016 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "AsymmetricMemoryBarrier.h"
#include <folly/Exception.h>
#include <folly/Indestructible.h>
#include <folly/portability/SysMembarrier.h>
#include <folly/portability/SysMman.h>
#include <mutex>
namespace folly {
namespace {
struct DummyPageCreator {
DummyPageCreator() {
get();
}
static void* get() {
static auto ptr =
kIsLinux && !detail::sysMembarrierAvailable() ? create() : nullptr;
return ptr;
}
private:
static void* create() {
auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
checkUnixError(reinterpret_cast<uintptr_t>(ptr), "mmap");
// Lock the memory so it can't get paged out. If it gets paged out, changing
// its protection won't accomplish anything.
auto r = mlock(ptr, 1);
checkUnixError(r, "mlock");
return ptr;
}
};
// Make sure dummy page is always initialized before shutdown
DummyPageCreator dummyPageCreator;
void mprotectMembarrier() {
auto dummyPage = dummyPageCreator.get();
// This function is required to be safe to call on shutdown,
// so we must leak the mutex.
static Indestructible<std::mutex> mprotectMutex;
std::lock_guard<std::mutex> lg(*mprotectMutex);
int r = 0;
r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE);
checkUnixError(r, "mprotect");
r = mprotect(dummyPage, 1, PROT_READ);
checkUnixError(r, "mprotect");
}
}
void asymmetricHeavyBarrier() {
if (kIsLinux) {
static const bool useSysMembarrier = detail::sysMembarrierAvailable();
if (useSysMembarrier) {
auto r = detail::sysMembarrier();
checkUnixError(r, "membarrier");
} else {
mprotectMembarrier();
}
} else {
std::atomic_thread_fence(std::memory_order_seq_cst);
}
}
}
/*
* Copyright 2016 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <atomic>
#include <folly/portability/Asm.h>
namespace folly {
FOLLY_ALWAYS_INLINE void asymmetricLightBarrier() {
if (kIsLinux) {
asm_volatile_memory();
} else {
std::atomic_thread_fence(std::memory_order_seq_cst);
}
}
void asymmetricHeavyBarrier();
}
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#pragma once #pragma once
#include <folly/ThreadLocal.h> #include <folly/ThreadLocal.h>
#include <folly/experimental/AsymmetricMemoryBarrier.h>
namespace folly { namespace folly {
...@@ -84,6 +85,8 @@ class TLRefCount { ...@@ -84,6 +85,8 @@ class TLRefCount {
state_ = State::GLOBAL_TRANSITION; state_ = State::GLOBAL_TRANSITION;
asymmetricHeavyBarrier();
std::weak_ptr<void> collectGuardWeak = collectGuard_; std::weak_ptr<void> collectGuardWeak = collectGuard_;
// Make sure we can't create new LocalRefCounts // Make sure we can't create new LocalRefCounts
...@@ -147,7 +150,14 @@ class TLRefCount { ...@@ -147,7 +150,14 @@ class TLRefCount {
return false; return false;
} }
auto count = count_ += delta; // This is equivalent to atomic fetch_add. We know that this operation
// is always performed from a single thread. asymmetricLightBarrier()
// makes things faster than atomic fetch_add on platforms with native
// support.
auto count = count_.load(std::memory_order_relaxed) + delta;
count_.store(count, std::memory_order_relaxed);
asymmetricLightBarrier();
if (UNLIKELY(refCount_.state_.load() != State::LOCAL)) { if (UNLIKELY(refCount_.state_.load() != State::LOCAL)) {
std::lock_guard<std::mutex> lg(collectMutex_); std::lock_guard<std::mutex> lg(collectMutex_);
......
/*
* Copyright 2016 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "SysMembarrier.h"
#include <mutex>
#include <folly/Portability.h>
#include <folly/portability/Unistd.h>
#if !defined(__NR_membarrier) && defined(FOLLY_X64)
#define __NR_membarrier 324
#define MEMBARRIER_CMD_QUERY 0
#define MEMBARRIER_CMD_SHARED 1
#endif
namespace folly {
namespace detail {
bool sysMembarrierAvailable() {
if (!kIsLinux) {
return false;
}
#ifdef __NR_membarrier
auto r = syscall(__NR_membarrier, MEMBARRIER_CMD_QUERY, /* flags = */ 0);
if (r == -1) {
return false;
}
return r & MEMBARRIER_CMD_SHARED;
#else
return false;
#endif
}
int sysMembarrier() {
#ifdef __NR_membarrier
return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, /* flags = */ 0);
#else
return -1;
#endif
}
}
}
/*
* Copyright 2016 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
namespace folly {
namespace detail {
int sysMembarrier();
bool sysMembarrierAvailable();
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment