Commit 5d134e78 authored by Robin Cheng's avatar Robin Cheng Committed by Facebook GitHub Bot

Attempt to fix a race condition around fiber stats

Summary: Mcrouter has a logger thread which queries the fiber manager for 3 stats; these stats are non-atomic and therefore this causes a race. This diff applies a simple fix to make these atomic. It does not feel like this is the best fix though, so let me know if anyone has ideas for a better fix. It feels that stats should be provided some other way (rather than a bunch of atomics)...

Reviewed By: yfeldblum

Differential Revision: D22878457

fbshipit-source-id: f17544b38836c454162b83cfa570c90c5e7e6b9a
parent f893bb0a
...@@ -120,8 +120,7 @@ void Fiber::recordStackPosition() { ...@@ -120,8 +120,7 @@ void Fiber::recordStackPosition() {
auto currentPosition = static_cast<size_t>( auto currentPosition = static_cast<size_t>(
fiberStackLimit_ + fiberStackSize_ - fiberStackLimit_ + fiberStackSize_ -
static_cast<unsigned char*>(static_cast<void*>(&stackDummy))); static_cast<unsigned char*>(static_cast<void*>(&stackDummy)));
fiberManager_.stackHighWatermark_ = fiberManager_.recordStackPosition(currentPosition);
std::max(fiberManager_.stackHighWatermark_, currentPosition);
VLOG(4) << "Stack usage: " << currentPosition; VLOG(4) << "Stack usage: " << currentPosition;
#endif #endif
} }
...@@ -154,13 +153,10 @@ void Fiber::recordStackPosition() { ...@@ -154,13 +153,10 @@ void Fiber::recordStackPosition() {
} }
if (UNLIKELY(recordStackUsed_)) { if (UNLIKELY(recordStackUsed_)) {
fiberManager_.stackHighWatermark_ = std::max( auto newHighWatermark = fiberManager_.recordStackPosition(
fiberManager_.stackHighWatermark_,
nonMagicInBytes(fiberStackLimit_, fiberStackSize_)); nonMagicInBytes(fiberStackLimit_, fiberStackSize_));
VLOG(3) << "Max stack usage: " << fiberManager_.stackHighWatermark_; VLOG(3) << "Max stack usage: " << newHighWatermark;
CHECK( CHECK_LT(newHighWatermark, fiberManager_.options_.stackSize - 64)
fiberManager_.stackHighWatermark_ <
fiberManager_.options_.stackSize - 64)
<< "Fiber stack overflow"; << "Fiber stack overflow";
} }
......
...@@ -127,12 +127,13 @@ Fiber* FiberManager::getFiber() { ...@@ -127,12 +127,13 @@ Fiber* FiberManager::getFiber() {
if (fibersPool_.empty()) { if (fibersPool_.empty()) {
fiber = new Fiber(*this); fiber = new Fiber(*this);
++fibersAllocated_; fibersAllocated_.store(fibersAllocated() + 1, std::memory_order_relaxed);
} else { } else {
fiber = &fibersPool_.front(); fiber = &fibersPool_.front();
fibersPool_.pop_front(); fibersPool_.pop_front();
assert(fibersPoolSize_ > 0); auto fibersPoolSize = fibersPoolSize_.load(std::memory_order_relaxed);
--fibersPoolSize_; assert(fibersPoolSize > 0);
fibersPoolSize_.store(fibersPoolSize - 1, std::memory_order_relaxed);
} }
assert(fiber); assert(fiber);
if (++fibersActive_ > maxFibersActiveLastPeriod_) { if (++fibersActive_ > maxFibersActiveLastPeriod_) {
...@@ -151,15 +152,15 @@ void FiberManager::setExceptionCallback(FiberManager::ExceptionCallback ec) { ...@@ -151,15 +152,15 @@ void FiberManager::setExceptionCallback(FiberManager::ExceptionCallback ec) {
} }
size_t FiberManager::fibersAllocated() const { size_t FiberManager::fibersAllocated() const {
return fibersAllocated_; return fibersAllocated_.load(std::memory_order_relaxed);
} }
size_t FiberManager::fibersPoolSize() const { size_t FiberManager::fibersPoolSize() const {
return fibersPoolSize_; return fibersPoolSize_.load(std::memory_order_relaxed);
} }
size_t FiberManager::stackHighWatermark() const { size_t FiberManager::stackHighWatermark() const {
return stackHighWatermark_; return stackHighWatermark_.load(std::memory_order_relaxed);
} }
void FiberManager::remoteReadyInsert(Fiber* fiber) { void FiberManager::remoteReadyInsert(Fiber* fiber) {
...@@ -184,14 +185,19 @@ void FiberManager::setPreemptRunner(InlineFunctionRunner* preemptRunner) { ...@@ -184,14 +185,19 @@ void FiberManager::setPreemptRunner(InlineFunctionRunner* preemptRunner) {
} }
void FiberManager::doFibersPoolResizing() { void FiberManager::doFibersPoolResizing() {
while (fibersAllocated_ > maxFibersActiveLastPeriod_ && while (true) {
fibersPoolSize_ > options_.maxFibersPoolSize) { auto fibersAllocated = this->fibersAllocated();
auto fibersPoolSize = this->fibersPoolSize();
if (!(fibersAllocated > maxFibersActiveLastPeriod_ &&
fibersPoolSize > options_.maxFibersPoolSize)) {
break;
}
auto fiber = &fibersPool_.front(); auto fiber = &fibersPool_.front();
assert(fiber != nullptr); assert(fiber != nullptr);
fibersPool_.pop_front(); fibersPool_.pop_front();
delete fiber; delete fiber;
--fibersPoolSize_; fibersPoolSize_.store(fibersPoolSize - 1, std::memory_order_relaxed);
--fibersAllocated_; fibersAllocated_.store(fibersAllocated - 1, std::memory_order_relaxed);
} }
maxFibersActiveLastPeriod_ = fibersActive_; maxFibersActiveLastPeriod_ = fibersActive_;
......
...@@ -234,6 +234,12 @@ void FiberManager::runFibersHelper(LoopFunc&& loopFunc) { ...@@ -234,6 +234,12 @@ void FiberManager::runFibersHelper(LoopFunc&& loopFunc) {
loopFunc(); loopFunc();
} }
inline size_t FiberManager::recordStackPosition(size_t position) {
auto newPosition = std::max(stackHighWatermark(), position);
stackHighWatermark_.store(newPosition, std::memory_order_relaxed);
return newPosition;
}
inline void FiberManager::loopUntilNoReadyImpl() { inline void FiberManager::loopUntilNoReadyImpl() {
runFibersHelper([&] { runFibersHelper([&] {
SCOPE_EXIT { SCOPE_EXIT {
......
...@@ -465,6 +465,8 @@ class FiberManager : public ::folly::Executor { ...@@ -465,6 +465,8 @@ class FiberManager : public ::folly::Executor {
template <typename LoopFunc> template <typename LoopFunc>
void runFibersHelper(LoopFunc&& loopFunc); void runFibersHelper(LoopFunc&& loopFunc);
size_t recordStackPosition(size_t position);
typedef folly::IntrusiveList<Fiber, &Fiber::listHook_> FiberTailQueue; typedef folly::IntrusiveList<Fiber, &Fiber::listHook_> FiberTailQueue;
typedef folly::IntrusiveList<Fiber, &Fiber::globalListHook_> typedef folly::IntrusiveList<Fiber, &Fiber::globalListHook_>
GlobalFiberTailQueue; GlobalFiberTailQueue;
...@@ -483,8 +485,10 @@ class FiberManager : public ::folly::Executor { ...@@ -483,8 +485,10 @@ class FiberManager : public ::folly::Executor {
GlobalFiberTailQueue allFibers_; /**< list of all Fiber objects owned */ GlobalFiberTailQueue allFibers_; /**< list of all Fiber objects owned */
size_t fibersAllocated_{0}; /**< total number of fibers allocated */ // total number of fibers allocated
size_t fibersPoolSize_{0}; /**< total number of fibers in the free pool */ std::atomic<size_t> fibersAllocated_{0};
// total number of fibers in the free pool
std::atomic<size_t> fibersPoolSize_{0};
size_t fibersActive_{0}; /**< number of running or blocked fibers */ size_t fibersActive_{0}; /**< number of running or blocked fibers */
size_t fiberId_{0}; /**< id of last fiber used */ size_t fiberId_{0}; /**< id of last fiber used */
...@@ -514,7 +518,7 @@ class FiberManager : public ::folly::Executor { ...@@ -514,7 +518,7 @@ class FiberManager : public ::folly::Executor {
/** /**
* Largest observed individual Fiber stack usage in bytes. * Largest observed individual Fiber stack usage in bytes.
*/ */
size_t stackHighWatermark_{0}; std::atomic<size_t> stackHighWatermark_{0};
/** /**
* Schedules a loop with loopController (unless already scheduled before). * Schedules a loop with loopController (unless already scheduled before).
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment