Commit 59e4f0d0 authored by Dave Watson's avatar Dave Watson Committed by Alecs King

use ServerBootstrap

Summary:
Use server bootstrap.

* code reuse
* Allows multiple accept threads easily.

Accept now doesn't happen in the server's eventBase, but a new thread.

I think I hit all the corner cases involving existingSocket, duplex, stopListening(), etc.

There are a lot of minor changes here to support all the cases, let me know if I should split anything to separate diffs

Test Plan: fbconfig -r thrift/lib/cpp2; fbmake runtests

Reviewed By: hans@fb.com

Subscribers: haijunz, yfeldblum, ruibalp, fbcode-common-diffs@, davejwatson, jsedgwick, trunkagent, doug, alandau, bmatheny, mshneer, folly-diffs@, mcduff, andrewcox, fugalh

FB internal diff: D1732895

Tasks: 5788102

Signature: t1:1732895:1423087631:379bbb131c35ce2221801bc7cec226f87ba0b1d9
parent 42f4cd7c
...@@ -225,6 +225,9 @@ nobase_follyinclude_HEADERS = \ ...@@ -225,6 +225,9 @@ nobase_follyinclude_HEADERS = \
wangle/acceptor/ServerSocketConfig.h \ wangle/acceptor/ServerSocketConfig.h \
wangle/acceptor/SocketOptions.h \ wangle/acceptor/SocketOptions.h \
wangle/acceptor/TransportInfo.h \ wangle/acceptor/TransportInfo.h \
wangle/bootstrap/ServerBootstrap.h \
wangle/bootstrap/ServerBootstrap-inl.h \
wangle/bootstrap/ClientBootstrap.h \
wangle/channel/AsyncSocketHandler.h \ wangle/channel/AsyncSocketHandler.h \
wangle/channel/ChannelHandler.h \ wangle/channel/ChannelHandler.h \
wangle/channel/ChannelHandlerContext.h \ wangle/channel/ChannelHandlerContext.h \
...@@ -343,6 +346,7 @@ libfolly_la_SOURCES = \ ...@@ -343,6 +346,7 @@ libfolly_la_SOURCES = \
wangle/acceptor/ManagedConnection.cpp \ wangle/acceptor/ManagedConnection.cpp \
wangle/acceptor/SocketOptions.cpp \ wangle/acceptor/SocketOptions.cpp \
wangle/acceptor/TransportInfo.cpp \ wangle/acceptor/TransportInfo.cpp \
wangle/bootstrap/ServerBootstrap.cpp \
wangle/concurrent/CPUThreadPoolExecutor.cpp \ wangle/concurrent/CPUThreadPoolExecutor.cpp \
wangle/concurrent/Codel.cpp \ wangle/concurrent/Codel.cpp \
wangle/concurrent/IOThreadPoolExecutor.cpp \ wangle/concurrent/IOThreadPoolExecutor.cpp \
......
...@@ -180,6 +180,14 @@ class Acceptor : ...@@ -180,6 +180,14 @@ class Acceptor :
*/ */
void drainAllConnections(); void drainAllConnections();
/**
* Drop all connections.
*
* forceStop() schedules dropAllConnections() to be called in the acceptor's
* thread.
*/
void dropAllConnections();
protected: protected:
friend class AcceptorHandshakeHelper; friend class AcceptorHandshakeHelper;
...@@ -238,14 +246,6 @@ class Acceptor : ...@@ -238,14 +246,6 @@ class Acceptor :
std::chrono::milliseconds acceptLatency, std::chrono::milliseconds acceptLatency,
SSLErrorEnum error) noexcept {} SSLErrorEnum error) noexcept {}
/**
* Drop all connections.
*
* forceStop() schedules dropAllConnections() to be called in the acceptor's
* thread.
*/
void dropAllConnections();
protected: protected:
/** /**
...@@ -340,7 +340,7 @@ class Acceptor : ...@@ -340,7 +340,7 @@ class Acceptor :
class AcceptorFactory { class AcceptorFactory {
public: public:
virtual std::shared_ptr<Acceptor> newAcceptor() = 0; virtual std::shared_ptr<Acceptor> newAcceptor(folly::EventBase*) = 0;
virtual ~AcceptorFactory() = default; virtual ~AcceptorFactory() = default;
}; };
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <glog/logging.h> #include <glog/logging.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <boost/thread.hpp>
using namespace folly::wangle; using namespace folly::wangle;
using namespace folly; using namespace folly;
...@@ -227,3 +228,11 @@ TEST(Bootstrap, SharedThreadPool) { ...@@ -227,3 +228,11 @@ TEST(Bootstrap, SharedThreadPool) {
server.stop(); server.stop();
CHECK(factory->pipelines == 5); CHECK(factory->pipelines == 5);
} }
TEST(Bootstrap, ExistingSocket) {
TestServer server;
auto factory = std::make_shared<TestPipelineFactory>();
server.childPipeline(factory);
folly::AsyncServerSocket::UniquePtr socket(new AsyncServerSocket);
server.bind(std::move(socket));
}
...@@ -53,10 +53,11 @@ class ServerAcceptor : public Acceptor { ...@@ -53,10 +53,11 @@ class ServerAcceptor : public Acceptor {
public: public:
explicit ServerAcceptor( explicit ServerAcceptor(
std::shared_ptr<PipelineFactory<Pipeline>> pipelineFactory) std::shared_ptr<PipelineFactory<Pipeline>> pipelineFactory,
EventBase* base)
: Acceptor(ServerSocketConfig()) : Acceptor(ServerSocketConfig())
, pipelineFactory_(pipelineFactory) { , pipelineFactory_(pipelineFactory) {
Acceptor::init(nullptr, &base_); Acceptor::init(nullptr, base);
} }
/* See Acceptor::onNewConnection for details */ /* See Acceptor::onNewConnection for details */
...@@ -74,13 +75,7 @@ class ServerAcceptor : public Acceptor { ...@@ -74,13 +75,7 @@ class ServerAcceptor : public Acceptor {
Acceptor::addConnection(connection); Acceptor::addConnection(connection);
} }
~ServerAcceptor() {
Acceptor::dropAllConnections();
}
private: private:
EventBase base_;
std::shared_ptr<PipelineFactory<Pipeline>> pipelineFactory_; std::shared_ptr<PipelineFactory<Pipeline>> pipelineFactory_;
}; };
...@@ -91,41 +86,51 @@ class ServerAcceptorFactory : public AcceptorFactory { ...@@ -91,41 +86,51 @@ class ServerAcceptorFactory : public AcceptorFactory {
std::shared_ptr<PipelineFactory<Pipeline>> factory) std::shared_ptr<PipelineFactory<Pipeline>> factory)
: factory_(factory) {} : factory_(factory) {}
std::shared_ptr<Acceptor> newAcceptor() { std::shared_ptr<Acceptor> newAcceptor(folly::EventBase* base) {
return std::make_shared<ServerAcceptor<Pipeline>>(factory_); return std::make_shared<ServerAcceptor<Pipeline>>(factory_, base);
} }
private: private:
std::shared_ptr<PipelineFactory<Pipeline>> factory_; std::shared_ptr<PipelineFactory<Pipeline>> factory_;
}; };
class ServerWorkerFactory : public folly::wangle::ThreadFactory { class ServerWorkerPool : public folly::wangle::ThreadPoolExecutor::Observer {
public: public:
explicit ServerWorkerFactory(std::shared_ptr<AcceptorFactory> acceptorFactory) explicit ServerWorkerPool(
: internalFactory_( std::shared_ptr<AcceptorFactory> acceptorFactory,
std::make_shared<folly::wangle::NamedThreadFactory>("BootstrapWorker")) folly::wangle::IOThreadPoolExecutor* exec,
, acceptorFactory_(acceptorFactory) std::vector<std::shared_ptr<folly::AsyncServerSocket>>* sockets)
{} : acceptorFactory_(acceptorFactory)
virtual std::thread newThread(folly::Func&& func) override; , exec_(exec)
, sockets_(sockets) {
void setInternalFactory( CHECK(exec);
std::shared_ptr<folly::wangle::NamedThreadFactory> internalFactory); }
void setNamePrefix(folly::StringPiece prefix);
template <typename F> template <typename F>
void forEachWorker(F&& f); void forEachWorker(F&& f) const;
void threadStarted(
folly::wangle::ThreadPoolExecutor::ThreadHandle*);
void threadStopped(
folly::wangle::ThreadPoolExecutor::ThreadHandle*);
void threadPreviouslyStarted(
folly::wangle::ThreadPoolExecutor::ThreadHandle* thread) {
threadStarted(thread);
}
void threadNotYetStopped(
folly::wangle::ThreadPoolExecutor::ThreadHandle* thread) {
threadStopped(thread);
}
private: private:
std::shared_ptr<folly::wangle::NamedThreadFactory> internalFactory_; std::map<folly::wangle::ThreadPoolExecutor::ThreadHandle*,
folly::RWSpinLock workersLock_; std::shared_ptr<Acceptor>> workers_;
std::map<int32_t, std::shared_ptr<Acceptor>> workers_;
int32_t nextWorkerId_{0};
std::shared_ptr<AcceptorFactory> acceptorFactory_; std::shared_ptr<AcceptorFactory> acceptorFactory_;
folly::wangle::IOThreadPoolExecutor* exec_{nullptr};
std::vector<std::shared_ptr<folly::AsyncServerSocket>>* sockets_;
}; };
template <typename F> template <typename F>
void ServerWorkerFactory::forEachWorker(F&& f) { void ServerWorkerPool::forEachWorker(F&& f) const {
folly::RWSpinLock::ReadHolder guard(workersLock_);
for (const auto& kv : workers_) { for (const auto& kv : workers_) {
f(kv.second.get()); f(kv.second.get());
} }
......
...@@ -19,36 +19,42 @@ ...@@ -19,36 +19,42 @@
namespace folly { namespace folly {
std::thread ServerWorkerFactory::newThread( void ServerWorkerPool::threadStarted(
folly::Func&& func) { folly::wangle::ThreadPoolExecutor::ThreadHandle* h) {
auto id = nextWorkerId_++; auto worker = acceptorFactory_->newAcceptor(exec_->getEventBase(h));
auto worker = acceptorFactory_->newAcceptor(); workers_.insert({h, worker});
{
folly::RWSpinLock::WriteHolder guard(workersLock_); for(auto socket : *sockets_) {
workers_.insert({id, worker}); socket->getEventBase()->runInEventBaseThread([this, worker, socket](){
socket->addAcceptCallback(worker.get(), worker->getEventBase());
});
} }
return internalFactory_->newThread([=](){
EventBaseManager::get()->setEventBase(worker->getEventBase(), false);
func();
EventBaseManager::get()->clearEventBase();
worker->drainAllConnections();
{
folly::RWSpinLock::WriteHolder guard(workersLock_);
workers_.erase(id);
}
});
} }
void ServerWorkerFactory::setInternalFactory( void ServerWorkerPool::threadStopped(
std::shared_ptr<wangle::NamedThreadFactory> internalFactory) { folly::wangle::ThreadPoolExecutor::ThreadHandle* h) {
CHECK(workers_.empty()); auto worker = workers_.find(h);
internalFactory_ = internalFactory; CHECK(worker != workers_.end());
}
for (auto& socket : *sockets_) {
folly::Baton<> barrier;
socket->getEventBase()->runInEventBaseThread([&]() {
socket->removeAcceptCallback(worker->second.get(), nullptr);
barrier.post();
});
barrier.wait();
}
CHECK(worker->second->getEventBase() != nullptr);
CHECK(!worker->second->getEventBase()->isInEventBaseThread());
folly::Baton<> barrier;
worker->second->getEventBase()->runInEventBaseThread([&]() {
worker->second->dropAllConnections();
barrier.post();
});
void ServerWorkerFactory::setNamePrefix(folly::StringPiece prefix) { barrier.wait();
CHECK(workers_.empty()); workers_.erase(worker);
internalFactory_->setNamePrefix(prefix);
} }
} // namespace } // namespace
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
#pragma once #pragma once
#include <folly/wangle/bootstrap/ServerBootstrap-inl.h> #include <folly/wangle/bootstrap/ServerBootstrap-inl.h>
#include <boost/thread.hpp> #include <folly/Baton.h>
namespace folly { namespace folly {
...@@ -37,6 +37,10 @@ namespace folly { ...@@ -37,6 +37,10 @@ namespace folly {
template <typename Pipeline> template <typename Pipeline>
class ServerBootstrap { class ServerBootstrap {
public: public:
~ServerBootstrap() {
stop();
}
/* TODO(davejwatson) /* TODO(davejwatson)
* *
* If there is any work to be done BEFORE handing the work to IO * If there is any work to be done BEFORE handing the work to IO
...@@ -103,34 +107,65 @@ class ServerBootstrap { ...@@ -103,34 +107,65 @@ class ServerBootstrap {
io_group = std::make_shared<folly::wangle::IOThreadPoolExecutor>( io_group = std::make_shared<folly::wangle::IOThreadPoolExecutor>(
32, std::make_shared<wangle::NamedThreadFactory>("IO Thread")); 32, std::make_shared<wangle::NamedThreadFactory>("IO Thread"));
} }
auto factoryBase = io_group->getThreadFactory();
CHECK(factoryBase);
auto factory = std::dynamic_pointer_cast<folly::wangle::NamedThreadFactory>(
factoryBase);
CHECK(factory); // Must be named thread factory
CHECK(acceptorFactory_ || pipelineFactory_); CHECK(acceptorFactory_ || pipelineFactory_);
if (acceptorFactory_) { if (acceptorFactory_) {
workerFactory_ = std::make_shared<ServerWorkerFactory>( workerFactory_ = std::make_shared<ServerWorkerPool>(
acceptorFactory_); acceptorFactory_, io_group.get(), &sockets_);
} else { } else {
workerFactory_ = std::make_shared<ServerWorkerFactory>( workerFactory_ = std::make_shared<ServerWorkerPool>(
std::make_shared<ServerAcceptorFactory<Pipeline>>(pipelineFactory_)); std::make_shared<ServerAcceptorFactory<Pipeline>>(pipelineFactory_),
io_group.get(), &sockets_);
} }
workerFactory_->setInternalFactory(factory);
io_group->addObserver(workerFactory_);
acceptor_group_ = accept_group; acceptor_group_ = accept_group;
io_group_ = io_group; io_group_ = io_group;
auto numThreads = io_group_->numThreads();
io_group_->setNumThreads(0);
io_group_->setThreadFactory(workerFactory_);
io_group_->setNumThreads(numThreads);
return this; return this;
} }
/*
* Bind to an existing socket
*
* @param sock Existing socket to use for accepting
*/
void bind(folly::AsyncServerSocket::UniquePtr s) {
if (!workerFactory_) {
group(nullptr);
}
// Since only a single socket is given,
// we can only accept on a single thread
CHECK(acceptor_group_->numThreads() == 1);
std::shared_ptr<folly::AsyncServerSocket> socket(
s.release(), DelayedDestruction::Destructor());
folly::Baton<> barrier;
acceptor_group_->add([&](){
socket->attachEventBase(EventBaseManager::get()->getEventBase());
socket->listen(1024);
socket->startAccepting();
barrier.post();
});
barrier.wait();
// Startup all the threads
workerFactory_->forEachWorker([this, socket](Acceptor* worker){
socket->getEventBase()->runInEventBaseThread([this, worker, socket](){
socket->addAcceptCallback(worker, worker->getEventBase());
});
});
sockets_.push_back(socket);
}
void bind(folly::SocketAddress address) {
bindImpl(-1, address);
}
/* /*
* Bind to a port and start listening. * Bind to a port and start listening.
* One of childPipeline or childHandler must be called before bind * One of childPipeline or childHandler must be called before bind
...@@ -138,8 +173,11 @@ class ServerBootstrap { ...@@ -138,8 +173,11 @@ class ServerBootstrap {
* @param port Port to listen on * @param port Port to listen on
*/ */
void bind(int port) { void bind(int port) {
// TODO take existing socket CHECK(port >= 0);
bindImpl(port, folly::SocketAddress());
}
void bindImpl(int port, folly::SocketAddress address) {
if (!workerFactory_) { if (!workerFactory_) {
group(nullptr); group(nullptr);
} }
...@@ -152,16 +190,20 @@ class ServerBootstrap { ...@@ -152,16 +190,20 @@ class ServerBootstrap {
std::mutex sock_lock; std::mutex sock_lock;
std::vector<std::shared_ptr<folly::AsyncServerSocket>> new_sockets; std::vector<std::shared_ptr<folly::AsyncServerSocket>> new_sockets;
auto startupFunc = [&](std::shared_ptr<boost::barrier> barrier){ auto startupFunc = [&](std::shared_ptr<folly::Baton<>> barrier){
auto socket = folly::AsyncServerSocket::newSocket(); auto socket = folly::AsyncServerSocket::newSocket();
sock_lock.lock(); sock_lock.lock();
new_sockets.push_back(socket); new_sockets.push_back(socket);
sock_lock.unlock(); sock_lock.unlock();
socket->setReusePortEnabled(reusePort); socket->setReusePortEnabled(reusePort);
socket->attachEventBase(EventBaseManager::get()->getEventBase()); socket->attachEventBase(EventBaseManager::get()->getEventBase());
socket->bind(port); if (port >= 0) {
// TODO Take ServerSocketConfig socket->bind(port);
socket->listen(1024); } else {
socket->bind(address);
port = address.getPort();
}
socket->listen(socketConfig.acceptBacklog);
socket->startAccepting(); socket->startAccepting();
if (port == 0) { if (port == 0) {
...@@ -170,18 +212,18 @@ class ServerBootstrap { ...@@ -170,18 +212,18 @@ class ServerBootstrap {
port = address.getPort(); port = address.getPort();
} }
barrier->wait(); barrier->post();
}; };
auto bind0 = std::make_shared<boost::barrier>(2); auto wait0 = std::make_shared<folly::Baton<>>();
acceptor_group_->add(std::bind(startupFunc, bind0)); acceptor_group_->add(std::bind(startupFunc, wait0));
bind0->wait(); wait0->wait();
auto barrier = std::make_shared<boost::barrier>(acceptor_group_->numThreads());
for (size_t i = 1; i < acceptor_group_->numThreads(); i++) { for (size_t i = 1; i < acceptor_group_->numThreads(); i++) {
auto barrier = std::make_shared<folly::Baton<>>();
acceptor_group_->add(std::bind(startupFunc, barrier)); acceptor_group_->add(std::bind(startupFunc, barrier));
barrier->wait();
} }
barrier->wait();
// Startup all the threads // Startup all the threads
for(auto socket : new_sockets) { for(auto socket : new_sockets) {
...@@ -201,34 +243,49 @@ class ServerBootstrap { ...@@ -201,34 +243,49 @@ class ServerBootstrap {
* Stop listening on all sockets. * Stop listening on all sockets.
*/ */
void stop() { void stop() {
auto barrier = std::make_shared<boost::barrier>(sockets_.size() + 1);
for (auto socket : sockets_) { for (auto socket : sockets_) {
socket->getEventBase()->runInEventBaseThread([barrier, socket]() { folly::Baton<> barrier;
socket->getEventBase()->runInEventBaseThread([&barrier, socket]() {
socket->stopAccepting(); socket->stopAccepting();
socket->detachEventBase(); socket->detachEventBase();
barrier->wait(); barrier.post();
}); });
barrier.wait();
} }
barrier->wait();
sockets_.clear(); sockets_.clear();
acceptor_group_->join(); if (acceptor_group_) {
io_group_->join(); acceptor_group_->join();
}
if (io_group_) {
io_group_->join();
}
} }
/* /*
* Get the list of listening sockets * Get the list of listening sockets
*/ */
std::vector<std::shared_ptr<folly::AsyncServerSocket>>& const std::vector<std::shared_ptr<folly::AsyncServerSocket>>&
getSockets() { getSockets() const {
return sockets_; return sockets_;
} }
std::shared_ptr<wangle::IOThreadPoolExecutor> getIOGroup() const {
return io_group_;
}
template <typename F>
void forEachWorker(F&& f) const {
workerFactory_->forEachWorker(f);
}
ServerSocketConfig socketConfig;
private: private:
std::shared_ptr<wangle::IOThreadPoolExecutor> acceptor_group_; std::shared_ptr<wangle::IOThreadPoolExecutor> acceptor_group_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_group_; std::shared_ptr<wangle::IOThreadPoolExecutor> io_group_;
std::shared_ptr<ServerWorkerFactory> workerFactory_; std::shared_ptr<ServerWorkerPool> workerFactory_;
std::vector<std::shared_ptr<folly::AsyncServerSocket>> sockets_; std::vector<std::shared_ptr<folly::AsyncServerSocket>> sockets_;
std::shared_ptr<AcceptorFactory> acceptorFactory_; std::shared_ptr<AcceptorFactory> acceptorFactory_;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment