Commit 6d88461a authored by Woo Xie's avatar Woo Xie Committed by Viswanath Sivakumar

close idle HTTPDownstreamSessions before load shedding

Summary:
when any system resource limit is reached, proxygen reduces the number of idle downstream sessions to accomodate new ones.

Test Plan:
canarying on edge241.01.ams3.  Here is the number of idle connection closed during pre load shedding stage.

https://www.facebook.com/pxlcld/ml7J

Reviewed By: afrind@fb.com

Subscribers: alandau, noamler, fugalh, bmatheny, folly-diffs@, jsedgwick, yfeldblum, chalfant, xning, alexkr

FB internal diff: D2030988

Tasks: 5698711

Signature: t1:2030988:1431369559:ce7328d51c7fd0afa7e9e5c19b0c66736d01fee1
parent c923afa9
...@@ -69,30 +69,30 @@ class AcceptorHandshakeHelper : ...@@ -69,30 +69,30 @@ class AcceptorHandshakeHelper :
socket_->sslAccept(this); socket_->sslAccept(this);
} }
virtual void timeoutExpired() noexcept { virtual void timeoutExpired() noexcept override {
VLOG(4) << "SSL handshake timeout expired"; VLOG(4) << "SSL handshake timeout expired";
sslError_ = SSLErrorEnum::TIMEOUT; sslError_ = SSLErrorEnum::TIMEOUT;
dropConnection(); dropConnection();
} }
virtual void describe(std::ostream& os) const { virtual void describe(std::ostream& os) const override {
os << "pending handshake on " << clientAddr_; os << "pending handshake on " << clientAddr_;
} }
virtual bool isBusy() const { virtual bool isBusy() const override {
return true; return true;
} }
virtual void notifyPendingShutdown() {} virtual void notifyPendingShutdown() override {}
virtual void closeWhenIdle() {} virtual void closeWhenIdle() override {}
virtual void dropConnection() { virtual void dropConnection() override {
VLOG(10) << "Dropping in progress handshake for " << clientAddr_; VLOG(10) << "Dropping in progress handshake for " << clientAddr_;
socket_->closeNow(); socket_->closeNow();
} }
virtual void dumpConnectionState(uint8_t loglevel) { virtual void dumpConnectionState(uint8_t loglevel) override {
} }
private: private:
// AsyncSSLSocket::HandshakeCallback API // AsyncSSLSocket::HandshakeCallback API
virtual void handshakeSuc(AsyncSSLSocket* sock) noexcept { virtual void handshakeSuc(AsyncSSLSocket* sock) noexcept override {
const unsigned char* nextProto = nullptr; const unsigned char* nextProto = nullptr;
unsigned nextProtoLength = 0; unsigned nextProtoLength = 0;
...@@ -147,7 +147,7 @@ class AcceptorHandshakeHelper : ...@@ -147,7 +147,7 @@ class AcceptorHandshakeHelper :
} }
virtual void handshakeErr(AsyncSSLSocket* sock, virtual void handshakeErr(AsyncSSLSocket* sock,
const AsyncSocketException& ex) noexcept { const AsyncSocketException& ex) noexcept override {
auto elapsedTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - acceptTime_); auto elapsedTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - acceptTime_);
VLOG(3) << "SSL handshake error after " << elapsedTime.count() << VLOG(3) << "SSL handshake error after " << elapsedTime.count() <<
" ms; " << sock->getRawBytesReceived() << " bytes received & " << " ms; " << sock->getRawBytesReceived() << " bytes received & " <<
......
...@@ -31,7 +31,8 @@ ConnectionManager::ConnectionManager(EventBase* eventBase, ...@@ -31,7 +31,8 @@ ConnectionManager::ConnectionManager(EventBase* eventBase,
eventBase_(eventBase), eventBase_(eventBase),
idleIterator_(conns_.end()), idleIterator_(conns_.end()),
idleLoopCallback_(this), idleLoopCallback_(this),
timeout_(timeout) { timeout_(timeout),
idleConnEarlyDropThreshold_(timeout_ / 2) {
} }
...@@ -46,7 +47,12 @@ ConnectionManager::addConnection(ManagedConnection* connection, ...@@ -46,7 +47,12 @@ ConnectionManager::addConnection(ManagedConnection* connection,
// We must remove it from that manager before adding it to this one. // We must remove it from that manager before adding it to this one.
oldMgr->removeConnection(connection); oldMgr->removeConnection(connection);
} }
conns_.push_back(*connection);
// put the connection into busy part first. This should not matter at all
// because the last callback for an idle connection must be onDeactivated(),
// so the connection must be moved to idle part then.
conns_.push_front(*connection);
connection->setConnectionManager(this); connection->setConnectionManager(this);
if (callback_) { if (callback_) {
callback_->onConnectionAdded(*this); callback_->onConnectionAdded(*this);
...@@ -173,4 +179,55 @@ ConnectionManager::dropAllConnections() { ...@@ -173,4 +179,55 @@ ConnectionManager::dropAllConnections() {
} }
} }
void
ConnectionManager::onActivated(ManagedConnection& conn) {
auto it = conns_.iterator_to(conn);
if (it == idleIterator_) {
idleIterator_++;
}
conns_.erase(it);
conns_.push_front(conn);
}
void
ConnectionManager::onDeactivated(ManagedConnection& conn) {
auto it = conns_.iterator_to(conn);
conns_.erase(it);
conns_.push_back(conn);
if (idleIterator_ == conns_.end()) {
idleIterator_--;
}
}
size_t
ConnectionManager::dropIdleConnections(size_t num) {
VLOG(4) << "attempt to drop " << num << " idle connections";
if (idleConnEarlyDropThreshold_ >= timeout_) {
return 0;
}
size_t count = 0;
while(count < num) {
auto it = idleIterator_;
if (it == conns_.end()) {
return count; // no more idle session
}
auto idleTime = it->getIdleTime();
if (idleTime == std::chrono::milliseconds(0) ||
idleTime <= idleConnEarlyDropThreshold_) {
VLOG(4) << "conn's idletime: " << idleTime.count()
<< ", earlyDropThreshold: " << idleConnEarlyDropThreshold_.count()
<< ", attempt to drop " << count << "/" << num;
return count; // idleTime cannot be further reduced
}
ManagedConnection& conn = *it;
idleIterator_++;
conn.timeoutExpired();
count++;
}
return count;
}
}} // folly::wangle }} // folly::wangle
...@@ -30,7 +30,8 @@ namespace folly { namespace wangle { ...@@ -30,7 +30,8 @@ namespace folly { namespace wangle {
/** /**
* A ConnectionManager keeps track of ManagedConnections. * A ConnectionManager keeps track of ManagedConnections.
*/ */
class ConnectionManager: public folly::DelayedDestruction { class ConnectionManager: public folly::DelayedDestruction,
private ManagedConnection::Callback {
public: public:
/** /**
...@@ -135,6 +136,25 @@ class ConnectionManager: public folly::DelayedDestruction { ...@@ -135,6 +136,25 @@ class ConnectionManager: public folly::DelayedDestruction {
return timeout_; return timeout_;
} }
void setLoweredIdleTimeout(std::chrono::milliseconds timeout) {
CHECK(timeout >= std::chrono::milliseconds(0));
CHECK(timeout <= timeout_);
idleConnEarlyDropThreshold_ = timeout;
}
/**
* try to drop num idle connections to release system resources. Return the
* actual number of dropped idle connections
*/
size_t dropIdleConnections(size_t num);
/**
* ManagedConnection::Callbacks
*/
void onActivated(ManagedConnection& conn);
void onDeactivated(ManagedConnection& conn);
private: private:
class CloseIdleConnsCallback : class CloseIdleConnsCallback :
public folly::EventBase::LoopCallback, public folly::EventBase::LoopCallback,
...@@ -181,7 +201,11 @@ class ConnectionManager: public folly::DelayedDestruction { ...@@ -181,7 +201,11 @@ class ConnectionManager: public folly::DelayedDestruction {
*/ */
void drainAllConnections(); void drainAllConnections();
/** All connections */ /**
* All the managed connections. idleIterator_ seperates them into two parts:
* idle and busy ones. [conns_.begin(), idleIterator_) are the busy ones,
* while [idleIterator_, conns_.end()) are the idle one. Moreover, the idle
* ones are organized in the decreasing idle time order. */
folly::CountedIntrusiveList< folly::CountedIntrusiveList<
ManagedConnection,&ManagedConnection::listHook_> conns_; ManagedConnection,&ManagedConnection::listHook_> conns_;
...@@ -199,7 +223,23 @@ class ConnectionManager: public folly::DelayedDestruction { ...@@ -199,7 +223,23 @@ class ConnectionManager: public folly::DelayedDestruction {
ManagedConnection,&ManagedConnection::listHook_>::iterator idleIterator_; ManagedConnection,&ManagedConnection::listHook_>::iterator idleIterator_;
CloseIdleConnsCallback idleLoopCallback_; CloseIdleConnsCallback idleLoopCallback_;
ShutdownAction action_{ShutdownAction::DRAIN1}; ShutdownAction action_{ShutdownAction::DRAIN1};
/**
* the default idle timeout for downstream sessions when no system resource
* limit is reached
*/
std::chrono::milliseconds timeout_; std::chrono::milliseconds timeout_;
/**
* The idle connections can be closed earlier that their idle timeout when any
* system resource limit is reached. This feature can be considerred as a pre
* load shedding stage for the system, and can be easily disabled by setting
* idleConnEarlyDropThreshold_ to defaultIdleTimeout_. Also,
* idleConnEarlyDropThreshold_ can be used to bottom the idle timeout. That
* is, connection manager will not early drop the idle connections whose idle
* time is less than idleConnEarlyDropThreshold_.
*/
std::chrono::milliseconds idleConnEarlyDropThreshold_;
}; };
}} // folly::wangle }} // folly::wangle
...@@ -36,6 +36,17 @@ class ManagedConnection: ...@@ -36,6 +36,17 @@ class ManagedConnection:
ManagedConnection(); ManagedConnection();
class Callback {
public:
virtual ~Callback() {}
/* Invoked when this connection becomes busy */
virtual void onActivated(ManagedConnection& conn) = 0;
/* Invoked when a connection becomes idle */
virtual void onDeactivated(ManagedConnection& conn) = 0;
};
// HHWheelTimer::Callback API (left for subclasses to implement). // HHWheelTimer::Callback API (left for subclasses to implement).
virtual void timeoutExpired() noexcept = 0; virtual void timeoutExpired() noexcept = 0;
...@@ -50,6 +61,14 @@ class ManagedConnection: ...@@ -50,6 +61,14 @@ class ManagedConnection:
*/ */
virtual bool isBusy() const = 0; virtual bool isBusy() const = 0;
/**
* Get the idle time of the connection. If it returning 0, that means the idle
* connections will never be dropped during pre load shedding stage.
*/
virtual std::chrono::milliseconds getIdleTime() const {
return std::chrono::milliseconds(0);
}
/** /**
* Notify the connection that a shutdown is pending. This method will be * Notify the connection that a shutdown is pending. This method will be
* called at the beginning of graceful shutdown. * called at the beginning of graceful shutdown.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment