Commit 6d88461a authored by Woo Xie's avatar Woo Xie Committed by Viswanath Sivakumar

close idle HTTPDownstreamSessions before load shedding

Summary:
when any system resource limit is reached, proxygen reduces the number of idle downstream sessions to accomodate new ones.

Test Plan:
canarying on edge241.01.ams3.  Here is the number of idle connection closed during pre load shedding stage.

https://www.facebook.com/pxlcld/ml7J

Reviewed By: afrind@fb.com

Subscribers: alandau, noamler, fugalh, bmatheny, folly-diffs@, jsedgwick, yfeldblum, chalfant, xning, alexkr

FB internal diff: D2030988

Tasks: 5698711

Signature: t1:2030988:1431369559:ce7328d51c7fd0afa7e9e5c19b0c66736d01fee1
parent c923afa9
......@@ -69,30 +69,30 @@ class AcceptorHandshakeHelper :
socket_->sslAccept(this);
}
virtual void timeoutExpired() noexcept {
virtual void timeoutExpired() noexcept override {
VLOG(4) << "SSL handshake timeout expired";
sslError_ = SSLErrorEnum::TIMEOUT;
dropConnection();
}
virtual void describe(std::ostream& os) const {
virtual void describe(std::ostream& os) const override {
os << "pending handshake on " << clientAddr_;
}
virtual bool isBusy() const {
virtual bool isBusy() const override {
return true;
}
virtual void notifyPendingShutdown() {}
virtual void closeWhenIdle() {}
virtual void notifyPendingShutdown() override {}
virtual void closeWhenIdle() override {}
virtual void dropConnection() {
virtual void dropConnection() override {
VLOG(10) << "Dropping in progress handshake for " << clientAddr_;
socket_->closeNow();
}
virtual void dumpConnectionState(uint8_t loglevel) {
virtual void dumpConnectionState(uint8_t loglevel) override {
}
private:
// AsyncSSLSocket::HandshakeCallback API
virtual void handshakeSuc(AsyncSSLSocket* sock) noexcept {
virtual void handshakeSuc(AsyncSSLSocket* sock) noexcept override {
const unsigned char* nextProto = nullptr;
unsigned nextProtoLength = 0;
......@@ -147,7 +147,7 @@ class AcceptorHandshakeHelper :
}
virtual void handshakeErr(AsyncSSLSocket* sock,
const AsyncSocketException& ex) noexcept {
const AsyncSocketException& ex) noexcept override {
auto elapsedTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - acceptTime_);
VLOG(3) << "SSL handshake error after " << elapsedTime.count() <<
" ms; " << sock->getRawBytesReceived() << " bytes received & " <<
......
......@@ -31,7 +31,8 @@ ConnectionManager::ConnectionManager(EventBase* eventBase,
eventBase_(eventBase),
idleIterator_(conns_.end()),
idleLoopCallback_(this),
timeout_(timeout) {
timeout_(timeout),
idleConnEarlyDropThreshold_(timeout_ / 2) {
}
......@@ -46,7 +47,12 @@ ConnectionManager::addConnection(ManagedConnection* connection,
// We must remove it from that manager before adding it to this one.
oldMgr->removeConnection(connection);
}
conns_.push_back(*connection);
// put the connection into busy part first. This should not matter at all
// because the last callback for an idle connection must be onDeactivated(),
// so the connection must be moved to idle part then.
conns_.push_front(*connection);
connection->setConnectionManager(this);
if (callback_) {
callback_->onConnectionAdded(*this);
......@@ -173,4 +179,55 @@ ConnectionManager::dropAllConnections() {
}
}
void
ConnectionManager::onActivated(ManagedConnection& conn) {
auto it = conns_.iterator_to(conn);
if (it == idleIterator_) {
idleIterator_++;
}
conns_.erase(it);
conns_.push_front(conn);
}
void
ConnectionManager::onDeactivated(ManagedConnection& conn) {
auto it = conns_.iterator_to(conn);
conns_.erase(it);
conns_.push_back(conn);
if (idleIterator_ == conns_.end()) {
idleIterator_--;
}
}
size_t
ConnectionManager::dropIdleConnections(size_t num) {
VLOG(4) << "attempt to drop " << num << " idle connections";
if (idleConnEarlyDropThreshold_ >= timeout_) {
return 0;
}
size_t count = 0;
while(count < num) {
auto it = idleIterator_;
if (it == conns_.end()) {
return count; // no more idle session
}
auto idleTime = it->getIdleTime();
if (idleTime == std::chrono::milliseconds(0) ||
idleTime <= idleConnEarlyDropThreshold_) {
VLOG(4) << "conn's idletime: " << idleTime.count()
<< ", earlyDropThreshold: " << idleConnEarlyDropThreshold_.count()
<< ", attempt to drop " << count << "/" << num;
return count; // idleTime cannot be further reduced
}
ManagedConnection& conn = *it;
idleIterator_++;
conn.timeoutExpired();
count++;
}
return count;
}
}} // folly::wangle
......@@ -30,7 +30,8 @@ namespace folly { namespace wangle {
/**
* A ConnectionManager keeps track of ManagedConnections.
*/
class ConnectionManager: public folly::DelayedDestruction {
class ConnectionManager: public folly::DelayedDestruction,
private ManagedConnection::Callback {
public:
/**
......@@ -135,6 +136,25 @@ class ConnectionManager: public folly::DelayedDestruction {
return timeout_;
}
void setLoweredIdleTimeout(std::chrono::milliseconds timeout) {
CHECK(timeout >= std::chrono::milliseconds(0));
CHECK(timeout <= timeout_);
idleConnEarlyDropThreshold_ = timeout;
}
/**
* try to drop num idle connections to release system resources. Return the
* actual number of dropped idle connections
*/
size_t dropIdleConnections(size_t num);
/**
* ManagedConnection::Callbacks
*/
void onActivated(ManagedConnection& conn);
void onDeactivated(ManagedConnection& conn);
private:
class CloseIdleConnsCallback :
public folly::EventBase::LoopCallback,
......@@ -181,7 +201,11 @@ class ConnectionManager: public folly::DelayedDestruction {
*/
void drainAllConnections();
/** All connections */
/**
* All the managed connections. idleIterator_ seperates them into two parts:
* idle and busy ones. [conns_.begin(), idleIterator_) are the busy ones,
* while [idleIterator_, conns_.end()) are the idle one. Moreover, the idle
* ones are organized in the decreasing idle time order. */
folly::CountedIntrusiveList<
ManagedConnection,&ManagedConnection::listHook_> conns_;
......@@ -199,7 +223,23 @@ class ConnectionManager: public folly::DelayedDestruction {
ManagedConnection,&ManagedConnection::listHook_>::iterator idleIterator_;
CloseIdleConnsCallback idleLoopCallback_;
ShutdownAction action_{ShutdownAction::DRAIN1};
/**
* the default idle timeout for downstream sessions when no system resource
* limit is reached
*/
std::chrono::milliseconds timeout_;
/**
* The idle connections can be closed earlier that their idle timeout when any
* system resource limit is reached. This feature can be considerred as a pre
* load shedding stage for the system, and can be easily disabled by setting
* idleConnEarlyDropThreshold_ to defaultIdleTimeout_. Also,
* idleConnEarlyDropThreshold_ can be used to bottom the idle timeout. That
* is, connection manager will not early drop the idle connections whose idle
* time is less than idleConnEarlyDropThreshold_.
*/
std::chrono::milliseconds idleConnEarlyDropThreshold_;
};
}} // folly::wangle
......@@ -36,6 +36,17 @@ class ManagedConnection:
ManagedConnection();
class Callback {
public:
virtual ~Callback() {}
/* Invoked when this connection becomes busy */
virtual void onActivated(ManagedConnection& conn) = 0;
/* Invoked when a connection becomes idle */
virtual void onDeactivated(ManagedConnection& conn) = 0;
};
// HHWheelTimer::Callback API (left for subclasses to implement).
virtual void timeoutExpired() noexcept = 0;
......@@ -50,6 +61,14 @@ class ManagedConnection:
*/
virtual bool isBusy() const = 0;
/**
* Get the idle time of the connection. If it returning 0, that means the idle
* connections will never be dropped during pre load shedding stage.
*/
virtual std::chrono::milliseconds getIdleTime() const {
return std::chrono::milliseconds(0);
}
/**
* Notify the connection that a shutdown is pending. This method will be
* called at the beginning of graceful shutdown.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment