Commit 06557e1f authored by Woo Xie's avatar Woo Xie Committed by Sara Golemon

opt proxygen with newly added OpenSSL functions

Summary: this taks advantage of SSL_write_iovec and SSL_read_buf functions to improve CPU.  This eliminates one malloc+memcpy+free operation for all HTTPS traffic, and save up to two for small packets.

It saves 10~15% CPU
https://fburl.com/99681397
https://fburl.com/99427544

Reviewed By: @djwatson

Differential Revision: D1993764
parent df324b0a
...@@ -1070,6 +1070,22 @@ AsyncSSLSocket::handleConnect() noexcept { ...@@ -1070,6 +1070,22 @@ AsyncSSLSocket::handleConnect() noexcept {
AsyncSocket::handleInitialReadWrite(); AsyncSocket::handleInitialReadWrite();
} }
void AsyncSSLSocket::prepareReadBuffer(void** buf, size_t* buflen) noexcept {
CHECK(readCallback_);
#ifdef SSL_MODE_MOVE_BUFFER_OWNERSHIP
// turn on the buffer movable in openssl
if (!isBufferMovable_ && readCallback_->isBufferMovable()) {
SSL_set_mode(ssl_, SSL_get_mode(ssl_) | SSL_MODE_MOVE_BUFFER_OWNERSHIP);
*buf = nullptr;
*buflen = 0;
isBufferMovable_ = true;
return;
}
#endif
// buf is necessary for SSLSocket without SSL_MODE_MOVE_BUFFER_OWNERSHIP
readCallback_->getReadBuffer(buf, buflen);
}
void void
AsyncSSLSocket::handleRead() noexcept { AsyncSSLSocket::handleRead() noexcept {
VLOG(5) << "AsyncSSLSocket::handleRead() this=" << this << ", fd=" << fd_ VLOG(5) << "AsyncSSLSocket::handleRead() this=" << this << ", fd=" << fd_
...@@ -1096,13 +1112,25 @@ AsyncSSLSocket::handleRead() noexcept { ...@@ -1096,13 +1112,25 @@ AsyncSSLSocket::handleRead() noexcept {
} }
ssize_t ssize_t
AsyncSSLSocket::performRead(void* buf, size_t buflen) { AsyncSSLSocket::performRead(void** buf, size_t* buflen, size_t* offset) {
VLOG(4) << "AsyncSSLSocket::performRead() this=" << this
<< ", buf=" << *buf << ", buflen=" << *buflen;
if (sslState_ == STATE_UNENCRYPTED) { if (sslState_ == STATE_UNENCRYPTED) {
return AsyncSocket::performRead(buf, buflen); return AsyncSocket::performRead(buf, buflen, offset);
} }
errno = 0; errno = 0;
ssize_t bytes = SSL_read(ssl_, buf, buflen); ssize_t bytes = 0;
if (!isBufferMovable_) {
bytes = SSL_read(ssl_, *buf, *buflen);
}
#ifdef SSL_MODE_MOVE_BUFFER_OWNERSHIP
else {
bytes = SSL_read_buf(ssl_, buf, (int *) offset, (int *) buflen);
}
#endif
if (server_ && renegotiateAttempted_) { if (server_ && renegotiateAttempted_) {
LOG(ERROR) << "AsyncSSLSocket(fd=" << fd_ << ", state=" << int(state_) LOG(ERROR) << "AsyncSSLSocket(fd=" << fd_ << ", state=" << int(state_)
<< ", sslstate=" << sslState_ << ", events=" << eventFlags_ << ", sslstate=" << sslState_ << ", events=" << eventFlags_
......
...@@ -673,7 +673,7 @@ class AsyncSSLSocket : public virtual AsyncSocket { ...@@ -673,7 +673,7 @@ class AsyncSSLSocket : public virtual AsyncSocket {
// Inherit event notification methods from AsyncSocket except // Inherit event notification methods from AsyncSocket except
// the following. // the following.
void prepareReadBuffer(void** buf, size_t* buflen) noexcept override;
void handleRead() noexcept override; void handleRead() noexcept override;
void handleWrite() noexcept override; void handleWrite() noexcept override;
void handleAccept() noexcept; void handleAccept() noexcept;
...@@ -687,7 +687,7 @@ class AsyncSSLSocket : public virtual AsyncSocket { ...@@ -687,7 +687,7 @@ class AsyncSSLSocket : public virtual AsyncSocket {
void handleInitialReadWrite() noexcept override {} void handleInitialReadWrite() noexcept override {}
int interpretSSLError(int rc, int error); int interpretSSLError(int rc, int error);
ssize_t performRead(void* buf, size_t buflen) override; ssize_t performRead(void** buf, size_t* buflen, size_t* offset) override;
ssize_t performWrite(const iovec* vec, uint32_t count, WriteFlags flags, ssize_t performWrite(const iovec* vec, uint32_t count, WriteFlags flags,
uint32_t* countWritten, uint32_t* partialWritten) uint32_t* countWritten, uint32_t* partialWritten)
override; override;
......
...@@ -1231,8 +1231,11 @@ void AsyncSocket::ioReady(uint16_t events) noexcept { ...@@ -1231,8 +1231,11 @@ void AsyncSocket::ioReady(uint16_t events) noexcept {
} }
} }
ssize_t AsyncSocket::performRead(void* buf, size_t buflen) { ssize_t AsyncSocket::performRead(void** buf, size_t* buflen, size_t* offset) {
ssize_t bytes = recv(fd_, buf, buflen, MSG_DONTWAIT); VLOG(5) << "AsyncSocket::performRead() this=" << this
<< ", buf=" << *buf << ", buflen=" << *buflen;
ssize_t bytes = recv(fd_, *buf, *buflen, MSG_DONTWAIT);
if (bytes < 0) { if (bytes < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) { if (errno == EAGAIN || errno == EWOULDBLOCK) {
// No more data to read right now. // No more data to read right now.
...@@ -1246,6 +1249,12 @@ ssize_t AsyncSocket::performRead(void* buf, size_t buflen) { ...@@ -1246,6 +1249,12 @@ ssize_t AsyncSocket::performRead(void* buf, size_t buflen) {
} }
} }
void AsyncSocket::prepareReadBuffer(void** buf, size_t* buflen) noexcept {
// no matter what, buffer should be preapared for non-ssl socket
CHECK(readCallback_);
readCallback_->getReadBuffer(buf, buflen);
}
void AsyncSocket::handleRead() noexcept { void AsyncSocket::handleRead() noexcept {
VLOG(5) << "AsyncSocket::handleRead() this=" << this << ", fd=" << fd_ VLOG(5) << "AsyncSocket::handleRead() this=" << this << ", fd=" << fd_
<< ", state=" << state_; << ", state=" << state_;
...@@ -1277,9 +1286,10 @@ void AsyncSocket::handleRead() noexcept { ...@@ -1277,9 +1286,10 @@ void AsyncSocket::handleRead() noexcept {
while (readCallback_ && eventBase_ == originalEventBase) { while (readCallback_ && eventBase_ == originalEventBase) {
// Get the buffer to read into. // Get the buffer to read into.
void* buf = nullptr; void* buf = nullptr;
size_t buflen = 0; size_t buflen = 0, offset = 0;
try { try {
readCallback_->getReadBuffer(&buf, &buflen); prepareReadBuffer(&buf, &buflen);
VLOG(5) << "prepareReadBuffer() buf=" << buf << ", buflen=" << buflen;
} catch (const AsyncSocketException& ex) { } catch (const AsyncSocketException& ex) {
return failRead(__func__, ex); return failRead(__func__, ex);
} catch (const std::exception& ex) { } catch (const std::exception& ex) {
...@@ -1294,7 +1304,7 @@ void AsyncSocket::handleRead() noexcept { ...@@ -1294,7 +1304,7 @@ void AsyncSocket::handleRead() noexcept {
"non-exception type"); "non-exception type");
return failRead(__func__, ex); return failRead(__func__, ex);
} }
if (buf == nullptr || buflen == 0) { if (!isBufferMovable_ && (buf == nullptr || buflen == 0)) {
AsyncSocketException ex(AsyncSocketException::BAD_ARGS, AsyncSocketException ex(AsyncSocketException::BAD_ARGS,
"ReadCallback::getReadBuffer() returned " "ReadCallback::getReadBuffer() returned "
"empty buffer"); "empty buffer");
...@@ -1302,9 +1312,23 @@ void AsyncSocket::handleRead() noexcept { ...@@ -1302,9 +1312,23 @@ void AsyncSocket::handleRead() noexcept {
} }
// Perform the read // Perform the read
ssize_t bytesRead = performRead(buf, buflen); ssize_t bytesRead = performRead(&buf, &buflen, &offset);
VLOG(4) << "this=" << this << ", AsyncSocket::handleRead() got "
<< bytesRead << " bytes";
if (bytesRead > 0) { if (bytesRead > 0) {
readCallback_->readDataAvailable(bytesRead); if (!isBufferMovable_) {
readCallback_->readDataAvailable(bytesRead);
} else {
CHECK(kOpenSslModeMoveBufferOwnership);
VLOG(5) << "this=" << this << ", AsyncSocket::handleRead() got "
<< "buf=" << buf << ", " << bytesRead << "/" << buflen
<< ", offset=" << offset;
auto readBuf = folly::IOBuf::takeOwnership(buf, buflen);
readBuf->trimStart(offset);
readBuf->trimEnd(buflen - offset - bytesRead);
readCallback_->readBufferAvailable(std::move(readBuf));
}
// Fall through and continue around the loop if the read // Fall through and continue around the loop if the read
// completely filled the available buffer. // completely filled the available buffer.
// Note that readCallback_ may have been uninstalled or changed inside // Note that readCallback_ may have been uninstalled or changed inside
......
...@@ -636,6 +636,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper { ...@@ -636,6 +636,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper {
void ioReady(uint16_t events) noexcept; void ioReady(uint16_t events) noexcept;
virtual void checkForImmediateRead() noexcept; virtual void checkForImmediateRead() noexcept;
virtual void handleInitialReadWrite() noexcept; virtual void handleInitialReadWrite() noexcept;
virtual void prepareReadBuffer(void** buf, size_t* buflen) noexcept;
virtual void handleRead() noexcept; virtual void handleRead() noexcept;
virtual void handleWrite() noexcept; virtual void handleWrite() noexcept;
virtual void handleConnect() noexcept; virtual void handleConnect() noexcept;
...@@ -651,7 +652,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper { ...@@ -651,7 +652,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper {
* READ_ERROR on error, or READ_BLOCKING if the operation will * READ_ERROR on error, or READ_BLOCKING if the operation will
* block. * block.
*/ */
virtual ssize_t performRead(void* buf, size_t buflen); virtual ssize_t performRead(void** buf, size_t* buflen, size_t* offset);
/** /**
* Populate an iovec array from an IOBuf and attempt to write it. * Populate an iovec array from an IOBuf and attempt to write it.
...@@ -762,6 +763,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper { ...@@ -762,6 +763,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper {
ShutdownSocketSet* shutdownSocketSet_; ShutdownSocketSet* shutdownSocketSet_;
size_t appBytesReceived_; ///< Num of bytes received from socket size_t appBytesReceived_; ///< Num of bytes received from socket
size_t appBytesWritten_; ///< Num of bytes written to socket size_t appBytesWritten_; ///< Num of bytes written to socket
bool isBufferMovable_{false};
}; };
......
...@@ -23,6 +23,16 @@ ...@@ -23,6 +23,16 @@
#include <folly/io/async/EventBase.h> #include <folly/io/async/EventBase.h>
#include <folly/io/async/AsyncSocketBase.h> #include <folly/io/async/AsyncSocketBase.h>
#include <openssl/ssl.h>
constexpr bool kOpenSslModeMoveBufferOwnership =
#ifdef SSL_MODE_MOVE_BUFFER_OWNERSHIP
true
#else
false
#endif
;
namespace folly { namespace folly {
class AsyncSocketException; class AsyncSocketException;
...@@ -376,8 +386,53 @@ class AsyncTransportWrapper : virtual public AsyncTransport { ...@@ -376,8 +386,53 @@ class AsyncTransportWrapper : virtual public AsyncTransport {
* *
* @param len The number of bytes placed in the buffer. * @param len The number of bytes placed in the buffer.
*/ */
virtual void readDataAvailable(size_t len) noexcept = 0; virtual void readDataAvailable(size_t len) noexcept = 0;
/**
* When data becomes available, isBufferMovable() will be invoked to figure
* out which API will be used, readBufferAvailable() or
* readDataAvailable(). If isBufferMovable() returns true, that means
* ReadCallback supports the IOBuf ownership transfer and
* readBufferAvailable() will be used. Otherwise, not.
* By default, isBufferMovable() always return false. If
* readBufferAvailable() is implemented and to be invoked, You should
* overwrite isBufferMovable() and return true in the inherited class.
*
* This method allows the AsyncSocket/AsyncSSLSocket do buffer allocation by
* itself until data becomes available. Compared with the pre/post buffer
* allocation in getReadBuffer()/readDataAvailabe(), readBufferAvailable()
* has two advantages. First, this can avoid memcpy. E.g., in
* AsyncSSLSocket, the decrypted data was copied from the openssl internal
* buffer to the readbuf buffer. With the buffer ownership transfer, the
* internal buffer can be directly "moved" to ReadCallback. Second, the
* memory allocation can be more precise. The reason is
* AsyncSocket/AsyncSSLSocket can allocate the memory of precise size
* because they have more context about the available data than
* ReadCallback. Think about the getReadBuffer() pre-allocate 4072 bytes
* buffer, but the available data is always 16KB (max OpenSSL record size).
*/
virtual bool isBufferMovable() noexcept {
return false;
}
/**
* readBufferAvailable() will be invoked when data has been successfully
* read.
*
* Note that only either readBufferAvailable() or readDataAvailable() will
* be invoked according to the return value of isBufferMovable(). The timing
* and aftereffect of readBufferAvailable() are the same as
* readDataAvailable()
*
* @param readBuf The unique pointer of read buffer.
*/
virtual void readBufferAvailable(std::unique_ptr<IOBuf> readBuf)
noexcept {};
/** /**
* readEOF() will be invoked when the transport is closed. * readEOF() will be invoked when the transport is closed.
* *
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment