Commit a8b4b5ea authored by Tudor Bosman's avatar Tudor Bosman Committed by Jordan DeLong

Rework folly::AsyncIO interface to make it easier for other classes to use Op

Summary:
AsyncIOOp no longer requires derivation to be able to use callbacks; the
callback is passed in.  This makes composition easier (see AsyncIOQueue, added
in this diff).

Test Plan: async_io_test, test added

Reviewed By: lucian@fb.com

FB internal diff: D709648
parent cf583f13
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <cerrno> #include <cerrno>
#include <boost/intrusive/parent_from_member.hpp>
#include <glog/logging.h> #include <glog/logging.h>
#include "folly/Exception.h" #include "folly/Exception.h"
...@@ -27,6 +28,77 @@ ...@@ -27,6 +28,77 @@
namespace folly { namespace folly {
AsyncIOOp::AsyncIOOp(NotificationCallback cb)
: cb_(std::move(cb)),
state_(UNINITIALIZED),
result_(-EINVAL) {
memset(&iocb_, 0, sizeof(iocb_));
}
void AsyncIOOp::reset(NotificationCallback cb) {
CHECK_NE(state_, PENDING);
cb_ = std::move(cb);
state_ = UNINITIALIZED;
result_ = -EINVAL;
memset(&iocb_, 0, sizeof(iocb_));
}
AsyncIOOp::~AsyncIOOp() {
CHECK_NE(state_, PENDING);
}
void AsyncIOOp::start() {
DCHECK_EQ(state_, INITIALIZED);
state_ = PENDING;
}
void AsyncIOOp::complete(ssize_t result) {
DCHECK_EQ(state_, PENDING);
state_ = COMPLETED;
result_ = result;
if (cb_) {
cb_(this);
}
}
ssize_t AsyncIOOp::result() const {
CHECK_EQ(state_, COMPLETED);
return result_;
}
void AsyncIOOp::pread(int fd, void* buf, size_t size, off_t start) {
init();
io_prep_pread(&iocb_, fd, buf, size, start);
}
void AsyncIOOp::pread(int fd, Range<unsigned char*> range, off_t start) {
pread(fd, range.begin(), range.size(), start);
}
void AsyncIOOp::preadv(int fd, const iovec* iov, int iovcnt, off_t start) {
init();
io_prep_preadv(&iocb_, fd, iov, iovcnt, start);
}
void AsyncIOOp::pwrite(int fd, const void* buf, size_t size, off_t start) {
init();
io_prep_pwrite(&iocb_, fd, const_cast<void*>(buf), size, start);
}
void AsyncIOOp::pwrite(int fd, Range<const unsigned char*> range, off_t start) {
pwrite(fd, range.begin(), range.size(), start);
}
void AsyncIOOp::pwritev(int fd, const iovec* iov, int iovcnt, off_t start) {
init();
io_prep_pwritev(&iocb_, fd, iov, iovcnt, start);
}
void AsyncIOOp::init() {
CHECK_EQ(state_, UNINITIALIZED);
state_ = INITIALIZED;
}
AsyncIO::AsyncIO(size_t capacity, PollMode pollMode) AsyncIO::AsyncIO(size_t capacity, PollMode pollMode)
: ctx_(0), : ctx_(0),
pending_(0), pending_(0),
...@@ -51,43 +123,6 @@ AsyncIO::~AsyncIO() { ...@@ -51,43 +123,6 @@ AsyncIO::~AsyncIO() {
} }
} }
void AsyncIO::pread(Op* op, int fd, void* buf, size_t size, off_t start) {
iocb cb;
io_prep_pread(&cb, fd, buf, size, start);
submit(op, &cb);
}
void AsyncIO::pread(Op* op, int fd, Range<unsigned char*> range,
off_t start) {
pread(op, fd, range.begin(), range.size(), start);
}
void AsyncIO::preadv(Op* op, int fd, const iovec* iov, int iovcnt,
off_t start) {
iocb cb;
io_prep_preadv(&cb, fd, iov, iovcnt, start);
submit(op, &cb);
}
void AsyncIO::pwrite(Op* op, int fd, const void* buf, size_t size,
off_t start) {
iocb cb;
io_prep_pwrite(&cb, fd, const_cast<void*>(buf), size, start);
submit(op, &cb);
}
void AsyncIO::pwrite(Op* op, int fd, Range<const unsigned char*> range,
off_t start) {
pwrite(op, fd, range.begin(), range.size(), start);
}
void AsyncIO::pwritev(Op* op, int fd, const iovec* iov, int iovcnt,
off_t start) {
iocb cb;
io_prep_pwritev(&cb, fd, iov, iovcnt, start);
submit(op, &cb);
}
void AsyncIO::initializeContext() { void AsyncIO::initializeContext() {
if (!ctx_) { if (!ctx_) {
int rc = io_queue_init(capacity_, &ctx_); int rc = io_queue_init(capacity_, &ctx_);
...@@ -97,11 +132,12 @@ void AsyncIO::initializeContext() { ...@@ -97,11 +132,12 @@ void AsyncIO::initializeContext() {
} }
} }
void AsyncIO::submit(Op* op, iocb* cb) { void AsyncIO::submit(Op* op) {
CHECK_EQ(op->state(), Op::UNINITIALIZED); CHECK_EQ(op->state(), Op::INITIALIZED);
CHECK_LT(pending_, capacity_) << "too many pending requests"; CHECK_LT(pending_, capacity_) << "too many pending requests";
initializeContext(); // on demand initializeContext(); // on demand
cb->data = op; iocb* cb = &op->iocb_;
cb->data = nullptr; // unused
if (pollFd_ != -1) { if (pollFd_ != -1) {
io_set_eventfd(cb, pollFd_); io_set_eventfd(cb, pollFd_);
} }
...@@ -158,62 +194,53 @@ Range<AsyncIO::Op**> AsyncIO::doWait(size_t minRequests, size_t maxRequests) { ...@@ -158,62 +194,53 @@ Range<AsyncIO::Op**> AsyncIO::doWait(size_t minRequests, size_t maxRequests) {
} }
for (size_t i = 0; i < count; ++i) { for (size_t i = 0; i < count; ++i) {
Op* op = static_cast<Op*>(events[i].data); DCHECK(events[i].obj);
DCHECK(op); Op* op = boost::intrusive::get_parent_from_member(
events[i].obj, &AsyncIOOp::iocb_);
--pending_;
op->complete(events[i].res); op->complete(events[i].res);
completed_.push_back(op); completed_.push_back(op);
} }
pending_ -= count;
return folly::Range<Op**>(&completed_.front(), count); return folly::Range<Op**>(&completed_.front(), count);
} }
AsyncIO::Op::Op() AsyncIOQueue::AsyncIOQueue(AsyncIO* asyncIO)
: state_(UNINITIALIZED), : asyncIO_(asyncIO) {
result_(-EINVAL) {
} }
void AsyncIO::Op::reset() { AsyncIOQueue::~AsyncIOQueue() {
CHECK_NE(state_, PENDING); CHECK_EQ(asyncIO_->pending(), 0);
state_ = UNINITIALIZED;
result_ = -EINVAL;
}
AsyncIO::Op::~Op() {
CHECK_NE(state_, PENDING);
} }
void AsyncIO::Op::start() { void AsyncIOQueue::submit(AsyncIOOp* op) {
DCHECK_EQ(state_, UNINITIALIZED); submit([op]() { return op; });
state_ = PENDING;
} }
void AsyncIO::Op::complete(ssize_t result) { void AsyncIOQueue::submit(OpFactory op) {
DCHECK_EQ(state_, PENDING); queue_.push_back(op);
state_ = COMPLETED; maybeDequeue();
result_ = result;
onCompleted();
} }
void AsyncIO::Op::onCompleted() { } // default: do nothing void AsyncIOQueue::onCompleted(AsyncIOOp* op) {
maybeDequeue();
ssize_t AsyncIO::Op::result() const {
CHECK_EQ(state_, COMPLETED);
return result_;
} }
CallbackOp::CallbackOp(Callback&& callback) : callback_(std::move(callback)) { } void AsyncIOQueue::maybeDequeue() {
while (!queue_.empty() && asyncIO_->pending() < asyncIO_->capacity()) {
CallbackOp::~CallbackOp() { } auto& opFactory = queue_.front();
auto op = opFactory();
queue_.pop_front();
CallbackOp* CallbackOp::make(Callback&& callback) { // Interpose our completion callback
// Ensure created on the heap auto& nextCb = op->notificationCallback();
return new CallbackOp(std::move(callback)); op->setNotificationCallback([this, nextCb](AsyncIOOp* op) {
} this->onCompleted(op);
if (nextCb) nextCb(op);
});
void CallbackOp::onCompleted() { asyncIO_->submit(op);
callback_(result()); }
delete this;
} }
} // namespace folly } // namespace folly
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <libaio.h> #include <libaio.h>
#include <cstdint> #include <cstdint>
#include <deque>
#include <functional> #include <functional>
#include <utility> #include <utility>
#include <vector> #include <vector>
...@@ -34,54 +35,43 @@ ...@@ -34,54 +35,43 @@
namespace folly { namespace folly {
/** /**
* C++ interface around Linux Async IO. * An AsyncIOOp represents a pending operation. You may set a notification
*/ * callback or you may use this class's methods directly.
class AsyncIO : private boost::noncopyable {
public:
enum PollMode {
NOT_POLLABLE,
POLLABLE
};
/**
* Create an AsyncIO context capable of holding at most 'capacity' pending
* requests at the same time. As requests complete, others can be scheduled,
* as long as this limit is not exceeded.
*
* Note: the maximum number of allowed concurrent requests is controlled
* by the fs.aio-max-nr sysctl, the default value is usually 64K.
* *
* If pollMode is POLLABLE, pollFd() will return a file descriptor that * The op must remain allocated until completion.
* can be passed to poll / epoll / select and will become readable when
* any IOs on this AsyncIO have completed. If you do this, you must use
* pollCompleted() instead of wait() -- do not read from the pollFd()
* file descriptor directly.
*/
explicit AsyncIO(size_t capacity, PollMode pollMode=NOT_POLLABLE);
~AsyncIO();
/**
* An Op represents a pending operation. You may inherit from Op (and
* override onCompleted) in order to be notified of completion (see
* CallbackOp below for an example), or you may use Op's methods directly.
*
* The Op must remain allocated until completion.
*/ */
class Op : private boost::noncopyable { class AsyncIOOp : private boost::noncopyable {
friend class AsyncIO; friend class AsyncIO;
public: public:
Op(); typedef std::function<void(AsyncIOOp*)> NotificationCallback;
virtual ~Op();
explicit AsyncIOOp(NotificationCallback cb = NotificationCallback());
~AsyncIOOp();
// There would be a cancel() method here if Linux AIO actually implemented // There would be a cancel() method here if Linux AIO actually implemented
// it. But let's not get your hopes up. // it. But let's not get your hopes up.
enum State { enum State {
UNINITIALIZED, UNINITIALIZED,
INITIALIZED,
PENDING, PENDING,
COMPLETED COMPLETED
}; };
/**
* Initiate a read request.
*/
void pread(int fd, void* buf, size_t size, off_t start);
void pread(int fd, Range<unsigned char*> range, off_t start);
void preadv(int fd, const iovec* iov, int iovcnt, off_t start);
/**
* Initiate a write request.
*/
void pwrite(int fd, const void* buf, size_t size, off_t start);
void pwrite(int fd, Range<const unsigned char*> range, off_t start);
void pwritev(int fd, const iovec* iov, int iovcnt, off_t start);
/** /**
* Return the current operation state. * Return the current operation state.
*/ */
...@@ -91,7 +81,10 @@ class AsyncIO : private boost::noncopyable { ...@@ -91,7 +81,10 @@ class AsyncIO : private boost::noncopyable {
* Reset the operation for reuse. It is an error to call reset() on * Reset the operation for reuse. It is an error to call reset() on
* an Op that is still pending. * an Op that is still pending.
*/ */
void reset(); void reset(NotificationCallback cb = NotificationCallback());
void setNotificationCallback(NotificationCallback cb) { cb_ = std::move(cb); }
const NotificationCallback& notificationCallback() const { return cb_; }
/** /**
* Retrieve the result of this operation. Returns >=0 on success, * Retrieve the result of this operation. Returns >=0 on success,
...@@ -105,28 +98,44 @@ class AsyncIO : private boost::noncopyable { ...@@ -105,28 +98,44 @@ class AsyncIO : private boost::noncopyable {
ssize_t result() const; ssize_t result() const;
private: private:
void init();
void start(); void start();
void complete(ssize_t result); void complete(ssize_t result);
virtual void onCompleted(); NotificationCallback cb_;
iocb iocb_;
State state_; State state_;
ssize_t result_; ssize_t result_;
}; };
/** /**
* Initiate a read request. * C++ interface around Linux Async IO.
*/ */
void pread(Op* op, int fd, void* buf, size_t size, off_t start); class AsyncIO : private boost::noncopyable {
void pread(Op* op, int fd, Range<unsigned char*> range, off_t start); public:
void preadv(Op* op, int fd, const iovec* iov, int iovcnt, off_t start); typedef AsyncIOOp Op;
enum PollMode {
NOT_POLLABLE,
POLLABLE
};
/** /**
* Initiate a write request. * Create an AsyncIO context capable of holding at most 'capacity' pending
* requests at the same time. As requests complete, others can be scheduled,
* as long as this limit is not exceeded.
*
* Note: the maximum number of allowed concurrent requests is controlled
* by the fs.aio-max-nr sysctl, the default value is usually 64K.
*
* If pollMode is POLLABLE, pollFd() will return a file descriptor that
* can be passed to poll / epoll / select and will become readable when
* any IOs on this AsyncIO have completed. If you do this, you must use
* pollCompleted() instead of wait() -- do not read from the pollFd()
* file descriptor directly.
*/ */
void pwrite(Op* op, int fd, const void* buf, size_t size, off_t start); explicit AsyncIO(size_t capacity, PollMode pollMode=NOT_POLLABLE);
void pwrite(Op* op, int fd, Range<const unsigned char*> range, off_t start); ~AsyncIO();
void pwritev(Op* op, int fd, const iovec* iov, int iovcnt, off_t start);
/** /**
* Wait for at least minRequests to complete. Returns the requests that * Wait for at least minRequests to complete. Returns the requests that
...@@ -160,9 +169,13 @@ class AsyncIO : private boost::noncopyable { ...@@ -160,9 +169,13 @@ class AsyncIO : private boost::noncopyable {
*/ */
Range<Op**> pollCompleted(); Range<Op**> pollCompleted();
/**
* Submit an op for execution.
*/
void submit(Op* op);
private: private:
void initializeContext(); void initializeContext();
void submit(Op* op, iocb* cb);
Range<Op**> doWait(size_t minRequests, size_t maxRequests); Range<Op**> doWait(size_t minRequests, size_t maxRequests);
io_context_t ctx_; io_context_t ctx_;
...@@ -173,20 +186,42 @@ class AsyncIO : private boost::noncopyable { ...@@ -173,20 +186,42 @@ class AsyncIO : private boost::noncopyable {
}; };
/** /**
* Implementation of AsyncIO::Op that calls a callback and then deletes * Wrapper around AsyncIO that allows you to schedule more requests than
* itself. * the AsyncIO's object capacity. Other requests are queued and processed
* in a FIFO order.
*/ */
class CallbackOp : public AsyncIO::Op { class AsyncIOQueue {
public: public:
typedef std::function<void(ssize_t)> Callback; /**
static CallbackOp* make(Callback&& callback); * Create a queue, using the given AsyncIO object.
* The AsyncIO object may not be used by anything else until the
* queue is destroyed.
*/
explicit AsyncIOQueue(AsyncIO* asyncIO);
~AsyncIOQueue();
size_t queued() const { return queue_.size(); }
/**
* Submit an op to the AsyncIO queue. The op will be queued until
* the AsyncIO object has room.
*/
void submit(AsyncIOOp* op);
/**
* Submit a delayed op to the AsyncIO queue; this allows you to postpone
* creation of the Op (which may require allocating memory, etc) until
* the AsyncIO object has room.
*/
typedef std::function<AsyncIOOp*()> OpFactory;
void submit(OpFactory op);
private: private:
explicit CallbackOp(Callback&& callback); void onCompleted(AsyncIOOp* op);
~CallbackOp(); void maybeDequeue();
void onCompleted() FOLLY_OVERRIDE;
AsyncIO* asyncIO_;
Callback callback_; std::deque<OpFactory> queue_;
}; };
} // namespace folly } // namespace folly
......
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
namespace fs = folly::fs; namespace fs = folly::fs;
using folly::AsyncIO; using folly::AsyncIO;
using folly::AsyncIOQueue;
namespace { namespace {
...@@ -116,6 +117,14 @@ TemporaryFile::~TemporaryFile() { ...@@ -116,6 +117,14 @@ TemporaryFile::~TemporaryFile() {
TemporaryFile tempFile(6 << 20); // 6MiB TemporaryFile tempFile(6 << 20); // 6MiB
typedef std::unique_ptr<char, void(*)(void*)> ManagedBuffer;
ManagedBuffer allocateAligned(size_t size) {
void* buf;
int rc = posix_memalign(&buf, 512, size);
CHECK_EQ(rc, 0) << strerror(rc);
return ManagedBuffer(reinterpret_cast<char*>(buf), free);
}
void testReadsSerially(const std::vector<TestSpec>& specs, void testReadsSerially(const std::vector<TestSpec>& specs,
AsyncIO::PollMode pollMode) { AsyncIO::PollMode pollMode) {
AsyncIO aioReader(1, pollMode); AsyncIO aioReader(1, pollMode);
...@@ -127,8 +136,9 @@ void testReadsSerially(const std::vector<TestSpec>& specs, ...@@ -127,8 +136,9 @@ void testReadsSerially(const std::vector<TestSpec>& specs,
}; };
for (int i = 0; i < specs.size(); i++) { for (int i = 0; i < specs.size(); i++) {
std::unique_ptr<char[]> buf(new char[specs[i].size]); auto buf = allocateAligned(specs[i].size);
aioReader.pread(&op, fd, buf.get(), specs[i].size, specs[i].start); op.pread(fd, buf.get(), specs[i].size, specs[i].start);
aioReader.submit(&op);
EXPECT_EQ(aioReader.pending(), 1); EXPECT_EQ(aioReader.pending(), 1);
auto ops = readerWait(&aioReader); auto ops = readerWait(&aioReader);
EXPECT_EQ(1, ops.size()); EXPECT_EQ(1, ops.size());
...@@ -145,7 +155,52 @@ void testReadsParallel(const std::vector<TestSpec>& specs, ...@@ -145,7 +155,52 @@ void testReadsParallel(const std::vector<TestSpec>& specs,
AsyncIO::PollMode pollMode) { AsyncIO::PollMode pollMode) {
AsyncIO aioReader(specs.size(), pollMode); AsyncIO aioReader(specs.size(), pollMode);
std::unique_ptr<AsyncIO::Op[]> ops(new AsyncIO::Op[specs.size()]); std::unique_ptr<AsyncIO::Op[]> ops(new AsyncIO::Op[specs.size()]);
std::vector<std::unique_ptr<char[]>> bufs(specs.size()); std::vector<ManagedBuffer> bufs;
int fd = ::open(tempFile.path().c_str(), O_DIRECT | O_RDONLY);
PCHECK(fd != -1);
SCOPE_EXIT {
::close(fd);
};
for (int i = 0; i < specs.size(); i++) {
bufs.push_back(allocateAligned(specs[i].size));
ops[i].pread(fd, bufs[i].get(), specs[i].size, specs[i].start);
aioReader.submit(&ops[i]);
}
std::vector<bool> pending(specs.size(), true);
size_t remaining = specs.size();
while (remaining != 0) {
EXPECT_EQ(remaining, aioReader.pending());
auto completed = readerWait(&aioReader);
size_t nrRead = completed.size();
EXPECT_NE(nrRead, 0);
remaining -= nrRead;
for (int i = 0; i < nrRead; i++) {
int id = completed[i] - ops.get();
EXPECT_GE(id, 0);
EXPECT_LT(id, specs.size());
EXPECT_TRUE(pending[id]);
pending[id] = false;
ssize_t res = ops[id].result();
EXPECT_LE(0, res) << folly::errnoStr(-res);
EXPECT_EQ(specs[id].size, res);
}
}
EXPECT_EQ(aioReader.pending(), 0);
for (int i = 0; i < pending.size(); i++) {
EXPECT_FALSE(pending[i]);
}
}
void testReadsQueued(const std::vector<TestSpec>& specs,
AsyncIO::PollMode pollMode) {
size_t readerCapacity = std::max(specs.size() / 2, size_t(1));
AsyncIO aioReader(readerCapacity, pollMode);
AsyncIOQueue aioQueue(&aioReader);
std::unique_ptr<AsyncIO::Op[]> ops(new AsyncIO::Op[specs.size()]);
std::vector<ManagedBuffer> bufs;
int fd = ::open(tempFile.path().c_str(), O_DIRECT | O_RDONLY); int fd = ::open(tempFile.path().c_str(), O_DIRECT | O_RDONLY);
PCHECK(fd != -1); PCHECK(fd != -1);
...@@ -153,15 +208,21 @@ void testReadsParallel(const std::vector<TestSpec>& specs, ...@@ -153,15 +208,21 @@ void testReadsParallel(const std::vector<TestSpec>& specs,
::close(fd); ::close(fd);
}; };
for (int i = 0; i < specs.size(); i++) { for (int i = 0; i < specs.size(); i++) {
bufs[i].reset(new char[specs[i].size]); bufs.push_back(allocateAligned(specs[i].size));
aioReader.pread(&ops[i], fd, bufs[i].get(), specs[i].size, ops[i].pread(fd, bufs[i].get(), specs[i].size, specs[i].start);
specs[i].start); aioQueue.submit(&ops[i]);
} }
std::vector<bool> pending(specs.size(), true); std::vector<bool> pending(specs.size(), true);
size_t remaining = specs.size(); size_t remaining = specs.size();
while (remaining != 0) { while (remaining != 0) {
if (remaining >= readerCapacity) {
EXPECT_EQ(readerCapacity, aioReader.pending());
EXPECT_EQ(remaining - readerCapacity, aioQueue.queued());
} else {
EXPECT_EQ(remaining, aioReader.pending()); EXPECT_EQ(remaining, aioReader.pending());
EXPECT_EQ(0, aioQueue.queued());
}
auto completed = readerWait(&aioReader); auto completed = readerWait(&aioReader);
size_t nrRead = completed.size(); size_t nrRead = completed.size();
EXPECT_NE(nrRead, 0); EXPECT_NE(nrRead, 0);
...@@ -179,6 +240,7 @@ void testReadsParallel(const std::vector<TestSpec>& specs, ...@@ -179,6 +240,7 @@ void testReadsParallel(const std::vector<TestSpec>& specs,
} }
} }
EXPECT_EQ(aioReader.pending(), 0); EXPECT_EQ(aioReader.pending(), 0);
EXPECT_EQ(aioQueue.queued(), 0);
for (int i = 0; i < pending.size(); i++) { for (int i = 0; i < pending.size(); i++) {
EXPECT_FALSE(pending[i]); EXPECT_FALSE(pending[i]);
} }
...@@ -188,6 +250,7 @@ void testReads(const std::vector<TestSpec>& specs, ...@@ -188,6 +250,7 @@ void testReads(const std::vector<TestSpec>& specs,
AsyncIO::PollMode pollMode) { AsyncIO::PollMode pollMode) {
testReadsSerially(specs, pollMode); testReadsSerially(specs, pollMode);
testReadsParallel(specs, pollMode); testReadsParallel(specs, pollMode);
testReadsQueued(specs, pollMode);
} }
} // anonymous namespace } // anonymous namespace
...@@ -275,8 +338,9 @@ TEST(AsyncIO, NonBlockingWait) { ...@@ -275,8 +338,9 @@ TEST(AsyncIO, NonBlockingWait) {
::close(fd); ::close(fd);
}; };
size_t size = 1024; size_t size = 1024;
std::unique_ptr<char[]> buf(new char[size]); auto buf = allocateAligned(size);
aioReader.pread(&op, fd, buf.get(), size, 0); op.pread(fd, buf.get(), size, 0);
aioReader.submit(&op);
EXPECT_EQ(aioReader.pending(), 1); EXPECT_EQ(aioReader.pending(), 1);
folly::Range<AsyncIO::Op**> completed; folly::Range<AsyncIO::Op**> completed;
...@@ -293,3 +357,4 @@ TEST(AsyncIO, NonBlockingWait) { ...@@ -293,3 +357,4 @@ TEST(AsyncIO, NonBlockingWait) {
EXPECT_EQ(aioReader.pending(), 0); EXPECT_EQ(aioReader.pending(), 0);
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment