Commit f78f293a authored by Ian Roddis's avatar Ian Roddis

Implementing deque for write queue to allow for multiple pending writes and to preserve write order

parent 7bb54435
......@@ -12,6 +12,8 @@
#include <pistache/async.h>
#include <pistache/stream.h>
#include <deque>
namespace Pistache {
namespace Tcp {
......@@ -199,7 +201,7 @@ private:
fd becomes ready again, we can write everything
*/
PollableQueue<WriteEntry> writesQueue;
std::unordered_map<Fd, WriteEntry> toWrite;
std::unordered_map<Fd, std::deque<WriteEntry> > toWrite;
PollableQueue<TimerEntry> timersQueue;
std::unordered_map<Fd, TimerEntry> timers;
......@@ -227,11 +229,8 @@ private:
void armTimerMsImpl(TimerEntry entry);
void asyncWriteImpl(Fd fd, WriteEntry& entry, WriteStatus status = FirstTry);
void asyncWriteImpl(
Fd fd, int flags, const BufferHolder& buffer,
Async::Deferred<ssize_t> deferred,
WriteStatus status = FirstTry);
// This will attempt to drain the write queue for the fd
void asyncWriteImpl(Fd fd);
void handlePeerDisconnection(const std::shared_ptr<Peer>& peer);
void handleIncoming(const std::shared_ptr<Peer>& peer);
......
......@@ -99,8 +99,8 @@ Transport::onReady(const Aio::FdSet& fds) {
reactor()->modifyFd(key(), fd, NotifyOn::Read, Polling::Mode::Edge);
auto& write = it->second;
asyncWriteImpl(fd, write, Retry);
// Try to drain the queue
asyncWriteImpl(fd);
}
}
}
......@@ -164,80 +164,90 @@ Transport::handlePeerDisconnection(const std::shared_ptr<Peer>& peer) {
throw std::runtime_error("Could not find peer to erase");
peers.erase(it);
toWrite.erase(fd);
close(fd);
}
void
Transport::asyncWriteImpl(Fd fd, Transport::WriteEntry& entry, WriteStatus status) {
asyncWriteImpl(fd, entry.flags, entry.buffer, std::move(entry.deferred), status);
}
void
Transport::asyncWriteImpl(
Fd fd, int flags, const BufferHolder& buffer,
Async::Deferred<ssize_t> deferred, WriteStatus status)
{
auto cleanUp = [&]() {
// Clean up buffers
auto & wq = toWrite[fd];
while (wq.size() > 0) {
auto & entry = wq.front();
const BufferHolder & buffer = entry.buffer;
if (buffer.isRaw()) {
auto raw = buffer.raw();
if (raw.isOwned) delete[] raw.data;
}
}
toWrite.erase(fd);
if (status == Retry)
toWrite.erase(fd);
};
size_t totalWritten = buffer.offset();
for (;;) {
ssize_t bytesWritten = 0;
auto len = buffer.size() - totalWritten;
if (buffer.isRaw()) {
auto raw = buffer.raw();
auto ptr = raw.data + totalWritten;
bytesWritten = ::send(fd, ptr, len, flags | MSG_NOSIGNAL);
} else {
auto file = buffer.fd();
off_t offset = totalWritten;
bytesWritten = ::sendfile(fd, file, &offset, len);
}
if (bytesWritten < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
// save for a future retry with the totalWritten offset.
if (status == Retry) {
toWrite.erase(fd);
}
toWrite.insert(
std::make_pair(fd,
WriteEntry(std::move(deferred), buffer.detach(totalWritten), flags)));
close(fd);
}
reactor()->modifyFd(key(), fd, NotifyOn::Read | NotifyOn::Write, Polling::Mode::Edge);
void
Transport::asyncWriteImpl(Fd fd)
{
auto & wq = toWrite[fd];
while (wq.size() > 0) {
auto & entry = wq.front();
int flags = entry.flags;
const BufferHolder &buffer = entry.buffer;
Async::Deferred<ssize_t> deferred = std::move(entry.deferred);
auto cleanUp = [&]() {
if (buffer.isRaw()) {
auto raw = buffer.raw();
if (raw.isOwned) delete[] raw.data;
}
else {
cleanUp();
deferred.reject(Pistache::Error::system("Could not write data"));
wq.pop_front();
if (wq.size() == 0) {
toWrite.erase(fd);
reactor()->modifyFd(key(), fd, NotifyOn::Read, Polling::Mode::Edge);
}
break;
}
else {
totalWritten += bytesWritten;
if (totalWritten >= buffer.size()) {
cleanUp();
if (buffer.isFile()) {
// done with the file buffer, nothing else knows whether to
// close it with the way the code is written.
::close(buffer.fd());
};
bool halt = false;
size_t totalWritten = buffer.offset();
for (;;) {
ssize_t bytesWritten = 0;
auto len = buffer.size() - totalWritten;
if (buffer.isRaw()) {
auto raw = buffer.raw();
auto ptr = raw.data + totalWritten;
bytesWritten = ::send(fd, ptr, len, flags | MSG_NOSIGNAL);
} else {
auto file = buffer.fd();
off_t offset = totalWritten;
bytesWritten = ::sendfile(fd, file, &offset, len);
}
if (bytesWritten < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
wq.pop_front();
wq.push_front(WriteEntry(std::move(deferred), buffer.detach(totalWritten), flags));
reactor()->modifyFd(key(), fd, NotifyOn::Read | NotifyOn::Write, Polling::Mode::Edge);
}
else {
cleanUp();
deferred.reject(Pistache::Error::system("Could not write data"));
halt = true;
}
// Cast to match the type of defered template
// to avoid a BadType exception
deferred.resolve(static_cast<ssize_t>(totalWritten));
break;
}
else {
totalWritten += bytesWritten;
if (totalWritten >= buffer.size()) {
cleanUp();
if (buffer.isFile()) {
// done with the file buffer, nothing else knows whether to
// close it with the way the code is written.
::close(buffer.fd());
}
// Cast to match the type of defered template
// to avoid a BadType exception
deferred.resolve(static_cast<ssize_t>(totalWritten));
break;
}
}
}
if (halt) break;
}
}
......@@ -299,7 +309,9 @@ Transport::handleWriteQueue() {
if (!entry) break;
auto &write = entry->data();
asyncWriteImpl(write.peerFd, write);
auto fd = write.peerFd;
toWrite[fd].push_back(std::move(write));
reactor()->modifyFd(key(), fd, NotifyOn::Read | NotifyOn::Write, Polling::Mode::Edge);
}
}
......
......@@ -14,7 +14,7 @@ using namespace Pistache;
static const size_t N_LETTERS = 26;
static const size_t LETTER_REPEATS = 100000;
static const size_t SET_REPEATS = 10;
static const uint16_t PORT = 9080;
static const uint16_t PORT = 9082;
void dumpData(const Rest::Request&req, Http::ResponseWriter response) {
UNUSED(req);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment