Commit f78f293a authored by Ian Roddis's avatar Ian Roddis

Implementing deque for write queue to allow for multiple pending writes and to preserve write order

parent 7bb54435
...@@ -12,6 +12,8 @@ ...@@ -12,6 +12,8 @@
#include <pistache/async.h> #include <pistache/async.h>
#include <pistache/stream.h> #include <pistache/stream.h>
#include <deque>
namespace Pistache { namespace Pistache {
namespace Tcp { namespace Tcp {
...@@ -199,7 +201,7 @@ private: ...@@ -199,7 +201,7 @@ private:
fd becomes ready again, we can write everything fd becomes ready again, we can write everything
*/ */
PollableQueue<WriteEntry> writesQueue; PollableQueue<WriteEntry> writesQueue;
std::unordered_map<Fd, WriteEntry> toWrite; std::unordered_map<Fd, std::deque<WriteEntry> > toWrite;
PollableQueue<TimerEntry> timersQueue; PollableQueue<TimerEntry> timersQueue;
std::unordered_map<Fd, TimerEntry> timers; std::unordered_map<Fd, TimerEntry> timers;
...@@ -227,11 +229,8 @@ private: ...@@ -227,11 +229,8 @@ private:
void armTimerMsImpl(TimerEntry entry); void armTimerMsImpl(TimerEntry entry);
void asyncWriteImpl(Fd fd, WriteEntry& entry, WriteStatus status = FirstTry); // This will attempt to drain the write queue for the fd
void asyncWriteImpl( void asyncWriteImpl(Fd fd);
Fd fd, int flags, const BufferHolder& buffer,
Async::Deferred<ssize_t> deferred,
WriteStatus status = FirstTry);
void handlePeerDisconnection(const std::shared_ptr<Peer>& peer); void handlePeerDisconnection(const std::shared_ptr<Peer>& peer);
void handleIncoming(const std::shared_ptr<Peer>& peer); void handleIncoming(const std::shared_ptr<Peer>& peer);
......
...@@ -99,8 +99,8 @@ Transport::onReady(const Aio::FdSet& fds) { ...@@ -99,8 +99,8 @@ Transport::onReady(const Aio::FdSet& fds) {
reactor()->modifyFd(key(), fd, NotifyOn::Read, Polling::Mode::Edge); reactor()->modifyFd(key(), fd, NotifyOn::Read, Polling::Mode::Edge);
auto& write = it->second; // Try to drain the queue
asyncWriteImpl(fd, write, Retry); asyncWriteImpl(fd);
} }
} }
} }
...@@ -164,80 +164,90 @@ Transport::handlePeerDisconnection(const std::shared_ptr<Peer>& peer) { ...@@ -164,80 +164,90 @@ Transport::handlePeerDisconnection(const std::shared_ptr<Peer>& peer) {
throw std::runtime_error("Could not find peer to erase"); throw std::runtime_error("Could not find peer to erase");
peers.erase(it); peers.erase(it);
toWrite.erase(fd);
close(fd); // Clean up buffers
} auto & wq = toWrite[fd];
while (wq.size() > 0) {
void auto & entry = wq.front();
Transport::asyncWriteImpl(Fd fd, Transport::WriteEntry& entry, WriteStatus status) { const BufferHolder & buffer = entry.buffer;
asyncWriteImpl(fd, entry.flags, entry.buffer, std::move(entry.deferred), status);
}
void
Transport::asyncWriteImpl(
Fd fd, int flags, const BufferHolder& buffer,
Async::Deferred<ssize_t> deferred, WriteStatus status)
{
auto cleanUp = [&]() {
if (buffer.isRaw()) { if (buffer.isRaw()) {
auto raw = buffer.raw(); auto raw = buffer.raw();
if (raw.isOwned) delete[] raw.data; if (raw.isOwned) delete[] raw.data;
} }
}
toWrite.erase(fd);
if (status == Retry) close(fd);
toWrite.erase(fd); }
};
size_t totalWritten = buffer.offset();
for (;;) {
ssize_t bytesWritten = 0;
auto len = buffer.size() - totalWritten;
if (buffer.isRaw()) {
auto raw = buffer.raw();
auto ptr = raw.data + totalWritten;
bytesWritten = ::send(fd, ptr, len, flags | MSG_NOSIGNAL);
} else {
auto file = buffer.fd();
off_t offset = totalWritten;
bytesWritten = ::sendfile(fd, file, &offset, len);
}
if (bytesWritten < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
// save for a future retry with the totalWritten offset.
if (status == Retry) {
toWrite.erase(fd);
}
toWrite.insert(
std::make_pair(fd,
WriteEntry(std::move(deferred), buffer.detach(totalWritten), flags)));
reactor()->modifyFd(key(), fd, NotifyOn::Read | NotifyOn::Write, Polling::Mode::Edge); void
Transport::asyncWriteImpl(Fd fd)
{
auto & wq = toWrite[fd];
while (wq.size() > 0) {
auto & entry = wq.front();
int flags = entry.flags;
const BufferHolder &buffer = entry.buffer;
Async::Deferred<ssize_t> deferred = std::move(entry.deferred);
auto cleanUp = [&]() {
if (buffer.isRaw()) {
auto raw = buffer.raw();
if (raw.isOwned) delete[] raw.data;
} }
else { wq.pop_front();
cleanUp(); if (wq.size() == 0) {
deferred.reject(Pistache::Error::system("Could not write data")); toWrite.erase(fd);
reactor()->modifyFd(key(), fd, NotifyOn::Read, Polling::Mode::Edge);
} }
break; };
}
else { bool halt = false;
totalWritten += bytesWritten; size_t totalWritten = buffer.offset();
if (totalWritten >= buffer.size()) { for (;;) {
cleanUp(); ssize_t bytesWritten = 0;
auto len = buffer.size() - totalWritten;
if (buffer.isFile()) { if (buffer.isRaw()) {
// done with the file buffer, nothing else knows whether to auto raw = buffer.raw();
// close it with the way the code is written. auto ptr = raw.data + totalWritten;
::close(buffer.fd()); bytesWritten = ::send(fd, ptr, len, flags | MSG_NOSIGNAL);
} else {
auto file = buffer.fd();
off_t offset = totalWritten;
bytesWritten = ::sendfile(fd, file, &offset, len);
}
if (bytesWritten < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
wq.pop_front();
wq.push_front(WriteEntry(std::move(deferred), buffer.detach(totalWritten), flags));
reactor()->modifyFd(key(), fd, NotifyOn::Read | NotifyOn::Write, Polling::Mode::Edge);
}
else {
cleanUp();
deferred.reject(Pistache::Error::system("Could not write data"));
halt = true;
} }
// Cast to match the type of defered template
// to avoid a BadType exception
deferred.resolve(static_cast<ssize_t>(totalWritten));
break; break;
} }
else {
totalWritten += bytesWritten;
if (totalWritten >= buffer.size()) {
cleanUp();
if (buffer.isFile()) {
// done with the file buffer, nothing else knows whether to
// close it with the way the code is written.
::close(buffer.fd());
}
// Cast to match the type of defered template
// to avoid a BadType exception
deferred.resolve(static_cast<ssize_t>(totalWritten));
break;
}
}
} }
if (halt) break;
} }
} }
...@@ -299,7 +309,9 @@ Transport::handleWriteQueue() { ...@@ -299,7 +309,9 @@ Transport::handleWriteQueue() {
if (!entry) break; if (!entry) break;
auto &write = entry->data(); auto &write = entry->data();
asyncWriteImpl(write.peerFd, write); auto fd = write.peerFd;
toWrite[fd].push_back(std::move(write));
reactor()->modifyFd(key(), fd, NotifyOn::Read | NotifyOn::Write, Polling::Mode::Edge);
} }
} }
......
...@@ -14,7 +14,7 @@ using namespace Pistache; ...@@ -14,7 +14,7 @@ using namespace Pistache;
static const size_t N_LETTERS = 26; static const size_t N_LETTERS = 26;
static const size_t LETTER_REPEATS = 100000; static const size_t LETTER_REPEATS = 100000;
static const size_t SET_REPEATS = 10; static const size_t SET_REPEATS = 10;
static const uint16_t PORT = 9080; static const uint16_t PORT = 9082;
void dumpData(const Rest::Request&req, Http::ResponseWriter response) { void dumpData(const Rest::Request&req, Http::ResponseWriter response) {
UNUSED(req); UNUSED(req);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment