Commit cbcc982b authored by octal's avatar octal

Started implementation of the http client

parent a425e933
......@@ -11,6 +11,7 @@
#include <functional>
#include <memory>
#include <atomic>
#include <vector>
#include "optional.h"
#include "typeid.h"
......
/*
Mathieu Stefani, 29 janvier 2016
The Http client
*/
#pragma once
#include "async.h"
#include "os.h"
#include "http.h"
#include "io.h"
#include <atomic>
#include <sys/types.h>
#include <sys/socket.h>
#include <deque>
namespace Net {
namespace Http {
class Connection;
class Transport : public Io::Handler {
public:
PROTOTYPE_OF(Io::Handler, Transport)
void onReady(const Io::FdSet& fds);
void registerPoller(Polling::Epoll& poller);
Async::Promise<void>
asyncConnect(Fd fd, const struct sockaddr* address, socklen_t addr_len);
template<typename Buf>
Async::Promise<ssize_t> asyncWrite(Fd fd, const Buf& buffer, int flags = 0) {
// If the I/O operation has been initiated from an other thread, we queue it and we'll process
// it in our own thread so that we make sure that every I/O operation happens in the right thread
if (std::this_thread::get_id() != io()->thread()) {
return Async::Promise<ssize_t>([=](Async::Resolver& resolve, Async::Rejection& reject) {
BufferHolder holder(buffer);
auto detached = holder.detach();
OnHoldWrite write(std::move(resolve), std::move(reject), detached, flags);
write.peerFd = fd;
auto *e = writesQueue.allocEntry(std::move(write));
writesQueue.push(e);
});
}
return Async::Promise<ssize_t>([&](Async::Resolver& resolve, Async::Rejection& reject) {
auto it = toWrite.find(fd);
if (it != std::end(toWrite)) {
reject(Net::Error("Multiple writes on the same fd"));
return;
}
std::cout << "I am in the same thread, wouhou" << std::endl;
asyncWriteImpl(fd, flags, BufferHolder(buffer), std::move(resolve), std::move(reject));
});
}
void addInFlight(Fd fd, Async::Resolver&& resolve, Async::Rejection&& reject) {
InflightRequest req(std::move(resolve), std::move(reject));
inflightRequests.insert(
std::make_pair(fd, std::move(req)));
}
private:
struct Event {
enum class Type {
Connection
};
virtual Type type() const = 0;
};
struct ConnectionEvent : public Event {
ConnectionEvent(const Connection* connection);
Type type() const { return Type::Connection; }
const Connection* connection_;
};
enum WriteStatus {
FirstTry,
Retry
};
struct BufferHolder {
enum Type { Raw, File };
explicit BufferHolder(const Buffer& buffer)
: type(Raw)
, u(buffer)
{
size_ = buffer.len;
}
explicit BufferHolder(const FileBuffer& buffer)
: type(File)
, u(buffer.fd())
{
size_ = buffer.size();
}
bool isFile() const { return type == File; }
bool isRaw() const { return type == Raw; }
size_t size() const { return size_; }
Fd fd() const {
if (!isFile())
throw std::runtime_error("Tried to retrieve fd of a non-filebuffer");
return u.fd;
}
Buffer raw() const {
if (!isRaw())
throw std::runtime_error("Tried to retrieve raw data of a non-buffer");
return u.raw;
}
BufferHolder detach(size_t offset = 0) const {
if (!isRaw())
return BufferHolder(u.fd, size_);
if (u.raw.isOwned)
return BufferHolder(u.raw);
auto detached = u.raw.detach(offset);
return BufferHolder(detached);
}
private:
BufferHolder(Fd fd, size_t size)
: u(fd)
, size_(size)
, type(File)
{ }
union U {
Buffer raw;
Fd fd;
U(Buffer buffer) : raw(buffer) { }
U(Fd fd) : fd(fd) { }
} u;
size_t size_;
Type type;
};
struct OnHoldWrite {
OnHoldWrite(Async::Resolver resolve, Async::Rejection reject,
BufferHolder buffer, int flags = 0)
: resolve(std::move(resolve))
, reject(std::move(reject))
, buffer(std::move(buffer))
, flags(flags)
, peerFd(-1)
{ }
Async::Resolver resolve;
Async::Rejection reject;
BufferHolder buffer;
int flags;
Fd peerFd;
};
struct PendingConnection {
PendingConnection(
Async::Resolver resolve, Async::Rejection reject,
Fd fd, const struct sockaddr* addr, socklen_t addr_len)
: resolve(std::move(resolve))
, reject(std::move(reject))
, fd(fd)
, addr(addr)
, addr_len(addr_len)
{ }
Async::Resolver resolve;
Async::Rejection reject;
Fd fd;
const struct sockaddr* addr;
socklen_t addr_len;
};
struct InflightRequest {
InflightRequest(
Async::Resolver resolve, Async::Rejection reject)
: resolve(std::move(resolve))
, reject(std::move(reject))
{
parser.reset(new Private::Parser<Http::Response>());
}
Async::Resolver resolve;
Async::Rejection reject;
std::shared_ptr<Private::Parser<Http::Response>> parser;
};
PollableQueue<OnHoldWrite> writesQueue;
PollableQueue<PendingConnection> connectionsQueue;
std::unordered_map<Fd, OnHoldWrite> toWrite;
std::unordered_map<Fd, PendingConnection> pendingConnections;
std::unordered_map<Fd, InflightRequest> inflightRequests;
void asyncWriteImpl(Fd fd, OnHoldWrite& entry, WriteStatus status = FirstTry);
void asyncWriteImpl(
Fd fd, int flags, const BufferHolder& buffer,
Async::Resolver resolve, Async::Rejection reject,
WriteStatus status = FirstTry);
void handleWriteQueue();
void handleConnectionQueue();
void handleIncoming(Fd fd);
void handleResponsePacket(Fd fd, const char* buffer, size_t totalBytes);
};
class ConnectionPool;
struct Connection {
friend class ConnectionPool;
enum State : uint32_t {
Idle,
Used,
Connecting,
Connected
};
Connection()
: fd(-1)
{
}
void connect(Net::Address addr);
bool isConnected() const;
bool hasTransport() const;
void associateTransport(const std::shared_ptr<Transport>& transport);
Async::Promise<Response> perform(const Http::Request& request);
Fd fd;
private:
void performImpl(
const Http::Request& request,
Async::Resolver resolve,
Async::Rejection reject);
void processRequestQueue();
struct RequestData {
RequestData(Async::Resolver resolve, Async::Rejection reject, const Http::Request& request)
: resolve(std::move(resolve))
, reject(std::move(reject))
, request(request)
{ }
Async::Resolver resolve;
Async::Rejection reject;
Http::Request request;
};
std::atomic<uint32_t> state_;
std::shared_ptr<Transport> transport_;
Queue<RequestData> requestsQueue;
};
struct ConnectionPool {
void init(size_t max = 1);
std::shared_ptr<Connection> pickConnection();
void returnConnection(const std::shared_ptr<Connection>& connection);
private:
std::vector<std::shared_ptr<Connection>> connections;
};
namespace Default {
constexpr int Threads = 1;
constexpr int MaxConnections = 8;
constexpr bool KeepAlive = true;
}
class Client {
public:
struct Options {
friend class Client;
Options()
: threads_(Default::Threads)
, maxConnections_(Default::MaxConnections)
, keepAlive_(Default::KeepAlive)
{ }
Options& threads(int val);
Options& maxConnections(int val);
Options& keepAlive(bool val);
private:
int threads_;
int maxConnections_;
bool keepAlive_;
};
Client(const std::string &base);
static Options options();
void init(const Options& options);
RequestBuilder newRequest(std::string resource);
Async::Promise<Response> get(
const Http::Request& request,
std::chrono::milliseconds timeout = std::chrono::milliseconds(0));
private:
Io::ServiceGroup io_;
std::string url_;
Net::Address addr_;
std::shared_ptr<Transport> transport_;
ConnectionPool pool;
std::atomic<uint64_t> ioIndex;
};
} // namespace Http
} // namespace Net
......@@ -26,7 +26,8 @@ namespace Net {
namespace Http {
namespace Private {
class Parser;
class ParserBase;
template<typename T> struct Parser;
class RequestLineStep;
class HeadersStep;
class BodyStep;
......@@ -72,20 +73,34 @@ namespace Uri {
Optional<std::string> get(const std::string& name) const;
bool has(const std::string& name) const;
void clear() {
params.clear();
}
private:
std::unordered_map<std::string, std::string> params;
};
} // namespace Uri
class RequestBuilder;
// 5. Request
class Request : private Message {
public:
friend class Private::RequestLineStep;
friend class Private::HeadersStep;
friend class Private::BodyStep;
friend class Private::Parser;
friend class Private::ParserBase;
friend class Private::Parser<Http::Request>;
friend class RequestBuilder;
Request(const Request& other) = default;
Request& operator=(const Request& other) = default;
Request(Request&& other) = default;
Request& operator=(Request&& other) = default;
friend class Handler;
Version version() const;
Method method() const;
......@@ -126,7 +141,33 @@ private:
std::string resource_;
Uri::Query query_;
#ifdef LIBSTDCPP_SMARTPTR_LOCK_FIXME
std::weak_ptr<Tcp::Peer> peer_;
#endif
};
class RequestBuilder {
public:
RequestBuilder& resource(std::string val);
RequestBuilder& params(const Uri::Query& query);
RequestBuilder& header(const std::shared_ptr<Header::Header>& header);
template<typename H, typename... Args>
typename
std::enable_if<
Header::IsHeader<H>::value, RequestBuilder&
>::type
header(Args&& ...args) {
return header(std::make_shared<H>(std::forward<Args>(args)...));
}
RequestBuilder& cookie(const Cookie& cookie);
RequestBuilder& body(std::string val);
const Request& request() const { return request_; }
operator Request() const { return request_; }
private:
Request request_;
};
class Handler;
......@@ -271,10 +312,12 @@ operator<<(ResponseStream& stream, ResponseStream & (*func)(ResponseStream &)) {
}
// 6. Response
class Response : private Message {
class Response : public Message {
public:
friend class Handler;
friend class Timeout;
friend class Private::ParserBase;
friend class Private::Parser<Response>;
friend Async::Promise<ssize_t> serveFile(Response&, const char *, const Mime::MediaType&);
......@@ -384,6 +427,12 @@ public:
}
private:
Response()
: Message()
, buf_(0)
, transport_(nullptr)
{ }
Response(Tcp::Transport* transport)
: Message()
, buf_(DefaultStreamSize)
......@@ -421,15 +470,15 @@ namespace Private {
enum class State { Again, Next, Done };
struct Step {
Step(Request* request)
: request(request)
Step(Message* request)
: message(request)
{ }
virtual State apply(StreamCursor& cursor) = 0;
void raise(const char* msg, Code code = Code::Bad_Request);
Request *request;
Message *message;
};
struct RequestLineStep : public Step {
......@@ -440,8 +489,16 @@ namespace Private {
State apply(StreamCursor& cursor);
};
struct ResponseLineStep : public Step {
ResponseLineStep(Response* response)
: Step(response)
{ }
State apply(StreamCursor& cursor);
};
struct HeadersStep : public Step {
HeadersStep(Request* request)
HeadersStep(Message* request)
: Step(request)
{ }
......@@ -449,7 +506,7 @@ namespace Private {
};
struct BodyStep : public Step {
BodyStep(Request* request)
BodyStep(Message* request)
: Step(request)
, bytesRead(0)
{ }
......@@ -460,22 +517,51 @@ namespace Private {
size_t bytesRead;
};
struct Parser {
struct ParserBase {
ParserBase()
: currentStep(0)
, cursor(&buffer)
{
}
Parser()
: contentLength(-1)
, currentStep(0)
ParserBase(const char* data, size_t len)
: currentStep(0)
, cursor(&buffer)
{
}
ParserBase(const ParserBase& other) = delete;
ParserBase(ParserBase&& other) = default;
bool feed(const char* data, size_t len);
virtual void reset();
State parse();
ArrayStreamBuf<Const::MaxBuffer> buffer;
StreamCursor cursor;
protected:
static constexpr size_t StepsCount = 3;
std::array<std::unique_ptr<Step>, StepsCount> allSteps;
size_t currentStep;
};
template<typename Message> struct Parser;
template<> struct Parser<Http::Request> : public ParserBase {
Parser()
: ParserBase()
{
allSteps[0].reset(new RequestLineStep(&request));
allSteps[1].reset(new HeadersStep(&request));
allSteps[2].reset(new BodyStep(&request));
}
Parser(const char* data, size_t len)
: contentLength(-1)
, currentStep(0)
, cursor(&buffer)
: ParserBase()
{
allSteps[0].reset(new RequestLineStep(&request));
allSteps[1].reset(new HeadersStep(&request));
......@@ -484,26 +570,38 @@ namespace Private {
feed(data, len);
}
Parser(const Parser& other) = delete;
Parser(Parser&& other) = default;
bool feed(const char* data, size_t len);
void reset();
State parse();
void reset() {
ParserBase::reset();
ArrayStreamBuf<Const::MaxBuffer> buffer;
StreamCursor cursor;
request.headers_.clear();
request.body_.clear();
request.resource_.clear();
request.query_.clear();
}
Request request;
};
private:
static constexpr size_t StepsCount = 3;
template<> struct Parser<Http::Response> : public ParserBase {
Parser()
: ParserBase()
{
allSteps[0].reset(new ResponseLineStep(&response));
allSteps[1].reset(new HeadersStep(&response));
allSteps[2].reset(new BodyStep(&response));
}
std::array<std::unique_ptr<Step>, StepsCount> allSteps;
size_t currentStep;
Parser(const char* data, size_t len)
: ParserBase()
{
allSteps[0].reset(new ResponseLineStep(&response));
allSteps[1].reset(new HeadersStep(&response));
allSteps[2].reset(new BodyStep(&response));
feed(data, len);
}
ssize_t contentLength;
Response response;
};
} // namespace Private
......@@ -520,7 +618,7 @@ public:
virtual void onTimeout(const Request& request, Response response);
private:
Private::Parser& getParser(const std::shared_ptr<Tcp::Peer>& peer) const;
Private::Parser<Http::Request>& getParser(const std::shared_ptr<Tcp::Peer>& peer) const;
};
} // namespace Http
......
......@@ -349,6 +349,10 @@ public:
NAME("User-Agent")
UserAgent() { }
explicit UserAgent(const char* ua)
: ua_(ua)
{ }
explicit UserAgent(const std::string& ua) :
ua_(ua)
{ }
......
......@@ -101,9 +101,12 @@ namespace Io {
void registerFd(Fd fd, Polling::NotifyOn interest, Polling::Mode mode = Polling::Mode::Level);
void registerFdOneShot(Fd fd, Polling::NotifyOn interest, Polling::Mode mode = Polling::Mode::Level);
void modifyFd(Fd fd, Polling::NotifyOn interest, Polling::Mode mode = Polling::Mode::Level);
void registerFd(Fd fd, Polling::NotifyOn interest, Polling::Tag tag, Polling::Mode mode = Polling::Mode::Level);
void registerFdOneShot(Fd fd, Polling::NotifyOn interest, Polling::Tag tag, Polling::Mode mode = Polling::Mode::Level);
void modifyFd(Fd fd, Polling::NotifyOn interest, Polling::Tag tag, Polling::Mode mode = Polling::Mode::Level);
void init(const std::shared_ptr<Handler>& handler);
void run();
void shutdown();
......@@ -168,7 +171,7 @@ namespace Io {
void shutdown();
std::shared_ptr<Service> service(Fd fd) const;
std::shared_ptr<Service> service(size_t index) const;
std::vector<Async::Promise<rusage>> load() const;
size_t size() const {
......
......@@ -4,6 +4,7 @@
#include "cookie.h"
#include "router.h"
#include "endpoint.h"
#include "client.h"
#include <iostream>
#include <cstring>
#include <algorithm>
......@@ -287,6 +288,7 @@ private:
};
#endif
/*
int main(int argc, char *argv[]) {
Net::Port port(9080);
......@@ -333,3 +335,23 @@ int main(int argc, char *argv[]) {
server->shutdown();
monitor.shutdown();
}
*/
#if 1
int main() {
Net::Http::Client client("http://www.foaas.com");
auto opts = Net::Http::Client::options()
.threads(1)
.maxConnections(1);
using namespace Net::Http;
client.init(opts);
client.get(
client
.newRequest("/off/octal/nask")
.header<Header::ContentType>(MIME(Text, Plain))
.cookie(Cookie("FOO", "bar")));
std::this_thread::sleep_for(std::chrono::seconds(1));
}
#endif
This diff is collapsed.
......@@ -129,6 +129,8 @@ namespace Private {
#undef METHOD
};
auto request = static_cast<Request *>(message);
bool found = false;
for (const auto& method: Methods) {
if (match_raw(method.str, method.len, cursor)) {
......@@ -220,6 +222,47 @@ namespace Private {
}
State
ResponseLineStep::apply(StreamCursor& cursor) {
StreamCursor::Revert revert(cursor);
if (match_raw("HTTP/1.1", sizeof("HTTP/1.1") - 1, cursor)) {
std::cout << "Matched http/1.1" << std::endl;
//response->version = Version::Http11;
}
else if (match_raw("HTTP/1.0", sizeof("HTTP/1.0") - 1, cursor)) {
}
else {
raise("Encountered invalid HTTP version");
}
int n;
// SP
if ((n = cursor.current()) != StreamCursor::Eof && n != ' ')
raise("Expected SPACE after http version");
if (!cursor.advance(1)) return State::Again;
StreamCursor::Token codeToken(cursor);
if (!match_until(' ', cursor))
return State::Again;
auto code = codeToken.text();
std::cout << "Code = " << code << std::endl;
if (!cursor.advance(1)) return State::Again;
StreamCursor::Token textToken(cursor);
while (!cursor.eol()) cursor.advance(1);
auto text = textToken.text();
std::cout << "Text = "<< text << std::endl;
if (!cursor.advance(2)) return State::Again;
revert.ignore();
return State::Next;
}
State
HeadersStep::apply(StreamCursor& cursor) {
StreamCursor::Revert revert(cursor);
......@@ -249,7 +292,7 @@ namespace Private {
}
if (name == "Cookie") {
request->cookies_.add(
message->cookies_.add(
Cookie::fromRaw(cursor.offset(start), cursor.diff(start))
);
}
......@@ -257,11 +300,11 @@ namespace Private {
else if (Header::Registry::isRegistered(name)) {
std::shared_ptr<Header::Header> header = Header::Registry::makeHeader(name);
header->parseRaw(cursor.offset(start), cursor.diff(start));
request->headers_.add(header);
message->headers_.add(header);
}
else {
std::string value(cursor.offset(start), cursor.diff(start));
request->headers_.addRaw(Header::Raw(std::move(name), std::move(value)));
message->headers_.addRaw(Header::Raw(std::move(name), std::move(value)));
}
// CRLF
......@@ -276,7 +319,7 @@ namespace Private {
State
BodyStep::apply(StreamCursor& cursor) {
auto cl = request->headers_.tryGet<Header::ContentLength>();
auto cl = message->headers_.tryGet<Header::ContentLength>();
if (!cl) return State::Done;
auto contentLength = cl->value();
......@@ -292,7 +335,7 @@ namespace Private {
// for that right now
if (available < remaining) {
cursor.advance(available);
request->body_ += token.text();
message->body_ += token.text();
bytesRead += available;
......@@ -300,7 +343,7 @@ namespace Private {
}
else {
cursor.advance(remaining);
request->body_ += token.text();
message->body_ += token.text();
}
}
......@@ -308,20 +351,20 @@ namespace Private {
else {
if (!cursor.advance(2)) return State::Again;
request->body_.reserve(contentLength);
message->body_.reserve(contentLength);
StreamCursor::Token token(cursor);
const size_t available = cursor.remaining();
// We have an incomplete body, read what we can
if (available < contentLength) {
cursor.advance(available);
request->body_ += token.text();
message->body_ += token.text();
bytesRead += available;
return State::Again;
}
cursor.advance(contentLength);
request->body_ = token.text();
message->body_ = token.text();
}
bytesRead = 0;
......@@ -329,7 +372,7 @@ namespace Private {
}
State
Parser::parse() {
ParserBase::parse() {
State state = State::Again;
do {
Step *step = allSteps[currentStep].get();
......@@ -344,20 +387,17 @@ namespace Private {
}
bool
Parser::feed(const char* data, size_t len) {
ParserBase::feed(const char* data, size_t len) {
return buffer.feed(data, len);
}
void
Parser::reset() {
ParserBase::reset() {
buffer.reset();
cursor.reset();
currentStep = 0;
request.headers_.clear();
request.body_.clear();
request.resource_.clear();
}
} // namespace Private
......@@ -446,6 +486,35 @@ Request::peer() const {
}
#endif
RequestBuilder&
RequestBuilder::resource(std::string val) {
request_.resource_ = std::move(val);
return *this;
}
RequestBuilder&
RequestBuilder::params(const Uri::Query& params) {
request_.query_ = params;
return *this;
}
RequestBuilder&
RequestBuilder::header(const std::shared_ptr<Header::Header>& header) {
request_.headers_.add(header);
return *this;
}
RequestBuilder&
RequestBuilder::cookie(const Cookie& cookie) {
request_.cookies_.add(cookie);
return *this;
}
RequestBuilder&
RequestBuilder::body(std::string val) {
request_.body_ = std::move(val);
}
ResponseStream::ResponseStream(
Message&& other,
std::weak_ptr<Tcp::Peer> peer,
......@@ -641,7 +710,7 @@ Handler::onInput(const char* buffer, size_t len, const std::shared_ptr<Tcp::Peer
void
Handler::onConnection(const std::shared_ptr<Tcp::Peer>& peer) {
peer->putData(ParserData, std::make_shared<Private::Parser>());
peer->putData(ParserData, std::make_shared<Private::Parser<Http::Request>>());
}
void
......@@ -663,9 +732,9 @@ Timeout::onTimeout(uint64_t numWakeup) {
}
Private::Parser&
Private::Parser<Http::Request>&
Handler::getParser(const std::shared_ptr<Tcp::Peer>& peer) const {
return *peer->getData<Private::Parser>(ParserData);
return *peer->getData<Private::Parser<Http::Request>>(ParserData);
}
......
......@@ -50,6 +50,21 @@ Service::modifyFd(Fd fd, Polling::NotifyOn interest, Polling::Mode mode) {
poller.rearmFd(fd, interest, Polling::Tag(fd), mode);
}
void
Service::registerFd(Fd fd, Polling::NotifyOn interest, Polling::Tag tag, Polling::Mode mode) {
poller.addFd(fd, interest, tag, mode);
}
void
Service::registerFdOneShot(Fd fd, Polling::NotifyOn interest, Polling::Tag tag, Polling::Mode mode) {
poller.addFdOneShot(fd, interest, tag, mode);
}
void
Service::modifyFd(Fd fd, Polling::NotifyOn interest, Polling::Tag tag, Polling::Mode mode) {
poller.rearmFd(fd, interest, tag, mode);
}
void
Service::init(const std::shared_ptr<Handler>& handler) {
handler_ = handler;
......@@ -247,6 +262,14 @@ ServiceGroup::service(Fd fd) const {
auto& wrk = workers_[worker];
return wrk->service();
}
std::shared_ptr<Service>
ServiceGroup::service(size_t index) const {
if (index >= workers_.size())
throw std::out_of_range("index out of range");
return workers_[index]->service();
}
ServiceGroup::Worker::Worker() {
service_.reset(new Service);
......
......@@ -178,6 +178,7 @@ Listener::bind(const Address& address) {
}
TRY(::listen(fd, backlog_));
break;
}
make_non_blocking(fd);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment