Parser now supports Chunked Transfer-Encoding

include/http.h

@@ -312,6 +312,7 @@ operator<<(ResponseStream& stream, ResponseStream & (*func)(ResponseStream &)) {
 }
 
 // 6. Response
+// @Investigate public inheritence
 class Response : public Message {
 public:
     friend class Handler;
@@ -506,14 +507,41 @@ namespace Private {
     };
 
     struct BodyStep : public Step {
-        BodyStep(Message* request)
-            : Step(request)
+        BodyStep(Message* message)
+            : Step(message)
+            , chunk(message)
             , bytesRead(0)
         { }
 
         State apply(StreamCursor& cursor);
 
     private:
+        struct Chunk {
+            enum Result { Complete, Incomplete, Final };
+
+            Chunk(Message* message)
+              : message(message)
+              , bytesRead(0)
+              , size(-1)
+            { }
+
+            Result parse(StreamCursor& cursor);
+
+            void reset() {
+                bytesRead = 0;
+                size = -1;
+            }
+
+        private:
+            Message* message;
+            size_t bytesRead;
+            ssize_t size;
+        };
+
+        State parseContentLength(StreamCursor& cursor, const std::shared_ptr<Header::ContentLength>& cl);
+        State parseTransferEncoding(StreamCursor& cursor, const std::shared_ptr<Header::TransferEncoding>& te);
+
+        Chunk chunk;
         size_t bytesRead;
     };
 

include/http_header.h

@@ -218,6 +218,10 @@ class TransferEncoding : public EncodingHeader {
 public:
     NAME("Transfer-Encoding")
 
+    TransferEncoding()
+        : EncodingHeader(Encoding::Identity)
+    { }
+
     explicit TransferEncoding(Encoding encoding)
        : EncodingHeader(encoding)
     { }

include/stream.h

@@ -99,7 +99,8 @@ public:
         }
 
         memcpy(bytes + size, data, len);
-        Base::setg(bytes, bytes + size, bytes + size + len);
+        Base::setg(bytes, Base::gptr(), bytes + size + len);
+
         size += len;
         return true;
     }

main.cc

@@ -351,7 +351,11 @@ int main() {
             client
             .newRequest("/off/octal/nask")
             .header<Header::ContentType>(MIME(Text, Plain))
-            .cookie(Cookie("FOO", "bar")));
+            .cookie(Cookie("FOO", "bar")))
+    .then([](const Http::Response& response) {
+        std::cout << "code = " << response.code() << std::endl;
+        std::cout << "body = " << response.body_ << std::endl;
+    }, Async::NoExcept);
     std::this_thread::sleep_for(std::chrono::seconds(1));
 }
 #endif

src/client/client.cc

@@ -236,7 +236,6 @@ Transport::handleConnectionQueue() {
 
 void
 Transport::handleIncoming(Fd fd) {
-    std::cout << "Handling incoming on fd " << fd << std::endl;
     char buffer[Const::MaxBuffer];
     memset(buffer, 0, sizeof buffer);
 
@@ -283,7 +282,7 @@ Transport::handleResponsePacket(Fd fd, const char* buffer, size_t totalBytes) {
     auto &req = it->second;
     req.parser->feed(buffer, totalBytes);
     if (req.parser->parse() == Private::State::Done) {
-        std::cout << "DOne parsing!" << std::endl;
+        req.resolve(std::move(req.parser->response));
     }
 }
 
@@ -524,10 +523,9 @@ Client::get(const Http::Request& request, std::chrono::milliseconds timeout)
             conn->connect(addr_);
         }
 
-        conn->perform(request);
+        return conn->perform(request);
     }
 
-    return Async::Promise<Http::Response>::rejected(std::runtime_error("Unimplemented"));
 }
 
 } // namespace Http

src/common/http.cc

@@ -226,8 +226,9 @@ namespace Private {
     ResponseLineStep::apply(StreamCursor& cursor) {
         StreamCursor::Revert revert(cursor);
 
+        auto *response = static_cast<Response *>(message);
+
         if (match_raw("HTTP/1.1", sizeof("HTTP/1.1") - 1, cursor)) {
-            std::cout << "Matched http/1.1" << std::endl;
             //response->version = Version::Http11;
         }
         else if (match_raw("HTTP/1.0", sizeof("HTTP/1.0") - 1, cursor)) {
@@ -246,16 +247,16 @@ namespace Private {
         if (!match_until(' ', cursor))
             return State::Again;
 
-        auto code = codeToken.text();
-        std::cout << "Code = " << code << std::endl;
+        char *end;
+        auto code = strtol(codeToken.rawText(), &end, 10);
+        if (*end != ' ')
+            raise("Failed to parsed return code");
+        response->code_ = static_cast<Http::Code>(code);
+
         if (!cursor.advance(1)) return State::Again;
 
-        StreamCursor::Token textToken(cursor);
         while (!cursor.eol()) cursor.advance(1);
 
-        auto text = textToken.text();
-        std::cout << "Text = "<< text << std::endl;
-
         if (!cursor.advance(2)) return State::Again;
 
         revert.ignore();
@@ -319,58 +320,130 @@ namespace Private {
 
     State
     BodyStep::apply(StreamCursor& cursor) {
+        if (message->body_.empty()) {
+            /* If this is the first time we are reading the body, skip the CRLF */
+            if (!cursor.advance(2)) return State::Again;
+        }
+
         auto cl = message->headers_.tryGet<Header::ContentLength>();
-        if (!cl) return State::Done;
+        auto te = message->headers_.tryGet<Header::TransferEncoding>();
+
+        if (cl && te)
+            raise("Got mutually exclusive ContentLength and TransferEncoding header");
+
+        if (cl)
+            return parseContentLength(cursor, cl);
 
+        if (te)
+            return parseTransferEncoding(cursor, te);
+
+        return State::Done;
+
+    }
+
+    State
+    BodyStep::parseContentLength(StreamCursor& cursor, const std::shared_ptr<Header::ContentLength>& cl) {
         auto contentLength = cl->value();
-        // We already started to read some bytes but we got an incomplete payload
-        if (bytesRead > 0) {
-            // How many bytes do we still need to read ?
-            const size_t remaining = contentLength - bytesRead;
 
+        auto readBody = [&](size_t size) {
             StreamCursor::Token token(cursor);
             const size_t available = cursor.remaining();
 
-            // Could be refactored in a single function / lambda but I'm too lazy
-            // for that right now
-            if (available < remaining) {
+            // We have an incomplete body, read what we can
+            if (available < size) {
                 cursor.advance(available);
-                message->body_ += token.text();
+                message->body_.append(token.rawText(), token.size());
 
                 bytesRead += available;
 
-                return State::Again;
-            }
-            else {
-                cursor.advance(remaining);
-                message->body_ += token.text();
+                return false;
             }
 
+            cursor.advance(size);
+            message->body_.append(token.rawText(), token.size());
+            return true;
+        };
+
+        // We already started to read some bytes but we got an incomplete payload
+        if (bytesRead > 0) {
+            // How many bytes do we still need to read ?
+            const size_t remaining = contentLength - bytesRead;
+            if (!readBody(remaining)) return State::Again;
         }
         // This is the first time we are reading the payload
         else {
-            if (!cursor.advance(2)) return State::Again;
-
             message->body_.reserve(contentLength);
-
-            StreamCursor::Token token(cursor);
-            const size_t available = cursor.remaining();
-            // We have an incomplete body, read what we can
-            if (available < contentLength) {
-                cursor.advance(available);
-                message->body_ += token.text();
-                bytesRead += available;
-                return State::Again;
-            }
-
-            cursor.advance(contentLength);
-            message->body_ = token.text();
+            if (!readBody(contentLength)) return State::Again;
         }
 
         bytesRead = 0;
         return State::Done;
     }
 
+    BodyStep::Chunk::Result
+    BodyStep::Chunk::parse(StreamCursor& cursor) {
+        if (size == -1) {
+            StreamCursor::Revert revert(cursor);
+            StreamCursor::Token chunkSize(cursor);
+
+            while (!cursor.eol()) if (!cursor.advance(1)) return Incomplete;
+
+            char *end;
+            const char *raw = chunkSize.rawText();
+            auto sz = std::strtol(raw, &end, 16);
+            if (*end != '\r') throw std::runtime_error("Invalid chunk size");
+
+            // CRLF
+            if (!cursor.advance(2)) return Incomplete;
+
+            revert.ignore();
+
+            size = sz;
+        }
+
+        if (size == 0)
+            return Final;
+
+        message->body_.reserve(size);
+        StreamCursor::Token chunkData(cursor);
+        const size_t available = cursor.remaining();
+
+        if (available < size) {
+            cursor.advance(available);
+            message->body_.append(chunkData.rawText(), available);
+            return Incomplete;
+        }
+        cursor.advance(size);
+
+        if (!cursor.advance(2)) return Incomplete;
+
+        message->body_.append(chunkData.rawText(), size);
+        return Complete;
+    }
+
+    State
+    BodyStep::parseTransferEncoding(StreamCursor& cursor, const std::shared_ptr<Header::TransferEncoding>& te) {
+        auto encoding = te->encoding();
+        if (encoding == Http::Header::Encoding::Chunked) {
+            Chunk::Result result;
+            try {
+                while ((result = chunk.parse(cursor)) != Chunk::Final) {
+                    if (result == Chunk::Incomplete) return State::Again;
+
+                    chunk.reset();
+                    if (cursor.eof()) return State::Again;
+                }
+            } catch (const std::exception& e) {
+                raise(e.what());
+            }
+
+            return State::Done;
+        }
+        else {
+            raise("Unsupported Transfer-Encoding", Code::Not_Implemented);
+        }
+    }
+
     State
     ParserBase::parse() {
         State state = State::Again;

src/common/http_headers.cc

@@ -24,6 +24,7 @@ RegisterHeader(Accept);
 RegisterHeader(Allow);
 RegisterHeader(CacheControl);
 RegisterHeader(ContentEncoding);
+RegisterHeader(TransferEncoding);
 RegisterHeader(ContentLength);
 RegisterHeader(ContentType);
 RegisterHeader(Date);