HTTP/2 Client

.github/workflows/codespell.yml

@@ -12,4 +12,4 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: pip install --user codespell[toml]
-    - run: codespell --ignore-words-list="coo,folx,ot,statics,xwindows" --skip="*.csp"
+    - run: codespell --ignore-words-list="coo,folx,ot,statics,xwindows,indexs,NotIn" --skip="*.csp"

.gitmodules

@@ -2,3 +2,6 @@
 	path = trantor
 	url = https://github.com/an-tao/trantor.git
 	branch = master
+[submodule "third_party/eric-hpack-core"]
+	path = third_party/eric-hpack-core
+	url = https://gitlab.com/joe1231231218/eric-hpack-core.git

CMakeLists.txt

@@ -244,6 +244,10 @@ if (BUILD_BROTLI)
     endif (Brotli_FOUND)
 endif (BUILD_BROTLI)
 
+
+  target_include_directories(
+      ${PROJECT_NAME}
+      PRIVATE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third_party/eric-hpack-core>)
 set(DROGON_SOURCES
     lib/src/AOPAdvice.cc
     lib/src/AccessLogger.cc
@@ -260,6 +264,8 @@ set(DROGON_SOURCES
     lib/src/Hodor.cc
     lib/src/HttpAppFrameworkImpl.cc
     lib/src/HttpBinder.cc
+    lib/src/Http2Transport.cc
+    lib/src/Http1xTransport.cc
     lib/src/HttpClientImpl.cc
     lib/src/HttpConnectionLimit.cc
     lib/src/HttpControllerBinder.cc
@@ -296,7 +302,8 @@ set(DROGON_SOURCES
     lib/src/WebSocketClientImpl.cc
     lib/src/WebSocketConnectionImpl.cc
     lib/src/YamlConfigAdapter.cc
-    lib/src/drogon_test.cc)
+    lib/src/drogon_test.cc
+    third_party/eric-hpack-core/hpack.cpp)
 set(private_headers
     lib/src/AOPAdvice.h
     lib/src/CacheFile.h
@@ -305,6 +312,8 @@ set(private_headers
     lib/src/MiddlewaresFunction.h
     lib/src/HttpAppFrameworkImpl.h
     lib/src/HttpClientImpl.h
+    lib/src/Http2Transport.h
+    lib/src/Http1xTransport.h
     lib/src/HttpConnectionLimit.h
     lib/src/HttpControllerBinder.h
     lib/src/HttpControllersRouter.h
@@ -332,7 +341,8 @@ set(private_headers
     lib/src/ConfigAdapterManager.h
     lib/src/JsonConfigAdapter.h
     lib/src/YamlConfigAdapter.h
-    lib/src/ConfigAdapter.h)
+    lib/src/ConfigAdapter.h
+    third_party/eric-hpack-core/hpack.h)
 
 if (NOT WIN32)
     set(DROGON_SOURCES

README.md

@@ -14,7 +14,7 @@ Drogon is a cross-platform framework, It supports Linux, macOS, FreeBSD, OpenBSD
 
 * Use a non-blocking I/O network lib based on epoll (kqueue under macOS/FreeBSD) to provide high-concurrency, high-performance network IO, please visit the [TFB Tests Results](https://www.techempower.com/benchmarks/#section=data-r19&hw=ph&test=composite) for more details;
 * Provide a completely asynchronous programming mode;
-* Support Http1.0/1.1 (server side and client side);
+* Support HTTP/2 (and 1.0/1.1) client and HTTP 1.1/1.0 server
 * Based on template, a simple reflection mechanism is implemented to completely decouple the main program framework, controllers and views.
 * Support cookies and built-in sessions;
 * Support back-end rendering, the controller generates the data to the view to generate the Html page. Views are described by CSP template files, C++ codes are embedded into Html pages through CSP tags. And the drogon command-line tool automatically generates the C++ code files for compilation;

lib/inc/drogon/HttpClient.h

@@ -184,6 +184,11 @@ class DROGON_EXPORT HttpClient : public trantor::NonCopyable
      * If this method is not called, the default depth value is 0 which means
      * the pipelining is disabled. For details about pipelining, see
      * rfc2616-8.1.2.2
+     *
+     * @param depth The depth value.
+     * @note This option is only valid for HTTP/1.x. If the client running in
+     * HTTP/2 mode, this settings have no effect. The maximum concurrent
+     * requests for HTTP/2 is 100 (unless the server has a different setting).
      */
     virtual void setPipeliningDepth(size_t depth) = 0;
 
@@ -238,15 +243,20 @@ class DROGON_EXPORT HttpClient : public trantor::NonCopyable
      * enabled for HTTPS.
      * @param validateCert If the parameter is set to true, the client validates
      * the server certificate when SSL handshaking.
+     * @param targetVersion The target TLS version to use for HTTPS connections.
+     * This is a mere hint, and the actual version used will depend on the
+     * server and the client's capabilities.
      * @return HttpClientPtr The smart pointer to the new client object.
      * @note: The ip parameter support for both ipv4 and ipv6 address
      */
-    static HttpClientPtr newHttpClient(const std::string &ip,
-                                       uint16_t port,
-                                       bool useSSL = false,
-                                       trantor::EventLoop *loop = nullptr,
-                                       bool useOldTLS = false,
-                                       bool validateCert = true);
+    static HttpClientPtr newHttpClient(
+        const std::string &ip,
+        uint16_t port,
+        bool useSSL = false,
+        trantor::EventLoop *loop = nullptr,
+        bool useOldTLS = false,
+        bool validateCert = true,
+        std::optional<Version> targetVersion = std::nullopt);
 
     /// Get the event loop of the client;
     virtual trantor::EventLoop *getLoop() = 0;
@@ -303,6 +313,16 @@ class DROGON_EXPORT HttpClient : public trantor::NonCopyable
         const std::vector<std::pair<std::string, std::string>>
             &sslConfCmds) = 0;
 
+    /**
+     * @brief get the protocol version used by the HTTP connection
+     * @return std::optional<Version> the protocol version used by the HTTP
+     *
+     * NOTE: It could return std::nullopt if the connection is not established
+     * or is still negotiating the protocol. This is IMPORTANT as the client
+     * COULD be lazy and not connecting until the first request arrives.
+     */
+    virtual std::optional<Version> protocolVersion() const = 0;
+
     /// Create a Http client using the hostString to connect to server
     /**
      *
@@ -329,15 +349,25 @@ class DROGON_EXPORT HttpClient : public trantor::NonCopyable
      * @param validateCert If the parameter is set to true, the client validates
      * the server certificate when SSL handshaking.
      *
+     * @param targetVersion The target HTTP version that the client will attempt
+     * to use. **THIS IS ONLY A HINT**. The server may choose to ignore this as
+     * the server may not support the requested version. The preference is as
+     * follows:
+     *   HTTP/2 > HTTP/1.1 > HTTP/1.0
+     * Note that there is no way to auto-detect HTTP/1.0 servers, so it is only
+     * used if explicitly requested.
+     *
      * @note Don't add path and parameters in hostString, the request path and
      * parameters should be set in HttpRequestPtr when calling the sendRequest()
      * method.
      *
      */
-    static HttpClientPtr newHttpClient(const std::string &hostString,
-                                       trantor::EventLoop *loop = nullptr,
-                                       bool useOldTLS = false,
-                                       bool validateCert = true);
+    static HttpClientPtr newHttpClient(
+        const std::string &hostString,
+        trantor::EventLoop *loop = nullptr,
+        bool useOldTLS = false,
+        bool validateCert = true,
+        std::optional<Version> targetVersion = std::nullopt);
 
     virtual ~HttpClient()
     {

lib/inc/drogon/HttpRequest.h

@@ -262,6 +262,7 @@ class DROGON_EXPORT HttpRequest
     /**
      * kHttp10 means Http version is 1.0
      * kHttp11 means Http version is 1.1
+     * kHttp20 means Http version is 2.0
      */
     virtual Version version() const = 0;
 

lib/inc/drogon/HttpResponse.h

@@ -328,6 +328,7 @@ class DROGON_EXPORT HttpResponse
     /**
      * kHttp10 means Http version is 1.0
      * kHttp11 means Http version is 1.1
+     * kHttp2 means Http version is 2.0
      */
     virtual Version version() const = 0;
 

lib/inc/drogon/HttpTypes.h

@@ -89,11 +89,14 @@ enum HttpStatusCode
     k511NetworkAuthenticationRequired = 511
 };
 
+// TODO: Add an option to use default HTTP version
+// as the server may not support HTTP/2.
 enum class Version
 {
     kUnknown = 0,
     kHttp10,
-    kHttp11
+    kHttp11,
+    kHttp2,
 };
 
 enum ContentType

lib/src/Http1xTransport.cc

@@ -0,0 +1,105 @@
+#include "Http1xTransport.h"
+#include "HttpResponseParser.h"
+
+using namespace drogon;
+
+Http1xTransport::Http1xTransport(trantor::TcpConnectionPtr connPtr,
+                                 Version version,
+                                 size_t *bytesSent,
+                                 size_t *bytesReceived)
+    : connPtr(connPtr),
+      bytesSent_(bytesSent),
+      bytesReceived_(bytesReceived),
+      version_(version)
+{
+    connPtr->setContext(std::make_shared<HttpResponseParser>(connPtr));
+}
+
+void Http1xTransport::sendRequestInLoop(const HttpRequestPtr &req,
+                                        HttpReqCallback &&callback)
+{
+    sendReq(req);
+    pipeliningCallbacks_.emplace(std::move(req), std::move(callback));
+}
+
+void Http1xTransport::onRecvMessage(const trantor::TcpConnectionPtr &conn,
+                                    trantor::MsgBuffer *msg)
+{
+    auto responseParser = connPtr->getContext<HttpResponseParser>();
+    assert(responseParser != nullptr);
+    assert(connPtr.get() == conn.get());
+
+    // LOG_TRACE << "###:" << msg->readableBytes();
+    auto msgSize = msg->readableBytes();
+    while (msg->readableBytes() > 0)
+    {
+        if (pipeliningCallbacks_.empty())
+        {
+            LOG_ERROR << "More responses than expected!";
+            connPtr->shutdown();
+            return;
+        }
+        auto &firstReq = pipeliningCallbacks_.front();
+        if (firstReq.first->method() == Head)
+        {
+            responseParser->setForHeadMethod();
+        }
+        if (!responseParser->parseResponse(msg))
+        {
+            *bytesReceived_ += (msgSize - msg->readableBytes());
+            errorCallback(ReqResult::BadResponse);
+            return;
+        }
+        if (responseParser->gotAll())
+        {
+            auto resp = responseParser->responseImpl();
+            responseParser->reset();
+            *bytesReceived_ += (msgSize - msg->readableBytes());
+            msgSize = msg->readableBytes();
+            respCallback(resp, std::move(firstReq), conn);
+
+            pipeliningCallbacks_.pop();
+        }
+        else
+        {
+            *bytesReceived_ += (msgSize - msg->readableBytes());
+            break;
+        }
+    }
+}
+
+Http1xTransport::~Http1xTransport()
+{
+}
+
+bool Http1xTransport::handleConnectionClose()
+{
+    auto responseParser = connPtr->getContext<HttpResponseParser>();
+    if (responseParser && responseParser->parseResponseOnClose() &&
+        responseParser->gotAll())
+    {
+        auto &firstReq = pipeliningCallbacks_.front();
+        if (firstReq.first->method() == Head)
+        {
+            responseParser->setForHeadMethod();
+        }
+        auto resp = responseParser->responseImpl();
+        responseParser->reset();
+        respCallback(resp, std::move(firstReq), connPtr);
+        return false;
+    }
+    return true;
+}
+
+void Http1xTransport::sendReq(const HttpRequestPtr &req)
+{
+    trantor::MsgBuffer buffer;
+    assert(req);
+    auto implPtr = static_cast<HttpRequestImpl *>(req.get());
+    assert(version_ == Version::kHttp10 || version_ == Version::kHttp11);
+    implPtr->appendToBuffer(&buffer, version_);
+    LOG_TRACE << "Send request:"
+              << std::string_view(buffer.peek(), buffer.readableBytes());
+    *bytesSent_ += buffer.readableBytes();
+    connPtr->send(std::move(buffer));
+}

lib/src/Http1xTransport.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <trantor/net/EventLoop.h>
+#include <trantor/net/TcpClient.h>
+#include <list>
+#include <queue>
+#include <vector>
+#include "HttpTransport.h"
+
+namespace drogon
+{
+class Http1xTransport : public HttpTransport
+{
+  private:
+    std::queue<std::pair<HttpRequestPtr, HttpReqCallback>> pipeliningCallbacks_;
+    trantor::TcpConnectionPtr connPtr;
+    size_t *bytesSent_;
+    size_t *bytesReceived_;
+    Version version_{Version::kHttp11};
+
+    void sendReq(const HttpRequestPtr &req);
+
+  public:
+    Http1xTransport(trantor::TcpConnectionPtr connPtr,
+                    Version version,
+                    size_t *bytesSent,
+                    size_t *bytesReceived);
+    virtual ~Http1xTransport();
+    void sendRequestInLoop(const HttpRequestPtr &req,
+                           HttpReqCallback &&callback) override;
+    void onRecvMessage(const trantor::TcpConnectionPtr &,
+                       trantor::MsgBuffer *) override;
+
+    size_t requestsInFlight() const override
+    {
+        return pipeliningCallbacks_.size();
+    }
+
+    bool handleConnectionClose() override;
+
+    void onError(ReqResult result) override
+    {
+        while (!pipeliningCallbacks_.empty())
+        {
+            auto &cb = pipeliningCallbacks_.front().second;
+            cb(result, nullptr);
+            pipeliningCallbacks_.pop();
+        }
+    }
+};
+
+}  // namespace drogon