New RPCConnectionManager: Single unified manager, no ringbuffer, OpenSSL-owned sockets

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.0.13]
+
+[7.0.13]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.13
+
+### Changed
+
+- TLS is now terminated by OpenSSL directly on the socket, rather than being relayed over the ringbuffer and decrypted through a memory BIO. The session interfaces in `include/ccf/node/session.h` and `include/ccf/research/custom_protocol_subsystem_interface.h` have changed shape accordingly: a session now receives and emits plaintext, taking ownership of the inbound buffer, and writes its output through a `ccf::SessionWriter` rather than a `tls::Context` (#8117).
+
+### Fixed
+
+- TLS interfaces without a configured certificate now remain unlistening until a certificate is supplied, so clients receive an immediate connection refusal rather than entering an unusable TLS connection (#8117).
+- Nodes configured with zero worker threads now wake the enclave main loop immediately when the OpenSSL transport queues work, rather than waiting for the next tick or polling timeout (#8117).
+
 ## [7.0.12]
 
 [7.0.12]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.12

CMakeLists.txt

@@ -715,10 +715,14 @@ if(BUILD_TESTS)
     )
 
     add_unit_test(
-      rpc_connections_test
-      ${CMAKE_CURRENT_SOURCE_DIR}/src/host/test/rpc_connections.cpp
+      openssl_server_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/host/test/openssl_server_test.cpp
+    )
+    target_link_libraries(openssl_server_test PRIVATE ccf_tasks uv)
+    target_compile_definitions(
+      openssl_server_test
+      PRIVATE TEST_HYBRID_TLS_GROUPS=$<BOOL:${TEST_HYBRID_TLS_GROUPS}>
     )
-    target_link_libraries(rpc_connections_test PRIVATE uv)
 
     add_unit_test(
       raft_test
@@ -874,12 +878,11 @@ if(BUILD_TESTS)
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/node_info_json.cpp
     )
 
-    add_unit_test(tls_test ${CMAKE_CURRENT_SOURCE_DIR}/src/tls/test/main.cpp)
-    target_link_libraries(tls_test PRIVATE ${CMAKE_THREAD_LIBS_INIT})
-    target_compile_definitions(
+    add_unit_test(
       tls_test
-      PRIVATE TEST_HYBRID_TLS_GROUPS=$<BOOL:${TEST_HYBRID_TLS_GROUPS}>
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/clients/tls/test/main.cpp
     )
+    target_link_libraries(tls_test PRIVATE ${CMAKE_THREAD_LIBS_INIT})
 
     add_unit_test(
       base64_test

doc/contribute/onboarding.rst

@@ -63,12 +63,13 @@ Note that this diagram deliberately does not represent host-to-enclave communica
 .. mermaid::
 
     flowchart TB
-        Client[HTTPS/1.1 Client <a href='../build_apps/auth/index.html'>auth</a>] -- TLS 1.2 or 1.3 --> TLSSession
-        TLSSession[TLS Session <a href='https://github.com/microsoft/CCF/blob/main/src/enclave/tls_session.h'>src</a>] -- PlainText --> HTTPSession
+        Client[HTTPS/1.1 Client <a href='../build_apps/auth/index.html'>auth</a>] -- TLS 1.2 or 1.3 --> TLSConnection
+        TLSConnection[TLS Connection <a href='https://github.com/microsoft/CCF/blob/main/src/host/tls/openssl_server.h'>src</a>] -- PlainText --> HTTPSession
+        HTTPSession -- PlainText --> SessionWriter[Session Writer <a href='https://github.com/microsoft/CCF/blob/main/src/enclave/session_writer.h'>src</a>]
+        SessionWriter --> TLSConnection
         HTTPSession[HTTP Session <a href='https://github.com/microsoft/CCF/blob/main/src/http/http_session.h'>src</a>] -- Request --> Endpoint[Application Endpoint <a href='../build_apps/api.html#application-endpoint-registration'>doc</a>]
         Endpoint -- Response --> HTTPSession
-        HTTPSession --> TLSSession
-        TLSSession --> Client
+        TLSConnection --> Client
         Endpoint -- WriteSet --> Store[Store <a href='../build_apps/kv/index.html'>doc</a>]
         Store -- LedgerEntry --> Ledger[Ledger <a href='../architecture/ledger.html'>doc</a>]
         Ledger -- LedgerEntry --> Disk

include/ccf/node/session.h

@@ -3,7 +3,7 @@
 #pragma once
 
 #include <cstdint>
-#include <span>
+#include <vector>
 
 namespace ccf
 {
@@ -12,8 +12,55 @@ namespace ccf
   public:
     virtual ~Session() = default;
 
-    virtual void handle_incoming_data(std::span<const uint8_t> data) = 0;
+    // Inbound bytes for this session. Ownership is transferred, so that the
+    // buffer the transport has already built is moved through the session
+    // rather than copied again.
+    virtual void handle_incoming_data(std::vector<uint8_t>&& data) = 0;
     virtual void send_data(std::vector<uint8_t>&& data) = 0;
     virtual void close_session() = 0;
   };
+
+  // Abstract output sink injected into Sessions: a Session hands its outbound
+  // bytes (and connection-teardown requests) to a SessionWriter, which is
+  // implemented by the RPC transport.
+  //
+  // IMPORTANT: Sessions may invoke these methods from worker threads, so
+  // implementations MUST be thread-safe and must marshal any socket operations
+  // onto their I/O thread if required.
+  class SessionWriter
+  {
+  public:
+    virtual ~SessionWriter() = default;
+
+    // `id` throughout is the connection identifier the session was created
+    // with (ccf::tls::ConnID).
+
+    // Queue bytes to be written to the socket associated with `id`. Ownership
+    // is transferred, so that a response which may be arbitrarily large is
+    // moved through to the transport rather than copied again.
+    //
+    // Fire-and-forget: there is currently no backpressure signal.
+    //
+    // FUTURE: to surface genuine TCP-layer backpressure, an implementation
+    // should report when a connection's pending-write queue exceeds a watermark
+    // (tracking per-connection queued bytes) and return a writable/would-block
+    // status here.
+    virtual void write_outbound(int64_t id, std::vector<uint8_t>&& data) = 0;
+
+    // Tear down the connection: stop the underlying socket and drop the
+    // session.
+    virtual void close_socket(int64_t id) = 0;
+
+    // Report that `bytes` of previously delivered inbound data have now been
+    // processed. The transport uses this to decide when it may read more: it
+    // stops reading once the node is holding more unprocessed inbound data
+    // than it is willing to, and resumes as sessions catch up. Without it a
+    // client could make the node queue work faster than it retires it, for as
+    // long as it liked.
+    //
+    // A session which does not report is not penalised beyond its own
+    // connection - the transport releases whatever is still outstanding when
+    // the connection closes - so this defaults to a no-op.
+    virtual void inbound_consumed(int64_t /*id*/, size_t /*bytes*/) {}
+  };
 }

include/ccf/node/startup_config.h

@@ -29,6 +29,12 @@ namespace ccf
 
     ccf::ds::SizeString historical_cache_soft_limit = {"512MB"};
 
+    // How long an idle RPC (client TLS) connection is kept before it is closed.
+    // std::nullopt disables idle closure (connections are never closed for
+    // being idle).
+    std::optional<ccf::ds::TimeString> idle_connection_timeout =
+      ccf::ds::TimeString("60s");
+
     ccf::consensus::Configuration consensus = {};
     ccf::NodeInfoNetwork network;
 

include/ccf/research/custom_protocol_subsystem_interface.h

@@ -16,15 +16,14 @@ namespace ccf
 {
   namespace tls
   {
-    class Context;
     using ConnID = int64_t;
   }
 
   class CustomProtocolSubsystemInterface : public AbstractNodeSubSystem
   {
   public:
     using CreateSessionFn = std::function<std::shared_ptr<Session>(
-      ccf::tls::ConnID, const std::unique_ptr<tls::Context>&&)>;
+      ccf::tls::ConnID, ccf::SessionWriter&)>;
 
     ~CustomProtocolSubsystemInterface() override = default;
 
@@ -41,7 +40,7 @@ namespace ccf
     virtual std::shared_ptr<Session> create_session(
       const std::string& protocol_name,
       ccf::tls::ConnID conn_id,
-      const std::unique_ptr<tls::Context>&& ctx) = 0;
+      ccf::SessionWriter& writer) = 0;
 
     struct Essentials
     {

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "7.0.12"
+version = "7.0.13"
 authors = [
   { name="CCF Team", email="CCF-Sec@microsoft.com" },
 ]

src/clients/tls/README.md

@@ -0,0 +1,14 @@
+# TLS certificate helpers (client tooling only)
+
+These helpers configure an OpenSSL client with a trusted root and an owned
+identity:
+
+- `CA` parses one or more root certificates and populates an `SSL_CTX`'s
+  trusted certificate store.
+- `Cert` holds a client's own certificate and private key, and applies them
+  (and the peer verification mode) to an `SSL_CTX` or `SSL`.
+
+Their only consumer is `TlsClient` in the parent directory, used by the C++
+test and perf clients. **No node code uses them.** The node's inbound TLS is
+handled by `src/host/tls`, which builds its own `SSL_CTX`, and its outbound
+requests go through libcurl.

src/tls/cert.h → src/clients/tls/cert.h

@@ -3,9 +3,9 @@
 #pragma once
 
 #include "ccf/crypto/openssl/openssl_wrappers.h"
+#include "clients/tls/ca.h"
 #include "crypto/openssl/ec_key_pair.h"
 #include "ds/internal_logger.h"
-#include "tls/ca.h"
 
 #include <cstring>
 #include <memory>

src/clients/tls/test/main.cpp

@@ -0,0 +1,129 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+
+// Unit tests for the certificate helpers used by the C++ test clients, which
+// configure an OpenSSL context and connection with a trusted root and an owned
+// identity. The TLS transport itself is tested in
+// src/host/test/openssl_server_test.cpp.
+
+#include "ccf/crypto/ec_key_pair.h"
+#include "ccf/crypto/verifier.h"
+#include "ccf/ds/x509_time_fmt.h"
+#include "clients/tls/ca.h"
+#include "clients/tls/cert.h"
+#include "crypto/certs.h"
+
+#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+#include <chrono>
+#include <doctest/doctest.h>
+#include <memory>
+#include <optional>
+#include <string>
+
+namespace
+{
+  constexpr size_t certificate_validity_period_days = 365;
+
+  std::string valid_from_yesterday()
+  {
+    using namespace std::literals;
+    return ccf::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h);
+  }
+
+  ccf::crypto::Pem generate_self_signed_cert(
+    const ccf::crypto::ECKeyPairPtr& kp, const std::string& name)
+  {
+    return ccf::crypto::create_self_signed_cert(
+      kp, name, {}, valid_from_yesterday(), certificate_validity_period_days);
+  }
+
+  struct NetworkCA
+  {
+    ccf::crypto::ECKeyPairPtr kp;
+    ccf::crypto::Pem cert;
+  };
+
+  /// Get self-signed CA certificate.
+  NetworkCA get_ca()
+  {
+    auto kp = ccf::crypto::make_ec_key_pair();
+    return {kp, generate_self_signed_cert(kp, "CN=issuer")};
+  }
+
+  /// Creates a ::tls::Cert endorsed by the given CA.
+  std::unique_ptr<::tls::Cert> get_dummy_cert(
+    NetworkCA& net_ca, const std::string& name, bool auth_required = true)
+  {
+    auto ca = std::make_unique<::tls::CA>(net_ca.cert.str());
+
+    // Create a signing request and sign with the CA
+    auto kp = ccf::crypto::make_ec_key_pair();
+    auto crt = ccf::crypto::create_endorsed_cert(
+      kp,
+      "CN=" + name,
+      {},
+      valid_from_yesterday(),
+      certificate_validity_period_days,
+      net_ca.kp->private_key_pem(),
+      net_ca.cert);
+
+    // Verify node certificate with the CA's certificate
+    auto v = ccf::crypto::make_verifier(crt);
+    REQUIRE(v->verify_certificate({&net_ca.cert}));
+
+    return std::make_unique<::tls::Cert>(
+      std::move(ca), crt, kp->private_key_pem(), std::nullopt, auth_required);
+  }
+}
+
+TEST_CASE("CA configures trusted certificate store")
+{
+  auto ca = get_ca();
+  ::tls::CA trusted_ca(ca.cert.str(), true);
+  ccf::crypto::OpenSSL::Unique_SSL_CTX ctx(TLS_method());
+
+  trusted_ca.configure_trusted_cert_store(ctx);
+
+  auto* store = SSL_CTX_get_cert_store(ctx);
+  REQUIRE(store != nullptr);
+  auto* params = X509_STORE_get0_param(store);
+  REQUIRE(params != nullptr);
+  REQUIRE(
+    (X509_VERIFY_PARAM_get_flags(params) & X509_V_FLAG_PARTIAL_CHAIN) != 0);
+}
+
+TEST_CASE("Cert configures TLS verification and own certificate")
+{
+  auto ca = get_ca();
+
+  SUBCASE("auth_required requires a peer certificate")
+  {
+    auto cert = get_dummy_cert(ca, "server");
+    ccf::crypto::OpenSSL::Unique_SSL_CTX ctx(TLS_method());
+
+    cert->configure_context(ctx);
+    ccf::crypto::OpenSSL::Unique_SSL ssl(ctx);
+    cert->configure_connection(ssl);
+
+    constexpr auto expected_verify_mode =
+      SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+    REQUIRE(SSL_CTX_get_verify_mode(ctx) == expected_verify_mode);
+    REQUIRE(SSL_get_verify_mode(ssl) == expected_verify_mode);
+    REQUIRE(SSL_CTX_get0_certificate(ctx) != nullptr);
+    REQUIRE(SSL_get_certificate(ssl) != nullptr);
+  }
+
+  SUBCASE("without auth_required a peer certificate is requested, not required")
+  {
+    auto cert = get_dummy_cert(ca, "server", false);
+    ccf::crypto::OpenSSL::Unique_SSL_CTX ctx(TLS_method());
+
+    cert->configure_context(ctx);
+    ccf::crypto::OpenSSL::Unique_SSL ssl(ctx);
+    cert->configure_connection(ssl);
+
+    // The connection inherits the context's verification mode
+    REQUIRE((SSL_get_verify_mode(ssl) & SSL_VERIFY_PEER) != 0);
+    REQUIRE((SSL_get_verify_mode(ssl) & SSL_VERIFY_FAIL_IF_NO_PEER_CERT) == 0);
+  }
+}

src/clients/tls_client.h

@@ -3,9 +3,9 @@
 #pragma once
 
 #include "ccf/crypto/openssl/openssl_wrappers.h"
+#include "clients/tls/ca.h"
+#include "clients/tls/cert.h"
 #include "ds/internal_logger.h"
-#include "tls/ca.h"
-#include "tls/cert.h"
 
 #include <cstdint>
 #include <cstring>

src/common/configuration.h

@@ -143,7 +143,8 @@ namespace ccf
     files_cleanup,
     node_to_node_message_limit,
     historical_cache_soft_limit,
-    identity_history_fetch);
+    identity_history_fetch,
+    idle_connection_timeout);
 
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(RecoveryDecisionProtocolConfig);
   DECLARE_JSON_REQUIRED_FIELDS(

src/ds/work_beacon.h

@@ -55,6 +55,24 @@ namespace ccf::ds
 
       condition_variable.notify_all();
     }
+
+    void notify_work_available_coalesced()
+    {
+      bool notify = false;
+      {
+        std::lock_guard<std::mutex> lock(mutex);
+        if (work_available == 0)
+        {
+          work_available = 1;
+          notify = true;
+        }
+      }
+
+      if (notify)
+      {
+        condition_variable.notify_all();
+      }
+    }
   };
 
   using WorkBeaconPtr = std::shared_ptr<WorkBeacon>;