rdma: single PUT for oversize buffers + fix >2 GiB upload truncation

.github/workflows/ci-rdma.yml

@@ -45,12 +45,19 @@ jobs:
       uses: actions/checkout@v4
       with:
         repository: microsoft/vcpkg
+        ref: "2026.07.29"
         path: "vcpkg"
 
+    # arm64 Linux has no prebuilt vcpkg CMake, so vcpkg falls back to the system
+    # cmake; the apt version (3.28) is too old for vcpkg's SPDX generation
+    # (string(JSON ... STRING_ENCODE)). Provide a recent cmake on PATH.
+    - name: Install recent CMake and Ninja
+      uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2
+
     - name: Install dependencies
       run: |
         sudo apt-get -qy update
-        sudo apt-get -qy install cmake libibverbs-dev librdmacm-dev libnuma-dev
+        sudo apt-get -qy install libibverbs-dev librdmacm-dev libnuma-dev
         cmake --version
 
     - name: Configure and Build

.github/workflows/ci.yml

@@ -103,8 +103,15 @@ jobs:
       uses: actions/checkout@v4
       with:
         repository: microsoft/vcpkg
+        ref: "2026.07.29"
         path: "vcpkg"
 
+    # arm64 Linux has no prebuilt vcpkg CMake, so vcpkg falls back to the system
+    # cmake; the apt version (3.28) is too old for vcpkg's SPDX generation
+    # (string(JSON ... STRING_ENCODE)). Provide a recent cmake on PATH.
+    - name: Install recent CMake and Ninja
+      uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2
+
     - name: Print env
       run: |
         echo github.event.action: ${{ github.event.action }}
@@ -113,8 +120,6 @@ jobs:
     - name: Install dependencies if Ubuntu
       if: startsWith(matrix.config.name, 'Ubuntu_Latest_GCC')
       run: |
-        sudo apt-get -qy update
-        sudo apt-get -qy install cmake
         wget --quiet https://dl.min.io/aistor/minio/release/linux-${{ matrix.config.arch }}/minio
         chmod +x minio
         cmake --version

README.md

@@ -179,6 +179,17 @@ for convenience; see `vendor/cuobj/NOTICE` for the applicable NVIDIA
 license terms. The default build (`MINIO_CPP_ENABLE_RDMA=OFF`) omits
 the entire RDMA stack and has no dependency on any of those libraries.
 
+### Buffer size limit
+
+A single cuObject registration (`cuMemObjGetDescriptor`) can pin at most
+**4 GiB** (`kCuObjMaxMemoryRegSize`). `PutObject`/`GetObject` therefore use
+RDMA only for buffers up to that size; a larger buffer is transferred over a
+single ordinary HTTP request instead (AIStor accepts a single PUT up to 5 TiB,
+far beyond the RDMA registration ceiling and beyond anything a client can
+realistically pin or allocate). The SDK does not chunk registrations — sizing
+the buffer you hand to the RDMA API is the caller's responsibility; supply a
+buffer ≤ 4 GiB to keep the transfer on the RDMA fast path.
+
 ## License
 
 This SDK is distributed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](https://github.com/minio/minio-cpp/blob/master/LICENSE) for more information.

include/miniocpp/rdma.h

@@ -67,6 +67,15 @@ inline constexpr int kRDMAReplyNotImplemented = 501;
 // Return codes for rdmaPut/rdmaGet
 inline constexpr ssize_t kRDMANotSupported = -2;
 
+// Maximum buffer a single cuObject registration (cuMemObjGetDescriptor) can
+// pin — 4 GiB. RDMA is only attempted for buffers up to this size; a larger
+// buffer cannot be registered for RDMA, so PutObject/GetObject transfer it over
+// a single ordinary HTTP request instead (the buffer is already resident, and
+// AIStor accepts a single PUT up to kMaxObjectSize == 5 TiB — far beyond this
+// limit and beyond anything a client can pin or allocate). Sizing the buffer is
+// the caller's responsibility; the SDK does not chunk registrations.
+inline constexpr size_t kCuObjMaxMemoryRegSize = 4ULL * 1024 * 1024 * 1024;
+
 // RDMA control-plane timeouts (seconds). The HTTP exchange carries only
 // the token and a few headers — keep them aggressive so a dead NIC surfaces
 // fast and the retry path can pick up the failover NIC.
@@ -214,8 +223,13 @@ inline static ssize_t rdmaPut(s3_rdma_client_ctx_t* sctx, const char* token,
   return static_cast<ssize_t>(size);
 }
 
+// range_offset < 0 reads the whole object; range_offset >= 0 reads size bytes
+// starting at that object offset. The object offset travels in a Range header
+// (the server derives its rangeBase from it) and is independent of the buffer
+// address carried in the RDMA token — see AIStor rdmaTransferBounds().
 inline static ssize_t rdmaGet(s3_rdma_client_ctx_t* sctx, const char* token,
-                              const void* buf, size_t size) {
+                              const void* buf, size_t size,
+                              int64_t range_offset = -1) {
   char rdma_token[256];
   snprintf(rdma_token, sizeof(rdma_token), "%s:%016lx:%016lx", token,
            (uint64_t)buf, (uint64_t)size);
@@ -240,6 +254,19 @@ inline static ssize_t rdmaGet(s3_rdma_client_ctx_t* sctx, const char* token,
   sign_headers.Add("x-amz-content-sha256", kUnsignedPayload);
   sign_headers.Add(kAmzRDMAToken, rdma_token);
 
+  // A byte-range request; added before signing so the server accepts the
+  // SignedHeaders. bytes=<offset>-<offset+size-1> selects the object range;
+  // the server replies 206 (kRDMAReplyPartialContent) for it.
+  if (range_offset >= 0) {
+    if (size == 0) return -1;  // a zero-length range would emit bytes=X-(X-1)
+    char range_hdr[64];
+    snprintf(
+        range_hdr, sizeof(range_hdr), "bytes=%lld-%lld",
+        static_cast<long long>(range_offset),
+        static_cast<long long>(range_offset + static_cast<int64_t>(size) - 1));
+    sign_headers.Add("Range", range_hdr);
+  }
+
   if (!creds.session_token.empty()) {
     sign_headers.Add("X-Amz-Security-Token", creds.session_token);
   }
@@ -330,7 +357,7 @@ inline static ssize_t rdmaPutWithRetry(cuObjClient* rdmaclient,
 // lifecycle and one retry for NIC failover.
 inline static ssize_t rdmaGetWithRetry(cuObjClient* rdmaclient,
                                        s3_rdma_client_ctx_t* sctx, void* buf,
-                                       size_t size) {
+                                       size_t size, int64_t range_offset = -1) {
   ssize_t ret = -1;
   for (int attempt = 0; attempt < kRDMAMaxAttempts; ++attempt) {
     char* token = nullptr;
@@ -339,7 +366,7 @@ inline static ssize_t rdmaGetWithRetry(cuObjClient* rdmaclient,
     if (terr != CU_OBJ_SUCCESS || token == nullptr) {
       return -1;
     }
-    ret = rdmaGet(sctx, token, buf, size);
+    ret = rdmaGet(sctx, token, buf, size, range_offset);
     rdmaclient->cuMemObjPutRDMAToken(token);
     if (ret > 0 || ret == kRDMANotSupported) {
       return ret;

src/client.cc

@@ -696,9 +696,20 @@ Result<GetObjectResponse> Client::GetObject(GetObjectArgs args) {
 
     const size_t size = *args.size;
 
+    // An explicit offset selects an object byte-range (size bytes from it),
+    // letting a caller stream an object larger than one registration as a
+    // sequence of <= 4 GiB ranged GETs. Unset means read the whole object.
+    const int64_t range_offset =
+        args.offset.has_value() ? static_cast<int64_t>(*args.offset) : -1;
+
     // Process-wide cuObjClient — see client.h for the race rationale.
+    // A buffer larger than a single cuObject registration
+    // (kCuObjMaxMemoryRegSize, 4 GiB) cannot be pinned for RDMA; skip straight
+    // to the HTTP path rather than issue a registration that is guaranteed to
+    // fail.
     cuObjClient& rdma_client = SharedRDMAClient();
-    bool use_rdma = (rdma_client.cuMemObjGetDescriptor(args.buf, size) == 0);
+    bool use_rdma = size <= kCuObjMaxMemoryRegSize &&
+                    rdma_client.cuMemObjGetDescriptor(args.buf, size) == 0;
 
     if (use_rdma) {
       s3_rdma_client_ctx getCtx = {
@@ -710,7 +721,8 @@ Result<GetObjectResponse> Client::GetObject(GetObjectArgs args) {
           .op = CUOBJ_GET,
       };
 
-      ssize_t ret = rdmaGetWithRetry(&rdma_client, &getCtx, args.buf, size);
+      ssize_t ret =
+          rdmaGetWithRetry(&rdma_client, &getCtx, args.buf, size, range_offset);
       rdma_client.cuMemObjPutDescriptor(args.buf);
 
       if (ret > 0) {
@@ -747,6 +759,12 @@ Result<GetObjectResponse> Client::GetObject(GetObjectArgs args) {
     targs.bucket = args.bucket;
     targs.object = args.object;
     targs.region = region;
+    // Mirror the RDMA range on the fallback: read the same size bytes from the
+    // same object offset (Headers() turns offset/length into a Range header).
+    if (range_offset >= 0) {
+      targs.offset = static_cast<size_t>(range_offset);
+      targs.length = size;
+    }
     targs.datafunc = [&ss = ss](minio::http::DataFunctionArgs args) -> bool {
       ss << args.datachunk;
       return true;
@@ -1213,8 +1231,12 @@ Result<PutObjectResponse> Client::PutObject(PutObjectArgs args) {
 
     const size_t size = *args.size;
 
+    // A buffer larger than a single cuObject registration
+    // (kCuObjMaxMemoryRegSize, 4 GiB) cannot be pinned for RDMA; skip straight
+    // to the single HTTP PUT below.
     cuObjClient& rdma_client = SharedRDMAClient();
-    bool use_rdma = (rdma_client.cuMemObjGetDescriptor(args.buf, size) == 0);
+    bool use_rdma = size <= kCuObjMaxMemoryRegSize &&
+                    rdma_client.cuMemObjGetDescriptor(args.buf, size) == 0;
 
     if (use_rdma) {
       s3_rdma_client_ctx putCtx = {
@@ -1269,15 +1291,21 @@ Result<PutObjectResponse> Client::PutObject(PutObjectArgs args) {
       src = stage.data();
     }
 
-    std::stringstream ss(std::ios_base::in | std::ios_base::out);
-    ss.rdbuf()->pubsetbuf(src, size);
-
-    PutObjectArgs http_args(ss, static_cast<uint64_t>(size), 16 * 1024 * 1024L);
-    http_args.bucket = args.bucket;
-    http_args.object = args.object;
-    http_args.region = region;
-    http_args.headers.Add("x-amz-content-sha256", "UNSIGNED-PAYLOAD");
-    return PutObject(http_args);
+    // Single PUT of the whole buffer via the request body — not multipart. The
+    // buffer is already fully resident (host, or staged from device above), so
+    // re-chunking it into parts buys nothing; AIStor accepts a single PUT up to
+    // kMaxObjectSize (5 TiB), far beyond kCuObjMaxMemoryRegSize (the 4 GiB RDMA
+    // registration ceiling that routed an oversized buffer here) and beyond
+    // anything a client can pin or allocate.
+    PutObjectApiArgs api_args;
+    api_args.bucket = args.bucket;
+    api_args.object = args.object;
+    api_args.region = region;
+    api_args.data = std::string_view(src, size);
+    api_args.buf = src;
+    api_args.size = size;
+    api_args.headers.Add("x-amz-content-sha256", "UNSIGNED-PAYLOAD");
+    return BaseClient::PutObject(api_args);
   }
 #endif
 
@@ -1321,7 +1349,8 @@ Result<PutObjectResponse> Client::PutObject(PutObjectArgs args) {
     if (rdma_connected) {
       for (unsigned int i = 0; i < max_inflight; i++) {
         char* pool_buf = static_cast<char*>(buf_pool[i].ptr);
-        if (rdma_client.cuMemObjGetDescriptor(pool_buf, args.part_size) == 0) {
+        if (args.part_size <= kCuObjMaxMemoryRegSize &&
+            rdma_client.cuMemObjGetDescriptor(pool_buf, args.part_size) == 0) {
           rdma_regs[i] = ScopedRDMARegistration(&rdma_client, pool_buf);
         }
       }
@@ -1613,7 +1642,7 @@ Result<PutObjectResponse> Client::PutObject(PutObjectArgs args) {
   // being non-null to even attempt the RDMA path.
   cuObjClient& rdma_client = SharedRDMAClient();
   ScopedRDMARegistration rdma_reg;
-  if (rdma_client.isConnected() &&
+  if (rdma_client.isConnected() && args.part_size <= kCuObjMaxMemoryRegSize &&
       rdma_client.cuMemObjGetDescriptor(buf, args.part_size) == 0) {
     rdma_reg = ScopedRDMARegistration(&rdma_client, buf);
     args.rdmaclient = &rdma_client;

src/http.cc

@@ -468,8 +468,12 @@ Response Request::execute() {
         headers.Add("Content-Length", std::to_string(body.size()));
       }
       request.setOpt(new curlpp::Options::ReadStream(&body_stream));
-      request.setOpt(
-          new curlpp::Options::InfileSize(static_cast<long>(body.size())));
+      // CURLOPT_INFILESIZE_LARGE (curl_off_t), not CURLOPT_INFILESIZE (long):
+      // the latter is documented to be capped at 2 GiB and silently truncates
+      // the upload for larger single-request bodies (e.g. a >4 GiB buffer that
+      // could not be RDMA-registered and falls back to a single PUT).
+      request.setOpt(new curlpp::Options::InfileSizeLarge(
+          static_cast<curl_off_t>(body.size())));
       request.setOpt(new curlpp::Options::Upload(true));
       break;
   }