feat(object-store): upload large packs through S3 multipart

config/config.exs

@@ -9,6 +9,8 @@ config :logger, :console,
     :epoch,
     :reason,
     :operation,
+    :object_key,
+    :upload_id,
     :service,
     :status,
     :listener,

config/runtime.exs

@@ -43,20 +43,50 @@ if config_env() == :prod or System.get_env("CODE_S3_BUCKET") do
       """
   end
 
+  # Multipart tuning is left absent unless explicitly set, so the S3 backend
+  # falls back to its own defaults (100 MiB threshold, 64 MiB parts) rather
+  # than pinning them here.
+  pos_int_opt = fn env, key ->
+    case System.get_env(env) do
+      nil ->
+        []
+
+      value ->
+        case Integer.parse(String.trim(value)) do
+          {n, ""} when n > 0 -> [{key, n}]
+          _ -> raise ArgumentError, "#{env} must be a positive integer, got #{inspect(value)}"
+        end
+    end
+  end
+
   object_store =
     {
       Code.ObjectStore.S3,
-      # Path style is what MinIO, Tigris and Ceph expect. Set to "false" for
-      # virtual-hosted-style buckets on AWS proper.
-      bucket: require_env.("CODE_S3_BUCKET"),
-      endpoint: require_env.("CODE_S3_ENDPOINT"),
-      region: get.("CODE_S3_REGION", "auto"),
-      access_key_id: require_env.("CODE_S3_ACCESS_KEY_ID"),
-      secret_access_key: require_env.("CODE_S3_SECRET_ACCESS_KEY"),
-      prefix: get.("CODE_S3_PREFIX", ""),
-      path_style: get.("CODE_S3_PATH_STYLE", "true") == "true"
+      [
+        # Path style is what MinIO, Tigris and Ceph expect. Set to "false" for
+        # virtual-hosted-style buckets on AWS proper.
+        bucket: require_env.("CODE_S3_BUCKET"),
+        endpoint: require_env.("CODE_S3_ENDPOINT"),
+        region: get.("CODE_S3_REGION", "auto"),
+        access_key_id: require_env.("CODE_S3_ACCESS_KEY_ID"),
+        secret_access_key: require_env.("CODE_S3_SECRET_ACCESS_KEY"),
+        prefix: get.("CODE_S3_PREFIX", ""),
+        path_style: get.("CODE_S3_PATH_STYLE", "true") == "true"
+      ] ++
+        pos_int_opt.("CODE_S3_MULTIPART_THRESHOLD_BYTES", :multipart_threshold) ++
+        pos_int_opt.("CODE_S3_MULTIPART_PART_SIZE_BYTES", :multipart_part_size)
     }
 
+  # S3 only rejects an out-of-range part size after the whole object has been
+  # sent, so refuse it at boot instead.
+  with {_, opts} <- object_store,
+       {:ok, part_size} <- Keyword.fetch(opts, :multipart_part_size),
+       false <- Code.ObjectStore.S3.valid_part_size?(part_size) do
+    raise ArgumentError,
+          "CODE_S3_MULTIPART_PART_SIZE_BYTES must be between 5242880 (5 MiB) and " <>
+            "5368709120 (5 GiB), the part sizes S3 accepts; got #{part_size}"
+  end
+
   auth_backend = get.("CODE_AUTH_BACKEND", "webhook")
   # Refuses `none` in production and names the valid choices for a typo,
   # rather than failing later with a CaseClauseError.

docs/architecture.md

@@ -97,10 +97,19 @@ against the pack (its own checksum, the pack checksum it records, its object
 count) and rebuilt with `git index-pack` otherwise; it carries no digest in the
 log, so it is a hint rather than something to trust.
 
-The remaining ceiling is S3's: a single `PUT` cannot exceed **5 GiB**, and
-multipart upload is not implemented. A repository whose pack exceeds that fails
-at upload rather than silently truncating. Compaction keeps the base pack at
-the size of the current tree rather than of all history, so reaching this needs
+A single `PUT` cannot exceed **5 GiB** on S3, so packs above the configured
+multipart threshold (100 MiB by default) are uploaded through S3 multipart
+instead. Each part is streamed off disk in the same 1 MiB sub-chunks a single
+`PUT` uses, so a multipart upload never buffers a whole part in memory either.
+Create-only still holds: the upload is completed with `If-None-Match: *`, so a
+second writer racing for the same pack loses at completion exactly as it would
+on a single `PUT`, and its parts are aborted.
+The multipart ceiling is `part_size × 10 000` — a few hundred gibibytes at the
+default 64 MiB part size, and adjustable through `CODE_S3_MULTIPART_PART_SIZE_BYTES`
+if a single repository ever needs more. A pack whose size exceeds even that
+fails loudly at upload with the effective limit named in the error, rather
+than being silently truncated. Compaction keeps the base pack at the size of
+the current tree rather than of all history, so reaching either ceiling needs
 a genuinely enormous single repository.
 
 ### Entries

docs/operations.md

@@ -22,12 +22,51 @@ unchanged to every node. The only per-node value is `CODE_NODE_ID`.
 | `CODE_S3_REGION` | `auto` | |
 | `CODE_S3_PREFIX` | | Key prefix, for sharing a bucket |
 | `CODE_S3_PATH_STYLE` | `true` | `false` for virtual-hosted AWS buckets |
+| `CODE_S3_MULTIPART_THRESHOLD_BYTES` | `104857600` | Packs above this size are uploaded through S3 multipart instead of a single `PUT`. Default is 100 MiB. Clamped internally to 5 GiB (the single-`PUT` ceiling), so raising it above that has no effect |
+| `CODE_S3_MULTIPART_PART_SIZE_BYTES` | `67108864` | Bytes per multipart part. Default 64 MiB. Must be between 5 MiB and 5 GiB, the part sizes S3 accepts; anything else stops the node from booting. S3 allows at most 10 000 parts, so the largest object is `part_size × 10 000` |
 
 The store **must** support conditional writes (`If-Match`, `If-None-Match`) and
 conditional reads (`If-None-Match`). AWS S3, MinIO, Tigris, Cloudflare R2 and
 Ceph all do. Without them the compare-and-swap that orders pushes does not
 exist, and Code will not be safe.
 
+Packs above the multipart threshold are completed with `If-None-Match: *` on
+`CompleteMultipartUpload`, so the store must honour that header there too.
+AWS S3 documents it, and RustFS, which the end-to-end suite runs against,
+answers `412` to it; support on MinIO, Cloudflare R2, Ceph and Tigris has not
+been verified. A store that ignores the header degrades that one step to
+last-writer-wins, which is harmless for packs because two writers of the same
+pack key write the same bytes; a store that rejects it fails every pack above
+the threshold, so check before raising a deployment's pack sizes past it.
+
+A `409` from the store on a pack upload means a concurrent write or delete of
+the same key, not that the pack is there. Code checks whether it is, and
+fails the push, for the client to retry, when it is not. Before starting one, Code sends a `HEAD` for the pack so that a
+pack already stored is not uploaded again; a `403` to that request, which is
+what AWS answers credentials without `s3:ListBucket`, is treated as "unknown"
+rather than as a failure.
+
+A node that dies in the middle of a multipart upload leaves its parts behind.
+Code aborts an upload on every failure it survives, but not on one it does
+not, so add an incomplete-upload rule to the bucket's lifecycle policy:
+
+```json
+{
+  "Rules": [
+    {
+      "ID": "code-incomplete-multipart",
+      "Filter": {"Prefix": ""},
+      "Status": "Enabled",
+      "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 1}
+    }
+  ]
+}
+```
+
+An abort that itself fails is logged as a warning with
+`operation=multipart_abort`, the object key and the upload identifier, and
+leaves those parts to the same rule.
+
 ### Behaviour
 
 | Variable | Default | Notes |