plugin: Add resource bundles defined in Package.swift to container images (#78)

euanh · web-flow · commit 1d1fac560255 · 2025-04-11T10:35:34.000+01:00
Motivation ---------- A target in `Package.swift` can include [resources](https://developer.apple.com/documentation/packagedescription/target/resources). When the target is built, the resources are copied into a bundle directory and a [`Bundle`](https://developer.apple.com/documentation/foundation/bundle) class is generated, giving the executable a convenient way to retrieve them at run time. A target with resources should be able to access them when it is packaged in a container image. For example, a web server might bundle resources such as images or fonts. Fixes #48 Modifications ------------- * Add a new `--resource` parameter to `containertool`. This specifies a resource bundle directory which is added to the container image using the directory archiving support added to the `Tar` module in #74. When `containertool` is used alone, multiple `--resource` flags can be specified and will be added to the image in the order in which they appear on the command line. A target in `Package.swift` can only define one resource bundle, so when used through the plugin only one bundle will be added. * Update the plugin to pass the `--resource` flag to `containertool` when an executable target has resources. Result ------ If a target includes resources, the resource bundle will be copied into the container image at a path where the generated `Bundle` class can find it. If a target does not include resources, the container image will only include the executable. Test Plan --------- * Existing tests continue to pass. * New integration tests verify that all expected resources are present.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -35,6 +35,28 @@ jobs:
               run: |
                   swift test
 
+    containertool-resources-test:
+        name: Containertool resources test
+        runs-on: ubuntu-latest
+        services:
+            registry:
+                image: registry:2
+                ports:
+                    - 5000:5000
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+              with:
+                  persist-credentials: false
+
+            - name: Mark the workspace as safe
+              # https://github.com/actions/checkout/issues/766
+              run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+            - name: Check plugin streaming output is reassembled and printed properly
+              run: |
+                  scripts/test-containertool-resources.sh
+
     plugin-streaming-output-test:
         name: Plugin streaming output test
         runs-on: ubuntu-latest
diff --git a/.github/workflows/interop_tests.yml b/.github/workflows/interop_tests.yml
@@ -2,14 +2,10 @@ name: Interop tests
 
 on:
     workflow_call:
-#        inputs:
-#            example:
-#                required: true
-#                type: string
 
 jobs:
     layering-test:
-        name: Layering test
+        name: Containertool layering test
         runs-on: ubuntu-latest
         services:
             registry:
@@ -45,7 +41,7 @@ jobs:
                   grep second second.payload
 
     elf-detection-test:
-        name: ELF detection test
+        name: Containertool ELF detection test
         runs-on: ubuntu-latest
         services:
             registry:
@@ -71,4 +67,4 @@ jobs:
             # Run the test script
             - name: Test ELF detection
               run: |
-                  scripts/test-elf-detection.sh
+                  scripts/test-containertool-elf-detection.sh
diff --git a/Plugins/ContainerImageBuilder/main.swift b/Plugins/ContainerImageBuilder/main.swift
@@ -89,10 +89,18 @@ extension PluginError: CustomStringConvertible {
 
         for built in builtExecutables { Diagnostics.remark("Built product: \(built.url.path)") }
 
+        let resources = builtExecutables[0].url
+            .deletingLastPathComponent()
+            .appendingPathComponent(
+                "\(context.package.displayName)_\(productName).resources"
+            )
+
         // Run a command line helper to upload the image
-        let helper = try context.tool(named: "containertool")
-        let helperURL = helper.url
-        let helperArgs = extractor.remainingArguments + builtExecutables.map { $0.url.path }
+        let helperURL = try context.tool(named: "containertool").url
+        let helperArgs =
+            (FileManager.default.fileExists(atPath: resources.path) ? ["--resources", resources.path] : [])
+            + extractor.remainingArguments
+            + builtExecutables.map { $0.url.path }
         let helperEnv = ProcessInfo.processInfo.environment.filter { $0.key.starts(with: "CONTAINERTOOL_") }
 
         let err = Pipe()
diff --git a/Sources/containertool/ELFDetect.swift b/Sources/containertool/ELFDetect.swift
@@ -12,6 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+import class Foundation.FileHandle
+import struct Foundation.URL
+
 struct ArrayField<T: Collection> where T.Element == UInt8 {
     var start: Int
     var count: Int
@@ -52,6 +55,11 @@ extension Array where Element == UInt8 {
 /// architecture and operating system ABI for which that object
 /// was created.
 struct ELF: Equatable {
+    /// Minimum ELF header length is 52 bytes for a 32-bit ELF header.
+    /// A 64-bit header is 64 bytes.   A potential header must be at
+    /// least 52 bytes or it cannot possibly be an ELF header.
+    static let minHeaderLength = 52
+
     /// Multibyte ELF fields are stored in the native endianness of the target system.
     /// This field records the endianness of objects in the file.
     enum Endianness: UInt8 {
@@ -189,7 +197,7 @@ extension ELF {
         /// Object type: 2 bytes
         static let EI_TYPE = IntField<UInt16>(start: 0x10)
 
-        //l Machine ISA (processor architecture): 2 bytes
+        /// Machine ISA (processor architecture): 2 bytes
         static let EI_MACHINE = IntField<UInt16>(start: 0x12)
     }
 
@@ -205,7 +213,7 @@ extension ELF {
     static func read(_ bytes: [UInt8]) -> ELF? {
         // An ELF file starts with a magic number which is the same in either endianness.
         // The only defined ELF header version is 1.
-        guard bytes.count > 0x13, bytes[Field.EI_MAGIC] == ELFMagic, bytes[Field.EI_VERSION] == 1 else {
+        guard bytes.count >= minHeaderLength, bytes[Field.EI_MAGIC] == ELFMagic, bytes[Field.EI_VERSION] == 1 else {
             return nil
         }
 
@@ -226,3 +234,13 @@ extension ELF {
         )
     }
 }
+
+extension ELF {
+    static func read(at path: URL) throws -> ELF? {
+        let handle = try FileHandle(forReadingFrom: path)
+        guard let header = try handle.read(upToCount: minHeaderLength) else {
+            return nil
+        }
+        return ELF.read([UInt8](header))
+    }
+}
diff --git a/Sources/containertool/Extensions/Archive+appending.swift b/Sources/containertool/Extensions/Archive+appending.swift
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftContainerPlugin open source project
+//
+// Copyright (c) 2025 Apple Inc. and the SwiftContainerPlugin project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftContainerPlugin project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import class Foundation.FileManager
+import struct Foundation.Data
+import struct Foundation.FileAttributeType
+import struct Foundation.URL
+
+import Tar
+
+extension URL {
+    var isDirectory: Bool {
+        (try? resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true
+    }
+}
+
+extension Archive {
+    /// Append a file or directory tree to the archive.   Directory trees are appended recursively.
+    /// Parameters:
+    /// - root: The path to the file or directory to add.
+    /// Returns:  A new archive made by appending `root` to the receiver.
+    public func appendingRecursively(atPath root: String) throws -> Self {
+        let url = URL(fileURLWithPath: root)
+        if url.isDirectory {
+            return try self.appendingDirectoryTree(at: url)
+        } else {
+            return try self.appendingFile(at: url)
+        }
+    }
+
+    /// Append a single file to the archive.
+    /// Parameters:
+    /// - path: The path to the file to add.
+    /// Returns:  A new archive made by appending `path` to the receiver.
+    func appendingFile(at path: URL) throws -> Self {
+        try self.appendingFile(name: path.lastPathComponent, data: try [UInt8](Data(contentsOf: path)))
+    }
+
+    /// Recursively append a single directory tree to the archive.
+    /// Parameters:
+    /// - root: The path to the directory to add.
+    /// Returns:  A new archive made by appending `root` to the receiver.
+    func appendingDirectoryTree(at root: URL) throws -> Self {
+        var ret = self
+
+        guard let enumerator = FileManager.default.enumerator(atPath: root.path) else {
+            throw ("Unable to read \(root.path)")
+        }
+
+        for case let subpath as String in enumerator {
+            // https://developer.apple.com/documentation/foundation/filemanager/1410452-attributesofitem
+            // https://developer.apple.com/documentation/foundation/fileattributekey
+
+            guard let filetype = enumerator.fileAttributes?[.type] as? FileAttributeType else {
+                throw ("Unable to get file type for \(subpath)")
+            }
+
+            switch filetype {
+            case .typeRegular:
+                let resource = try [UInt8](Data(contentsOf: root.appending(path: subpath)))
+                try ret.appendFile(name: subpath, prefix: root.lastPathComponent, data: resource)
+
+            case .typeDirectory:
+                try ret.appendDirectory(name: subpath, prefix: root.lastPathComponent)
+
+            default:
+                throw "Resource file \(subpath) of type \(filetype) is not supported"
+            }
+        }
+
+        return ret
+    }
+}
diff --git a/Sources/containertool/Extensions/RegistryClient+Layers.swift b/Sources/containertool/Extensions/RegistryClient+Layers.swift
@@ -36,16 +36,24 @@ extension RegistryClient {
         }
     }
 
+    typealias DiffID = String
+    struct ImageLayer {
+        var descriptor: ContentDescriptor
+        var diffID: DiffID
+    }
+
     // A layer is a tarball, optionally compressed using gzip or zstd
     // See https://github.com/opencontainers/image-spec/blob/main/media-types.md
-    func uploadImageLayer(
+    func uploadLayer(
         repository: String,
-        layer: Data,
+        contents: [UInt8],
         mediaType: String = "application/vnd.oci.image.layer.v1.tar+gzip"
-    ) async throws -> ContentDescriptor {
-        // The layer blob is the gzipped tarball
-        let blob = Data(gzip([UInt8](layer)))
-        log("Uploading application layer")
-        return try await putBlob(repository: repository, mediaType: mediaType, data: blob)
+    ) async throws -> ImageLayer {
+        // The diffID is the hash of the unzipped layer tarball
+        let diffID = digest(of: contents)
+        // The layer blob is the gzipped tarball;  the descriptor is the hash of this gzipped blob
+        let blob = Data(gzip(contents))
+        let descriptor = try await putBlob(repository: repository, mediaType: mediaType, data: blob)
+        return ImageLayer(descriptor: descriptor, diffID: diffID)
     }
 }
diff --git a/Sources/containertool/containertool.swift b/Sources/containertool/containertool.swift
@@ -38,6 +38,9 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
     @Argument(help: "Executable to package")
     private var executable: String
 
+    @Option(help: "Resource bundle directory")
+    private var resources: [String] = []
+
     @Option(help: "Username")
     private var username: String?
 
@@ -72,9 +75,6 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
         let baseimage = try ImageReference(fromString: from, defaultRegistry: defaultRegistry)
         var destination_image = try ImageReference(fromString: repository, defaultRegistry: defaultRegistry)
 
-        let executableURL = URL(fileURLWithPath: executable)
-        let payload = try Data(contentsOf: executableURL)
-
         let authProvider: AuthorizationProvider?
         if !netrc {
             authProvider = nil
@@ -87,8 +87,9 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
             authProvider = try NetrcAuthorizationProvider(defaultNetrc)
         }
 
-        // Create clients for the source and destination registries
-        // The base image may be stored on a different registry, so two clients are needed.
+        // MARK: Create registry clients
+
+        // The base image may be stored on a different registry to the final destination, so two clients are needed.
         // `scratch` is a special case and requires no source client.
         let source: RegistryClient?
         if from == "scratch" {
@@ -113,7 +114,10 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
 
         // MARK: Find the base image
 
-        let elfheader = ELF.read([UInt8](payload))
+        // Try to detect the architecture of the application executable so a suitable base image can be selected.
+        // This reduces the risk of accidentally creating an image which stacks an aarch64 executable on top of an x86_64 base image.
+        let executableURL = URL(fileURLWithPath: executable)
+        let elfheader = try ELF.read(at: executableURL)
         let architecture =
             architecture
             ?? ProcessInfo.processInfo.environment["CONTAINERTOOL_ARCHITECTURE"]
@@ -146,29 +150,39 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
             if verbose { log("Using scratch as base image") }
         }
 
-        // MARK: Build the application layer
+        // MARK: Upload resource layers
 
-        let payload_name = executableURL.lastPathComponent
-        let tardiff = try tar([UInt8](payload), filename: payload_name)
-        log("Built application layer")
+        var resourceLayers: [RegistryClient.ImageLayer] = []
+        for resourceDir in resources {
+            let resourceTardiff = try Archive().appendingRecursively(atPath: resourceDir).bytes
+            let resourceLayer = try await destination.uploadLayer(
+                repository: destination_image.repository,
+                contents: resourceTardiff
+            )
 
-        // MARK: Upload the application layer
+            if verbose {
+                log("resource layer: \(resourceLayer.descriptor.digest) (\(resourceLayer.descriptor.size) bytes)")
+            }
+
+            resourceLayers.append(resourceLayer)
+        }
 
-        let application_layer = try await destination.uploadImageLayer(
+        // MARK: Upload the application layer
+        let applicationLayer = try await destination.uploadLayer(
             repository: destination_image.repository,
-            layer: Data(tardiff)
+            contents: try Archive().appendingFile(at: executableURL).bytes
         )
-
-        if verbose { log("application layer: \(application_layer.digest) (\(application_layer.size) bytes)") }
+        if verbose {
+            log("application layer: \(applicationLayer.descriptor.digest) (\(applicationLayer.descriptor.size) bytes)")
+        }
 
         // MARK: Create the application configuration
-
         let timestamp = Date(timeIntervalSince1970: 0).ISO8601Format()
 
         // Inherit the configuration of the base image - UID, GID, environment etc -
         // and override the entrypoint.
         var inherited_config = baseimage_config.config ?? .init()
-        inherited_config.Entrypoint = ["/\(payload_name)"]
+        inherited_config.Entrypoint = ["/\(executableURL.lastPathComponent)"]
         inherited_config.Cmd = []
         inherited_config.WorkingDir = "/"
 
@@ -182,7 +196,9 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
                 // The diff_id is the digest of the _uncompressed_ layer archive.
                 // It is used by the runtime, which might not store the layers in
                 // the compressed form in which it received them from the registry.
-                diff_ids: baseimage_config.rootfs.diff_ids + [digest(of: tardiff)]
+                diff_ids: baseimage_config.rootfs.diff_ids
+                    + resourceLayers.map { $0.diffID }
+                    + [applicationLayer.diffID]
             ),
             history: [.init(created: timestamp, created_by: "containertool")]
         )
@@ -200,7 +216,9 @@ enum AllowHTTP: String, ExpressibleByArgument, CaseIterable { case source, desti
             schemaVersion: 2,
             mediaType: "application/vnd.oci.image.manifest.v1+json",
             config: config_blob,
-            layers: baseimage_manifest.layers + [application_layer]
+            layers: baseimage_manifest.layers
+                + resourceLayers.map { $0.descriptor }
+                + [applicationLayer.descriptor]
         )
 
         // MARK: Upload base image
diff --git a/scripts/test-containertool-elf-detection.sh b/scripts/test-containertool-elf-detection.sh
diff --git a/scripts/test-containertool-resources.sh b/scripts/test-containertool-resources.sh
