adopt package-benchmark (#41)

* init benchmarking

* more thresholdes + docker

* working set request benchmark w docker

* soundness

* update benchmark script

* #35

* 5.10 support

* soundness checks build/ in subfolders

* benchmark as methods

* cleanup

* make use of upperBound

* docker sandbox fix

* removed scaling factor

* soundness

Author: dkz2

Benchmarks/.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+/.build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

Benchmarks/Benchmarks/MemcacheBenchmarks/MemcacheBenchmarks.swift

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the swift-memcache-gsoc open source project
+//
+// Copyright (c) 2023 Apple Inc. and the swift-memcache-gsoc project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of swift-memcache-gsoc project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Memcache
+import NIOCore
+import NIOPosix
+
+private let eventLoopGroup = MultiThreadedEventLoopGroup.singleton.next()
+
+let benchmarks = {
+    let defaultMetrics: [BenchmarkMetric] = [
+        .mallocCountTotal,
+    ]
+
+    Benchmark(
+        "Set Request",
+        configuration: .init(
+            metrics: defaultMetrics,
+            timeUnits: .milliseconds
+        )
+    ) { benchmark in
+        try await runSetRequest(iterations: benchmark.scaledIterations.upperBound, eventLoopGroup: eventLoopGroup)
+    }
+
+    Benchmark(
+        "Set with TTL Request",
+        configuration: .init(
+            metrics: defaultMetrics,
+            timeUnits: .milliseconds
+        )
+    ) { benchmark in
+        try await runSetWithTTLRequest(iterations: benchmark.scaledIterations.upperBound, eventLoopGroup: eventLoopGroup)
+    }
+    Benchmark(
+        "Delete Request",
+        configuration: .init(
+            metrics: defaultMetrics,
+            timeUnits: .milliseconds
+        )
+    ) { benchmark in
+        try await runDeleteRequest(iterations: benchmark.scaledIterations.upperBound, eventLoopGroup: eventLoopGroup)
+    }
+
+    Benchmark(
+        "Increment Request",
+        configuration: .init(
+            metrics: defaultMetrics,
+            timeUnits: .milliseconds
+        )
+    ) { benchmark in
+        try await runIncrementRequest(iterations: benchmark.scaledIterations.upperBound, eventLoopGroup: eventLoopGroup)
+    }
+
+    Benchmark(
+        "Decrement Request",
+        configuration: .init(
+            metrics: defaultMetrics,
+            timeUnits: .milliseconds
+        )
+    ) { benchmark in
+        try await runDecrementRequest(iterations: benchmark.scaledIterations.upperBound, eventLoopGroup: eventLoopGroup)
+    }
+}

Benchmarks/Benchmarks/MemcacheBenchmarks/Utilities.swift

@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the swift-memcache-gsoc open source project
+//
+// Copyright (c) 2023 Apple Inc. and the swift-memcache-gsoc project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of swift-memcache-gsoc project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Memcache
+import NIOCore
+import NIOPosix
+
+func runSetRequest(iterations: Int, eventLoopGroup: EventLoopGroup) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { group in
+        let memcacheConnection = MemcacheConnection(host: "memcached", port: 11211, eventLoopGroup: eventLoopGroup)
+        group.addTask { try await memcacheConnection.run() }
+
+        let setValue = "bar"
+
+        for _ in 0..<iterations {
+            try await memcacheConnection.set("foo", value: setValue)
+        }
+        group.cancelAll()
+    }
+}
+
+func runSetWithTTLRequest(iterations: Int, eventLoopGroup: EventLoopGroup) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { group in
+
+        let memcacheConnection = MemcacheConnection(host: "memcached", port: 11211, eventLoopGroup: eventLoopGroup)
+        group.addTask { try await memcacheConnection.run() }
+
+        let setValue = "foo"
+        let now = ContinuousClock.Instant.now
+        let expirationTime = now.advanced(by: .seconds(90))
+        let timeToLive = TimeToLive.expiresAt(expirationTime)
+
+        for _ in 0..<iterations {
+            try await memcacheConnection.set("bar", value: setValue, timeToLive: timeToLive)
+        }
+        group.cancelAll()
+    }
+}
+
+func runDeleteRequest(iterations: Int, eventLoopGroup: EventLoopGroup) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { group in
+
+        let memcacheConnection = MemcacheConnection(host: "memcached", port: 11211, eventLoopGroup: eventLoopGroup)
+        group.addTask { try await memcacheConnection.run() }
+
+        let setValue = "foo"
+        try await memcacheConnection.set("bar", value: setValue)
+
+        for _ in 0..<iterations {
+            try await memcacheConnection.delete("bar")
+        }
+        group.cancelAll()
+    }
+}
+
+func runIncrementRequest(iterations: Int, eventLoopGroup: EventLoopGroup) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { group in
+
+        let memcacheConnection = MemcacheConnection(host: "memcached", port: 11211, eventLoopGroup: eventLoopGroup)
+        group.addTask { try await memcacheConnection.run() }
+
+        let initialValue = 1
+        try await memcacheConnection.set("increment", value: initialValue)
+
+        let incrementAmount = 100
+        for _ in 0..<iterations {
+            try await memcacheConnection.increment("increment", amount: incrementAmount)
+        }
+        group.cancelAll()
+    }
+}
+
+func runDecrementRequest(iterations: Int, eventLoopGroup: EventLoopGroup) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { group in
+
+        let memcacheConnection = MemcacheConnection(host: "memcached", port: 11211, eventLoopGroup: eventLoopGroup)
+        group.addTask { try await memcacheConnection.run() }
+
+        let initialValue = 100
+        try await memcacheConnection.set("decrement", value: initialValue)
+
+        let decrementAmount = 10
+        for _ in 0..<iterations {
+            try await memcacheConnection.decrement("decrement", amount: decrementAmount)
+        }
+        group.cancelAll()
+    }
+}

Benchmarks/Package.swift

@@ -0,0 +1,43 @@
+// swift-tools-version: 5.7
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftCertificates open source project
+//
+// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import PackageDescription
+
+let package = Package(
+    name: "Benchmarks",
+    platforms: [
+        .macOS(.v13),
+    ],
+    dependencies: [
+        .package(name: "swift-memcache-gsoc", path: "../"),
+        .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.11.1"),
+        .package(url: "https://github.com/apple/swift-nio.git", from: "2.56.0"),
+    ],
+    targets: [
+        .executableTarget(
+            name: "MemcacheBenchmarks",
+            dependencies: [
+                .product(name: "Memcache", package: "swift-memcache-gsoc"),
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "NIOCore", package: "swift-nio"),
+                .product(name: "NIOPosix", package: "swift-nio"),
+            ],
+            path: "Benchmarks/MemcacheBenchmarks",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
+            ]
+        ),
+    ]
+)

Benchmarks/Thresholds/5.10/MemcacheBenchmarks.Decrement_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.10/MemcacheBenchmarks.Delete_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.10/MemcacheBenchmarks.Increment_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.10/MemcacheBenchmarks.Set_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.10/MemcacheBenchmarks.Set_with_TTL_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.7/MemcacheBenchmarks.Decrement_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.7/MemcacheBenchmarks.Delete_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.7/MemcacheBenchmarks.Increment_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.7/MemcacheBenchmarks.Set_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.7/MemcacheBenchmarks.Set_with_TTL_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.8/MemcacheBenchmarks.Decrement_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.8/MemcacheBenchmarks.Delete_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.8/MemcacheBenchmarks.Increment_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.8/MemcacheBenchmarks.Set_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.8/MemcacheBenchmarks.Set_with_TTL_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.9/MemcacheBenchmarks.Decrement_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 506
+}

Benchmarks/Thresholds/5.9/MemcacheBenchmarks.Delete_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 506
+}

Benchmarks/Thresholds/5.9/MemcacheBenchmarks.Increment_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/5.9/MemcacheBenchmarks.Set_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/5.9/MemcacheBenchmarks.Set_with_TTL_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/main/MemcacheBenchmarks.Decrement_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/main/MemcacheBenchmarks.Delete_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 507
+}

Benchmarks/Thresholds/main/MemcacheBenchmarks.Increment_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 506
+}

Benchmarks/Thresholds/main/MemcacheBenchmarks.Set_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}

Benchmarks/Thresholds/main/MemcacheBenchmarks.Set_with_TTL_Request.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 578
+}