Adding Benchmarking to CI

.github/workflows/build.yml

@@ -7,6 +7,10 @@ on:
     branches: [ master ]
   merge_group:
 
+permissions:
+  contents: write  # Required to push to the repository
+  pages: write
+
 jobs:
   build:
     name: Build (Java ${{ matrix.java }} - ${{ matrix.os }})
@@ -84,3 +88,43 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           file: opendc-web/opendc-web-runner/Dockerfile
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Validate Gradle wrapper
+        uses: gradle/actions/wrapper-validation@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: 21
+      - name: Run benchmark
+        uses: gradle/actions/setup-gradle@v3
+        with:
+          arguments: jmh
+      - name: Combine JMH benchmark results
+        run: |
+          mkdir -p combined-results
+          echo "[]" > combined-results/results.json
+          find . -name "results.json" -exec cat {} + > all-results.tmp.json
+          jq -s 'add' all-results.tmp.json > combined-results/results.json
+          rm all-results.tmp.json
+      - name: Store benchmark results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: OpenDC benchmarks
+          tool: 'jmh'
+          output-file-path: combined-results/results.json
+          # Access token to deploy GitHub Pages branch
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+          comment-on-alert: true
+          alert-comment-cc-users: '@DanteNiewenhuis'
+
+          # Push and deploy GitHub pages branch automatically
+          auto-push: true
+          summary-always: true
+          benchmark-data-dir-path: site/dev/benchmarks

buildSrc/src/main/kotlin/benchmark-conventions.gradle.kts

@@ -34,12 +34,14 @@ configure<AllOpenExtension> {
 }
 
 jmh {
-    jmhVersion.set("1.35")
+    jmhVersion.set("1.37")
 
     profilers.add("stack")
     profilers.add("gc")
 
     includeTests.set(false) // Do not include tests by default
+
+    resultFormat = "JSON"
 }
 
 tasks.named("jmh", JMHTask::class) {

buildSrc/src/main/kotlin/testing-conventions.gradle.kts

@@ -42,21 +42,9 @@ dependencies {
     testRuntimeOnly(versionCatalog["junit.jupiter.engine"])
 }
 
-tasks.register<Test>("testsOn18") {
+tasks.register<Test>("testsOn21") {
     javaLauncher.set(javaToolchains.launcherFor {
-        languageVersion.set(JavaLanguageVersion.of(18))
-    })
-
-    useJUnitPlatform()
-
-    minHeapSize = "512m"
-    maxHeapSize = "1024m"
-    jvmArgs = listOf("-XX:MaxMetaspaceSize=512m")
-}
-
-tasks.register<Test>("testsOn19") {
-    javaLauncher.set(javaToolchains.launcherFor {
-        languageVersion.set(JavaLanguageVersion.of(19))
+        languageVersion.set(JavaLanguageVersion.of(21))
     })
 
     useJUnitPlatform()

gradle/libs.versions.toml

@@ -10,7 +10,7 @@ jackson = "2.16.1"
 jandex-gradle = "1.1.0"
 java = "21"
 jline = "3.25.1"
-jmh-gradle = "0.7.0"
+jmh-gradle = "0.7.3"
 jakarta = "3.0.2"
 junit-jupiter = "5.10.2"
 kotlin = "1.9.22"

opendc-experiments/opendc-experiments-base/build.gradle.kts

@@ -1,3 +1,7 @@
+import me.champeau.jmh.JMHTask
+import org.gradle.kotlin.dsl.assign
+import org.gradle.kotlin.dsl.named
+
 /*
  * Copyright (c) 2022 AtLarge Research
  *
@@ -27,6 +31,7 @@ plugins {
     `kotlin-library-conventions`
     `testing-conventions`
     `jacoco-conventions`
+    `benchmark-conventions`
     distribution
     kotlin("plugin.serialization") version "1.9.22"
 }
@@ -58,6 +63,17 @@ val createScenarioApp by tasks.creating(CreateStartScripts::class) {
     outputDir = layout.buildDirectory.dir("scripts").get().asFile
 }
 
+//tasks.named("jmh", JMHTask::class) {
+//    outputs.upToDateWhen { false } // XXX Do not cache the output of this task
+//    jvmArgs = listOf("-Djmh.outputFormat=json", "-Djmh.output=/reports/jmh/results-${project.name}.json")
+//    testRuntimeClasspath.setFrom() // XXX Clear test runtime classpath to eliminate duplicate dependencies on classpath
+//}
+
+//jmh {
+//    resultFormat = "JSON" // <--- Key: This sets the output format
+//    resultsFile = file("$buildDir/reports/jmh/results-${project.name}.json") // <--- Sets the output file
+//}
+
 // Create custom Scenario distribution
 distributions {
     main {

opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BasicBenchmark.kt

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2025 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base
+
+import org.opendc.compute.workload.Task
+import org.opendc.simulator.compute.workload.trace.TraceFragment
+import org.openjdk.jmh.annotations.Benchmark
+import org.openjdk.jmh.annotations.BenchmarkMode
+import org.openjdk.jmh.annotations.Fork
+import org.openjdk.jmh.annotations.Level
+import org.openjdk.jmh.annotations.Measurement
+import org.openjdk.jmh.annotations.Mode
+import org.openjdk.jmh.annotations.OutputTimeUnit
+import org.openjdk.jmh.annotations.Scope
+import org.openjdk.jmh.annotations.Setup
+import org.openjdk.jmh.annotations.State
+import org.openjdk.jmh.annotations.Warmup
+import java.util.ArrayList
+import java.util.concurrent.TimeUnit
+
+@BenchmarkMode(Mode.AverageTime) // or Mode.AverageTime
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.MILLISECONDS)
+@Warmup(iterations = 2, time = 1, timeUnit = TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+@Fork(1)
+class BasicBenchmark {
+    private lateinit var list: MutableList<Int>
+
+    @Setup(Level.Iteration)
+    fun setUp() {
+        list = mutableListOf()
+    }
+
+    /**
+     * FlowDistributor test 13: 1000 tasks. This tests the performance
+     * In this test, two tasks are scheduled, and they can both fit.
+     * However, task 0 increases its demand which overloads the FlowDistributor.
+     * This test shows how the FlowDistributor handles transition from fitting to overloading when multiple tasks are running.
+     * We check if both the host and the Tasks show the correct cpu usage and demand during the two fragments.
+     */
+    @Benchmark
+    fun benchmarkFlowDistributor1() {
+        val workload: ArrayList<Task> =
+            arrayListOf<Task>().apply {
+                repeat(1000) {
+                    this.add(
+                        createBenchmarkTask(
+                            name = "0",
+                            fragments =
+                                arrayListOf(TraceFragment(10 * 60 * 1000, 2000.0, 1)),
+                        ),
+                    )
+                }
+            }
+        val topology = createTopology("single_1_2000.json")
+
+        val monitor = runBenchmark(topology, workload)
+    }
+}