Split scheduler_perf config into subdirectories

Author: macsko

build/dependencies.yaml

@@ -224,23 +224,23 @@ dependencies:
       match: registry.k8s.io\/pause:\d+\.\d+
     - path: test/integration/benchmark-controller.json
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-default.yaml
+    - path: test/integration/scheduler_perf/templates/pod-default.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-node-affinity.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-node-affinity.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-pod-affinity.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-pod-affinity.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-pod-anti-affinity.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-pod-anti-affinity.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-preferred-pod-affinity.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-preferred-pod-affinity.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-preferred-pod-anti-affinity.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-preferred-pod-anti-affinity.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-preferred-topology-spreading.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-preferred-topology-spreading.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-secret-volume.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-secret-volume.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
-    - path: test/integration/scheduler_perf/config/templates/pod-with-topology-spreading.yaml
+    - path: test/integration/scheduler_perf/templates/pod-with-topology-spreading.yaml
       match: registry.k8s.io\/pause:\d+\.\d+
     - path: test/utils/image/manifest.go
       match: configs\[Pause\] = Config{list\.GcRegistry, "pause", "\d+\.\d+(.\d+)?"}

test/integration/scheduler_perf/README.md

@@ -33,10 +33,10 @@ Currently the test suite has the following:
 
 ```shell
 # In Kubernetes root path
-make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling"
+make test-integration WHAT=./test/integration/scheduler_perf/... ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling"
 ```
 
-The benchmark suite runs all the tests specified under config/performance-config.yaml.
+The benchmark suite runs all the tests specified under subdirectories split by topic (`<topic>/performance-config.yaml`).
 By default, it runs all workloads that have the "performance" label. In the configuration,
 labels can be added to a test case and/or individual workloads. Each workload also has
 all labels of its test case. The `perf-scheduling-label-filter` command line flag can
@@ -46,11 +46,12 @@ a comma-separated list of label names. Each label may have a `+` or `-` as prefi
 be set. For example, this runs all performance benchmarks except those that are labeled
 as "integration-test":
 ```shell
-make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -perf-scheduling-label-filter=performance,-integration-test"
+make test-integration WHAT=./test/integration/scheduler_perf/... ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -perf-scheduling-label-filter=performance,-integration-test"
 ```
 
-Once the benchmark is finished, JSON file with metrics is available in the current directory (test/integration/scheduler_perf). Look for `BenchmarkPerfScheduling_benchmark_YYYY-MM-DDTHH:MM:SSZ.json`.
-You can use `-data-items-dir` to generate the metrics file elsewhere.
+Once the benchmark is finished, JSON files with metrics are available in the subdirectories (`test/integration/scheduler_perf/config/<topic>`). 
+Look for `BenchmarkPerfScheduling_benchmark_YYYY-MM-DDTHH:MM:SSZ.json`.
+You can use `-data-items-dir` to generate the metrics files elsewhere.
 
 In case you want to run a specific test in the suite, you can specify the test through `-bench` flag:
 
@@ -59,19 +60,19 @@ Otherwise, the golang benchmark framework will try to run a test more than once
 
 ```shell
 # In Kubernetes root path
-make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling/SchedulingBasic/5000Nodes/5000InitPods/1000PodsToSchedule"
+make test-integration WHAT=./test/integration/scheduler_perf/... ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling/SchedulingBasic/5000Nodes/5000InitPods/1000PodsToSchedule"
 ```
 
 To produce a cpu profile:
 
 ```shell
 # In Kubernetes root path
-make test-integration WHAT=./test/integration/scheduler_perf KUBE_TIMEOUT="-timeout=3600s" ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -cpuprofile ~/cpu-profile.out"
+make test-integration WHAT=./test/integration/scheduler_perf/... KUBE_TIMEOUT="-timeout=3600s" ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -cpuprofile ~/cpu-profile.out"
 ```
 
 ### How to configure benchmark tests
 
-Configuration file located under `config/performance-config.yaml` contains a list of templates.
+Configuration files located under `<topic>/performance-config.yaml` contain a list of templates.
 Each template allows to set:
 - node manifest
 - manifests for initial and testing pod
@@ -85,7 +86,7 @@ for available operations to build `WorkloadTemplate`.
 Initial pods create a state of a cluster before the scheduler performance measurement can begin.
 Testing pods are then subject to performance measurement.
 
-The configuration file under `config/performance-config.yaml` contains a default list of templates to cover
+The configuration files under `<topic>/performance-config.yaml` contain a default list of templates to cover
 various scenarios. In case you want to add your own, you can extend the list with new templates.
 It's also possible to extend `op` data type, respectively its underlying data types
 to extend configuration of possible test cases.
@@ -115,10 +116,10 @@ removes that file only if the test passed.
 
 To run integration tests, use:
 ```
-make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_ARGS=-use-testing-log
+make test-integration WHAT=./test/integration/scheduler_perf/... KUBE_TEST_ARGS=-use-testing-log
 ```
 
-Integration testing uses the same `config/performance-config.yaml` as
+Integration testing uses the same configs (`<topic>/performance-config.yaml`) as
 benchmarking. By default, workloads labeled as `integration-test`
 are executed as part of integration testing (in ci-kubernetes-integration-master job).
 `-test-scheduling-label-filter` can be used to change that.
@@ -139,7 +140,7 @@ The test cases labeled as `short` are executed in pull-kubernetes-integration jo
 | pull-kubernetes-integration      | integration-test,short |
 | ci-benchmark-scheduler-perf      | performance            |
 
-See the comment on [./config/performance-config.yaml](./config/performance-config.yaml) for the details.
+See the comment on [./misc/performance-config.yaml](./misc/performance-config.yaml) for the details.
 
 ## Scheduling throughput thresholds
 
@@ -182,15 +183,15 @@ Some support for visualizing progress over time is built into the
 benchmarks. The measurement operation which creates pods writes .dat files like
 this:
 
-     test/integration/scheduler_perf/SchedulingBasic_5000Nodes_2023-03-17T14:52:09Z.dat
+     test/integration/scheduler_perf/misc/SchedulingBasic_5000Nodes_2023-03-17T14:52:09Z.dat
 
 This file is in a text format that [gnuplot](http://www.gnuplot.info/) can
 read. A wrapper script selects some suitable parameters:
 
-     test/integration/scheduler_perf/gnuplot.sh test/integration/scheduler_perf/*.dat
+     test/integration/scheduler_perf/gnuplot.sh test/integration/scheduler_perf/*/*.dat
 
 It plots in an interactive window by default. To write into a file, use
 
     test/integration/scheduler_perf/gnuplot.sh \
        -e 'set term png; set output "<output>.png"' \
-       test/integration/scheduler_perf/*.dat
+       test/integration/scheduler_perf/*/*.dat

test/integration/scheduler_perf/affinity/affinity_test.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package affinity
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	_ "k8s.io/component-base/logs/json/register"
+	perf "k8s.io/kubernetes/test/integration/scheduler_perf"
+)
+
+func TestMain(m *testing.M) {
+	if err := perf.InitTests(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	m.Run()
+}
+
+func TestSchedulerPerf(t *testing.T) {
+	perf.RunIntegrationPerfScheduling(t, "performance-config.yaml")
+}
+
+func BenchmarkPerfScheduling(b *testing.B) {
+	perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "affinity", nil)
+}