Skip to content

Commit 83f67fa

Browse files
author
Abduqodiri Qurbonzoda
committed
Introduce benchmarks-runner project to ease benchmark running
1 parent 18b7c32 commit 83f67fa

23 files changed

+861
-8
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,5 @@
55
target
66
build
77
/kotlinx-collections-immutable/dependency-reduced-pom.xml
8+
/benchmarks-runner/benchmarkResults
9+
/benchmarks-runner/localReferenceBenchmarkResults

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ collection.mutate { some_actions_on(it) }
111111
112112
The library is published to [kotlinx](https://bintray.com/kotlin/kotlinx/kotlinx.collections.immutable) bintray repository.
113113

114-
The library depends on the Kotlin Standard Library of the version at least `1.3.30`.
114+
The library depends on the Kotlin Standard Library of the version at least `1.3.40`.
115115

116116
### Maven
117117

‎benchmarks-mpp/build.gradle

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
plugins {
2-
id 'org.jetbrains.kotlin.multiplatform' version '1.3.31'
3-
id 'org.jetbrains.gradle.benchmarks.plugin' version '0.1.7-dev-21'
2+
id 'org.jetbrains.kotlin.multiplatform' version '1.3.40'
3+
id 'kotlinx.benchmark' version '0.2.0-dev-2'
44
}
55

66
repositories {
77
mavenCentral()
88
maven { url 'https://dl.bintray.com/orangy/maven' }
9+
maven { url 'https://dl.bintray.com/kotlin/kotlinx' }
10+
maven { url 'https://dl.bintray.com/kotlin/kotlin-eap' }
911
}
1012

1113
apply plugin: 'maven-publish'
@@ -22,7 +24,7 @@ kotlin {
2224
dependencies {
2325
implementation kotlin('stdlib-jdk8')
2426
implementation project(path: ':kotlinx-collections-immutable')
25-
implementation 'org.jetbrains.gradle.benchmarks:runtime-jvm:0.1.7-dev-21'
27+
implementation 'org.jetbrains.kotlinx:kotlinx.benchmark.runtime-jvm:0.2.0-dev-2'
2628
}
2729
}
2830
jvmTest {
@@ -31,9 +33,19 @@ kotlin {
3133
}
3234

3335
benchmark {
34-
configurations {
36+
targets {
3537
register("jvm") {
3638
jmhVersion = "1.21"
3739
}
3840
}
3941
}
42+
43+
configurations {
44+
benchmarksJar
45+
}
46+
47+
afterEvaluate {
48+
artifacts {
49+
benchmarksJar jvmBenchmarkJar
50+
}
51+
}

‎benchmarks-runner/build.gradle

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
plugins {
2+
id 'org.jetbrains.kotlin.jvm' version '1.3.40'
3+
}
4+
5+
repositories {
6+
mavenCentral()
7+
jcenter()
8+
maven { url 'https://dl.bintray.com/orangy/maven' }
9+
}
10+
11+
dependencies {
12+
implementation "org.jetbrains.kotlin:kotlin-stdlib"
13+
implementation 'org.openjdk.jmh:jmh-core:1.21'
14+
15+
runtimeOnly project(path: ':benchmarks-mpp', configuration: 'benchmarksJar')
16+
runtimeOnly project(path: ':kotlinx-collections-immutable')
17+
}
18+
19+
// map
20+
task benchmarkHashMap(type: JavaExec, group: "Benchmark") {
21+
main = 'runners.HashMapRunnerKt'
22+
}
23+
24+
task benchmarkHashMapBuilder(type: JavaExec, group: "Benchmark") {
25+
main = 'runners.HashMapBuilderRunnerKt'
26+
}
27+
28+
task benchmarkOrderedMap(type: JavaExec, group: "Benchmark") {
29+
main = 'runners.OrderedMapRunnerKt'
30+
}
31+
32+
task benchmarkOrderedMapBuilder(type: JavaExec, group: "Benchmark") {
33+
main = 'runners.OrderedMapBuilderRunnerKt'
34+
}
35+
36+
task benchmarkAllMaps(group: "Benchmark") {
37+
dependsOn benchmarkHashMap
38+
dependsOn benchmarkHashMapBuilder
39+
dependsOn benchmarkOrderedMap
40+
dependsOn benchmarkOrderedMapBuilder
41+
}
42+
43+
// set
44+
task benchmarkHashSet(type: JavaExec, group: "Benchmark") {
45+
main = 'runners.HashSetRunnerKt'
46+
}
47+
48+
task benchmarkHashSetBuilder(type: JavaExec, group: "Benchmark") {
49+
main = 'runners.HashSetBuilderRunnerKt'
50+
}
51+
52+
task benchmarkOrderedSet(type: JavaExec, group: "Benchmark") {
53+
main = 'runners.OrderedSetRunnerKt'
54+
}
55+
56+
task benchmarkOrderedSetBuilder(type: JavaExec, group: "Benchmark") {
57+
main = 'runners.OrderedSetBuilderRunnerKt'
58+
}
59+
60+
task benchmarkAllSets(group: "Benchmark") {
61+
dependsOn benchmarkHashSet
62+
dependsOn benchmarkHashSetBuilder
63+
dependsOn benchmarkOrderedSet
64+
dependsOn benchmarkOrderedSetBuilder
65+
}
66+
67+
// list
68+
task benchmarkList(type: JavaExec, group: "Benchmark") {
69+
main = 'runners.ListRunnerKt'
70+
}
71+
72+
task benchmarkListBuilder(type: JavaExec, group: "Benchmark") {
73+
main = 'runners.ListBuilderRunnerKt'
74+
}
75+
76+
task benchmarkAllLists(group: "Benchmark") {
77+
dependsOn benchmarkList
78+
dependsOn benchmarkListBuilder
79+
}
80+
81+
// all
82+
task benchmarkAll(group: "Benchmark") {
83+
dependsOn benchmarkAllMaps
84+
dependsOn benchmarkAllSets
85+
dependsOn benchmarkAllLists
86+
}
87+
88+
89+
// configure runner tasks
90+
91+
def benchmarkParams = [
92+
'remote',
93+
'forks',
94+
'measurementIterations',
95+
'measurementTime',
96+
'warmupIterations',
97+
'warmupTime',
98+
// 'exclude',
99+
// 'include',
100+
'size',
101+
'hashCodeType',
102+
'immutablePercentage'
103+
]
104+
105+
tasks.withType(JavaExec) {
106+
if (group == "Benchmark") {
107+
classpath = sourceSets.main.runtimeClasspath
108+
109+
benchmarkParams.forEach { param ->
110+
if (project.hasProperty(param)) {
111+
systemProperty(param, project.property(param))
112+
}
113+
}
114+
}
115+
}

‎benchmarks-runner/src/main/kotlin/contants.kt

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2016-2019 JetBrains s.r.o.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import org.openjdk.jmh.runner.options.TimeValue
18+
19+
20+
const val localReferenceBenchmarkResultsDirectory = "localReferenceBenchmarkResults"
21+
const val remoteReferenceBenchmarkResultsDirectory = "remoteReferenceBenchmarkResults"
22+
const val benchmarkResultsDirectory = "benchmarkResults"
23+
24+
const val hashMapOutputFileName = "hashMap"
25+
const val hashMapBuilderOutputFileName = "hashMapBuilder"
26+
const val orderedMapOutputFileName = "orderedMap"
27+
const val orderedMapBuilderOutputFileName = "orderedMapBuilder"
28+
29+
const val hashSetOutputFileName = "hashSet"
30+
const val hashSetBuilderOutputFileName = "hashSetBuilder"
31+
const val orderedSetOutputFileName = "orderedSet"
32+
const val orderedSetBuilderOutputFileName = "orderedSetBuilder"
33+
34+
const val listOutputFileName = "list"
35+
const val listBuilderOutputFileName = "listBuilder"
36+
37+
38+
const val benchmarkMethod = "Benchmark"
39+
const val benchmarkScore = "Score(ns/op)"
40+
const val benchmarkScoreError = "ScoreError(ns/op)"
41+
const val benchmarkAllocRate = "AllocRate(B/op)"
42+
43+
const val benchmarkScoreRegressPercent = "Score(%)"
44+
const val benchmarkAllocRateRegressPercent = "AllocRate(%)"
45+
46+
47+
const val sizeParam = "size"
48+
const val hashCodeTypeParam = "hashCodeType"
49+
const val implementationParam = "implementation"
50+
const val immutablePercentageParam = "immutablePercentage"
51+
52+
53+
val jvmArgs = arrayOf("-Xms2048m", "-Xmx2048m")
54+
55+
const val forks = 1
56+
const val warmupIterations = 10
57+
const val measurementIterations = 20
58+
val warmupTime = TimeValue.milliseconds(500)!!
59+
val measurementTime = TimeValue.milliseconds(1000)!!
60+
61+
val sizeParamValues = arrayOf("1", "10", "100", "1000", "10000", "100000", "1000000")
62+
val hashCodeTypeParamValues = arrayOf("ascending", "random", "collision", "nonExisting")
63+
val immutablePercentageParamValues = arrayOf("0.0", "20.0", "50.0", "90.0")

‎benchmarks-runner/src/main/kotlin/csvPrinter.kt

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright 2016-2019 JetBrains s.r.o.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import java.io.File
18+
import java.io.FileReader
19+
import java.io.FileWriter
20+
21+
22+
fun printCsvResults(benchmarkResults: BenchmarkResults, outputPath: String) {
23+
val paramsNamesString = benchmarkResults.paramsNames.joinToString(",")
24+
val csvHeader = "$benchmarkMethod,$paramsNamesString,$benchmarkScore,$benchmarkScoreError,$benchmarkAllocRate"
25+
26+
File(outputPath).parentFile?.mkdirs()
27+
val fileWriter = FileWriter(outputPath)
28+
29+
fileWriter.appendln(csvHeader)
30+
benchmarkResults.runResults.forEach { res ->
31+
val paramsValuesString = benchmarkResults.paramsNames.joinToString(",") { res.paramValue(it) }
32+
val csvRow = "${res.benchmark},$paramsValuesString,${res.score.formatted()},${res.scoreError.formatted()},${res.allocRate.formatted()}"
33+
fileWriter.appendln(csvRow)
34+
}
35+
36+
fileWriter.flush()
37+
fileWriter.close()
38+
}
39+
40+
41+
fun readCsvResults(file: File): BenchmarkResults {
42+
val fileReader = FileReader(file)
43+
val lines = fileReader.readLines().map { it.split(',') }
44+
fileReader.close()
45+
46+
check(lines.isNotEmpty())
47+
check(lines.all { it.size == lines.first().size })
48+
49+
val header = lines.first()
50+
51+
val benchmarkColumn = header.indexOf(benchmarkMethod)
52+
val scoreColumn = header.indexOf(benchmarkScore)
53+
val scoreErrorColumn = header.indexOf(benchmarkScoreError)
54+
val allocRateColumn = header.indexOf(benchmarkAllocRate)
55+
56+
val paramsColumns = header.indices.filter {
57+
it !in listOf(benchmarkColumn, scoreColumn, scoreErrorColumn, allocRateColumn)
58+
}
59+
60+
val runResults = lines.drop(1).map { line ->
61+
BenchmarkRunResult(
62+
benchmark = line[benchmarkColumn],
63+
params = paramsColumns.associate { header[it] to line[it] },
64+
score = line[scoreColumn].toDouble(),
65+
scoreError = line[scoreErrorColumn].toDouble(),
66+
allocRate = line[allocRateColumn].toDouble()
67+
)
68+
}
69+
70+
return BenchmarkResults(paramsColumns.map { header[it] }, runResults)
71+
}
72+
73+
74+
fun Double.formatted(): String = "%.3f".format(this)

‎benchmarks-runner/src/main/kotlin/customRunResult.kt

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Copyright 2016-2019 JetBrains s.r.o.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import org.openjdk.jmh.results.RunResult
18+
19+
20+
class BenchmarkResults(
21+
val paramsNames: List<String>,
22+
val runResults: List<BenchmarkRunResult>
23+
)
24+
25+
fun Collection<RunResult>.toBenchmarkResults(): BenchmarkResults {
26+
val paramsNames = first().params.paramsKeys
27+
28+
check(all { it.params.paramsKeys == paramsNames })
29+
30+
return BenchmarkResults(paramsNames.toList(), map(RunResult::toBenchmarkRunResult))
31+
}
32+
33+
34+
class BenchmarkRunResult(
35+
val benchmark: String,
36+
val params: Map<String, String>,
37+
val score: Double,
38+
val scoreError: Double,
39+
val allocRate: Double
40+
) {
41+
fun paramValue(paramName: String): String = params.getValue(paramName)
42+
}
43+
44+
private fun RunResult.toBenchmarkRunResult(): BenchmarkRunResult {
45+
val allocRateLabel = "·gc.alloc.rate.norm"
46+
val allocRate = secondaryResults[allocRateLabel]!!
47+
48+
check(primaryResult.getScoreUnit() == "us/op")
49+
check(allocRate.getScoreUnit() == "B/op")
50+
51+
val nanosInMicros = 1000
52+
val size = params.getParam(sizeParam).toInt()
53+
54+
return BenchmarkRunResult(
55+
benchmark = params.benchmark,
56+
params = params.paramsKeys.associateWith { params.getParam(it) },
57+
score = primaryResult.getScore() * nanosInMicros / size,
58+
scoreError = primaryResult.getScoreError() * nanosInMicros / size,
59+
allocRate = allocRate.getScore() / size
60+
)
61+
}

0 commit comments

Comments
 (0)