diff --git a/CHANGELOG.md b/CHANGELOG.md index 76a51b66..7e1d51df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,16 @@ Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] [Unreleased]: https://github.com/atlassian/report/compare/release-4.5.0...master +### Added +- Add `DistributionComparator` which uses a proper Hodges-Lehmann implementation using pseudo-median instead of median + +### Fixed +- Fix calculating distribution ratio in `RelativeNonparametricPerformanceJudge` by `DistributionComparator` + +### Deprecated +- Deprecate `ShiftedDistributionRegressionTest` in favor of `DistributionComparator` + + ## [4.5.0] - 2024-07-01 [4.5.0]: https://github.com/atlassian/report/compare/release-4.4.0...release-4.5.0 diff --git a/src/main/kotlin/com/atlassian/performance/tools/report/api/ShiftedDistributionRegressionTest.kt b/src/main/kotlin/com/atlassian/performance/tools/report/api/ShiftedDistributionRegressionTest.kt index 1f74c9bf..02e4cc4e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/report/api/ShiftedDistributionRegressionTest.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/report/api/ShiftedDistributionRegressionTest.kt @@ -13,7 +13,9 @@ import org.apache.commons.math3.stat.descriptive.rank.Median * @experiment experiment durations * @param mwAlpha Mann-Whitney significance level * @param ksAlpha Kolmogorov-Smirnov significance level + * @deprecated Use [DistributionComparator] instead */ +@Deprecated("Use DistributionComparator instead") class ShiftedDistributionRegressionTest( private val baseline: DoubleArray, private val experiment: DoubleArray, @@ -57,7 +59,12 @@ class ShiftedDistributionRegressionTest( } internal fun overcomesTolerance(tolerance: Double): Boolean { - return isExperimentRegressed(tolerance) || isExperimentImproved(tolerance) + val isExperimentRegressed = isExperimentRegressed(tolerance) + val isExperimentImproved = isExperimentImproved(tolerance) + if (isExperimentImproved && isExperimentRegressed) { + throw IllegalArgumentException("Experiment can't be both regressed and improved at the same time") + } + return isExperimentRegressed || isExperimentImproved } /** diff --git a/src/main/kotlin/com/atlassian/performance/tools/report/api/distribution/DistributionComparator.kt b/src/main/kotlin/com/atlassian/performance/tools/report/api/distribution/DistributionComparator.kt new file mode 100644 index 00000000..0532b46b --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/report/api/distribution/DistributionComparator.kt @@ -0,0 +1,113 @@ +package com.atlassian.performance.tools.report.api.distribution + +import com.numericalmethod.suanshu.stats.test.rank.wilcoxon.WilcoxonRankSum +import org.apache.commons.math3.stat.descriptive.rank.Median +import org.apache.commons.math3.stat.ranking.NaNStrategy + +class DistributionComparator private constructor( + private val baseline: DoubleArray, + private val experiment: DoubleArray, + /** + * A percentage by which experiment can be slower/faster than baseline and not considered as a regression/improvement + */ + private val tolerance: Double, + private val significance: Double +) { + + + + /** + * Performs a one-tailed Mann–Whitney U test to check whether experiment is not slower than the baseline + * + * @return true if the experiment is slower than the baseline by more than tolerance, false otherwise + */ + private fun isExperimentRegressed(baselineMedian: Double): Boolean { + val mu = -tolerance * baselineMedian + return WilcoxonRankSum(baseline, experiment, mu).pValue1SidedLess < significance + } + + private fun isExperimentImproved(baselineMedian: Double): Boolean { + val mu = -tolerance * baselineMedian + val wilcoxon = WilcoxonRankSum(experiment, baseline, mu) + return wilcoxon.pValue1SidedLess < significance + } + + /** + * Pseudo-median: the median of the Walsh (pairwise) averages + */ + private fun pseudoMedian(array: DoubleArray): Double { + val n = array.size + val size = n * (n + 1) / 2 - n + val values = DoubleArray(size) + var k = 0 + for (i in 0 until n) { + for (j in i + 1 until n) { + values[k++] = (array[i] + array[j]) / 2 + } + } + return Median().evaluate(values) + } + + private fun median(func: (xi: Double, yj: Double) -> Double): Double { + val values = DoubleArray(baseline.size * experiment.size) + var k = 0 + for (i in baseline.indices) { + for (j in experiment.indices) { + values[k++] = func(baseline[i], experiment[j]) + } + } + return Median().withNaNStrategy(NaNStrategy.MINIMAL).evaluate(values) + } + + private fun shift(): Double { + return median { xi, yj -> yj - xi } + } + + private fun ratio(): Double { + return median { xi, yj -> yj / xi } + } + + /** + * Calculates the distance between two data sets based on the [Hodges-Lehmann estimator][]. + * [Hodges-Lehmann estimator]: https://en.wikipedia.org/wiki/Hodges%E2%80%93Lehmann_estimator + * https://aakinshin.net/hodges-lehmann-estimator/ + * https://github.com/AndreyAkinshin/perfolizer/blob/master/src/Perfolizer/Perfolizer/Mathematics/GenericEstimators/HodgesLehmannEstimator.cs + * + * Takes into account tolerance which answers the question "is change is big enough to matter?" + */ + fun compare(): DistributionComparison { + val experimentShift = shift() + val baselineMedian = pseudoMedian(baseline) + val experimentRatio = ratio() + val isExperimentImproved = isExperimentImproved(baselineMedian) + val isExperimentRegressed = isExperimentRegressed(baselineMedian) + val experimentRelativeChange = experimentRatio - 1 + return DistributionComparison( + experimentRelativeChange = experimentRelativeChange, + experimentAbsoluteChange = experimentShift, + isExperimentRegressed = isExperimentRegressed, + isExperimentImproved = isExperimentImproved + ) + } + + class Builder( + private var baseline: DoubleArray, + private var experiment: DoubleArray + ) { + private var significance: Double = 0.05 + private var tolerance: Double = 0.01 + + fun significance(significance: Double) = apply { this.significance = significance } + fun tolerance(tolerance: Double) = apply { this.tolerance = tolerance } + fun baseline(baseline: DoubleArray) = apply { this.baseline = baseline } + fun experiment(experiment: DoubleArray) = apply { this.experiment = experiment } + + fun build() = DistributionComparator( + baseline = baseline, + experiment = experiment, + tolerance = tolerance, + significance = significance + ) + + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/report/api/distribution/DistributionComparison.kt b/src/main/kotlin/com/atlassian/performance/tools/report/api/distribution/DistributionComparison.kt new file mode 100644 index 00000000..39d18bdf --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/report/api/distribution/DistributionComparison.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.report.api.distribution + +class DistributionComparison( + val experimentRelativeChange: Double, + val experimentAbsoluteChange: Double, + val isExperimentRegressed: Boolean, + val isExperimentImproved: Boolean +) { + + init { + if (isExperimentImproved && isExperimentRegressed) { + throw IllegalArgumentException("Experiment can't be both regressed and improved at the same time") + } + } + + fun hasImpact() = isExperimentRegressed || isExperimentImproved + +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/report/api/judge/RelativeNonparametricPerformanceJudge.kt b/src/main/kotlin/com/atlassian/performance/tools/report/api/judge/RelativeNonparametricPerformanceJudge.kt index bdc9d624..9d93bbbf 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/report/api/judge/RelativeNonparametricPerformanceJudge.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/report/api/judge/RelativeNonparametricPerformanceJudge.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.report.api.judge import com.atlassian.performance.tools.jiraactions.api.ActionType import com.atlassian.performance.tools.report.ActionMetricsReader -import com.atlassian.performance.tools.report.api.ShiftedDistributionRegressionTest +import com.atlassian.performance.tools.report.api.distribution.DistributionComparator import com.atlassian.performance.tools.report.api.junit.FailedAssertionJUnitReport import com.atlassian.performance.tools.report.api.junit.JUnitReport import com.atlassian.performance.tools.report.api.junit.SuccessfulJUnitReport @@ -70,14 +70,16 @@ class RelativeNonparametricPerformanceJudge private constructor( report = FailedAssertionJUnitReport(reportName, "No action $label results for $experimentCohort"), action = action ) - val test = ShiftedDistributionRegressionTest(baseline, experiment, mwAlpha = significance, ksAlpha = 0.0) - // shifts are negated, because ShiftedDistributionRegressionTest is relative to experiment, instead of baseline + val comparison = DistributionComparator.Builder(baseline, experiment) + .tolerance(toleranceRatio.toDouble()) + .build() + .compare() val impact = LatencyImpact.Builder( action, - -test.percentageShift, - reader.convertToDuration(-test.locationShift) + comparison.experimentRelativeChange, + reader.convertToDuration(comparison.experimentAbsoluteChange) ) - .relevant(test.overcomesTolerance(toleranceRatio.toDouble())) + .relevant(comparison.hasImpact()) .build() impactHandlers.forEach { it.accept(impact) } return if (impact.regression) { diff --git a/src/test/kotlin/com/atlassian/performance/tools/report/ShiftedDistributionRegressionTestTest.kt b/src/test/kotlin/com/atlassian/performance/tools/report/ShiftedDistributionRegressionTestTest.kt index 82439bd8..fa1e8dcd 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/report/ShiftedDistributionRegressionTestTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/report/ShiftedDistributionRegressionTestTest.kt @@ -1,6 +1,5 @@ package com.atlassian.performance.tools.report -import com.atlassian.performance.tools.jiraactions.api.* import com.atlassian.performance.tools.report.api.ShiftedDistributionRegressionTest import com.atlassian.performance.tools.report.api.result.FakeResults import com.atlassian.performance.tools.report.chart.Chart @@ -12,7 +11,7 @@ import org.apache.commons.math3.distribution.NormalDistribution import org.apache.commons.math3.distribution.NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY import org.apache.commons.math3.random.MersenneTwister import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.SoftAssertions.* +import org.assertj.core.api.SoftAssertions.assertSoftly import org.assertj.core.data.Offset import org.junit.Ignore import org.junit.Test @@ -112,7 +111,7 @@ class ShiftedDistributionRegressionTestTest { } /** - * In a 51% vs 49% case, small diffs should not dominate the big diffs. + * In a 51% slightly faster vs 49% much slower case, it should be a regression */ @Ignore("https://ecosystem.atlassian.net/browse/JPERF-1297") @Test @@ -152,16 +151,39 @@ class ShiftedDistributionRegressionTestTest { } } + @Test + @Ignore("percentageShift calculation is fixed in in DistributionComparator where Hodges-Lehmann estimator with pseudo-median is used") + fun shouldDetectImprovementWhenEveryPercentileBetter() { + // given + val baseline = + this.javaClass.getResource("/real-results/view issue 9.17.0 vs 10.0.0/baseline.csv").readText().lines() + .map { it.toDouble() }.toDoubleArray() + val experiment = + this.javaClass.getResource("/real-results/view issue 9.17.0 vs 10.0.0/experiment.csv").readText().lines() + .map { it.toDouble() }.toDoubleArray() + // when + val test = ShiftedDistributionRegressionTest(baseline, experiment) + // then + plotQuantiles(baseline, experiment) + assertSoftly { + it.assertThat(test.isExperimentRegressed(0.01)).`as`("isExperimentRegressed").isFalse() + it.assertThat(test.percentageShift).`as`("").`as`("percentageShift").isEqualTo(0.03941908713692943) + it.assertThat(test.locationShift).`as`("").`as`("locationShift").isEqualTo(20.0) + it.assertThat(test.overcomesTolerance(0.01)).`as`("overcomesTolerance").isTrue() + } + } private fun plotQuantiles( baseline: DoubleArray, experiment: DoubleArray ) { - val chart = Chart(listOf( - chartLine(baseline, "baseline"), - chartLine(experiment, "experiment") - )) + val chart = Chart( + listOf( + chartLine(baseline, "baseline"), + chartLine(experiment, "experiment") + ) + ) val htmlFile = Files.createTempFile("kebab", ".html") .also { println("Distribution comparison at $it") } DistributionComparison(GitRepo.findFromCurrentDirectory()).render(chart, htmlFile) @@ -174,7 +196,6 @@ class ShiftedDistributionRegressionTestTest { yAxisId = "latency-axis" ) - @Ignore("Known bug: https://ecosystem.atlassian.net/browse/JPERF-1188") @Test fun shouldSeeNoShiftAcrossTheSameResult() { val result = FakeResults.fastResult diff --git a/src/test/kotlin/com/atlassian/performance/tools/report/distribution/DistributionComparatorTest.kt b/src/test/kotlin/com/atlassian/performance/tools/report/distribution/DistributionComparatorTest.kt new file mode 100644 index 00000000..9c36e5e0 --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/report/distribution/DistributionComparatorTest.kt @@ -0,0 +1,193 @@ +package com.atlassian.performance.tools.report.distribution + +import com.atlassian.performance.tools.report.api.distribution.DistributionComparator +import com.atlassian.performance.tools.report.api.result.FakeResults +import com.atlassian.performance.tools.report.chart.Chart +import com.atlassian.performance.tools.report.chart.ChartLine +import com.atlassian.performance.tools.workspace.api.git.GitRepo +import org.apache.commons.math3.distribution.NormalDistribution +import org.apache.commons.math3.distribution.NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY +import org.apache.commons.math3.random.MersenneTwister +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.SoftAssertions.* +import org.junit.Ignore +import org.junit.Test +import java.nio.file.Files + +class DistributionComparatorTest { + + /** + * https://en.wikipedia.org/wiki/Robust_statistics + */ + @Test + fun shouldBeRobust() { + // given + val random = MersenneTwister(123) + val baseline = NormalDistribution(random, 400.0, 30.0, DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(500) + val outlier = 1_000_000_000_000.0 + val experiment = baseline + outlier + + // when + val comparison = DistributionComparator.Builder(baseline, experiment) + .tolerance(0.0) + .build().compare() + + // then + plotQuantiles(baseline, experiment) + assertSoftly { + it.assertThat(comparison.experimentAbsoluteChange).`as`("absolute change").isEqualTo(2.096466300542943E-4) + it.assertThat(comparison.experimentRelativeChange).`as`("relative change").isEqualTo(5.05322869548408E-7) + it.assertThat(comparison.isExperimentImproved).`as`("improved").isFalse + it.assertThat(comparison.isExperimentRegressed).`as`("regressed").isFalse + } + } + + @Test + fun shouldDescribeConstantSlowdown() { + // given + val random = MersenneTwister(123) + val fastMode = NormalDistribution(random, 400.0, 30.0, DEFAULT_INVERSE_ABSOLUTE_ACCURACY) + val slowMode = NormalDistribution(random, 6000.0, 150.0, DEFAULT_INVERSE_ABSOLUTE_ACCURACY) + val baseline = fastMode.sample(500) + slowMode.sample(90) + val slowdown = 800.0 + val experiment = baseline.map { it + slowdown }.toDoubleArray() + + // when + val comparison = DistributionComparator.Builder(baseline, experiment).build().compare() + + // then + plotQuantiles(baseline, experiment) + assertSoftly { + it.assertThat(comparison.experimentAbsoluteChange).`as`("absolute change").isEqualTo(800.0) + it.assertThat(comparison.experimentRelativeChange).`as`("relative change").isEqualTo(1.9951194021713592) + it.assertThat(comparison.isExperimentImproved).`as`("regressed").isFalse + it.assertThat(comparison.isExperimentRegressed).`as`("regressed").isTrue + } + } + + @Test + fun shouldDescribePartialSlowdown() { + // given + val random = MersenneTwister(123) + val fastMode = NormalDistribution(random, 400.0, 30.0, DEFAULT_INVERSE_ABSOLUTE_ACCURACY) + val slowMode = NormalDistribution(random, 6000.0, 150.0, DEFAULT_INVERSE_ABSOLUTE_ACCURACY) + val baseline = fastMode.sample(500) + slowMode.sample(90) + val verticalShift = 800.0 + val partBoundary = 0.40 * baseline.size + val experiment = baseline + .mapIndexed { index, latency -> + if (index < partBoundary) { + latency + verticalShift + } else { + latency + } + } + .toDoubleArray() + + // when + val comparison = DistributionComparator.Builder(baseline, experiment).build().compare() + + // then + plotQuantiles(baseline, experiment) + assertSoftly { + it.assertThat(comparison.experimentAbsoluteChange).`as`("absolute change").isEqualTo(66.57177057231809) + it.assertThat(comparison.experimentRelativeChange).`as`("relative change").isEqualTo(0.16145163153504116) + it.assertThat(comparison.isExperimentImproved).`as`("improved").isFalse + it.assertThat(comparison.isExperimentRegressed).`as`("regressed").isTrue + } + } + + /** + * In a 51% slightly faster vs 49% much slower case, it should be a regression + */ + @Test + @Ignore + fun shouldCareAboutHeightOfTheDifferences() { + // given + val random = MersenneTwister(123) + val fastMode = NormalDistribution(random, 400.0, 30.0, DEFAULT_INVERSE_ABSOLUTE_ACCURACY) + val baseline = fastMode.sample(500) + val slowdown = 800.0 + val speedup = -200.0 + val partBoundary = 0.51 * baseline.size + val experiment = baseline + .sorted() + .mapIndexed { index, latency -> + if (index < partBoundary) { + latency + speedup + } else { + latency + slowdown + } + } + .toDoubleArray() + + // when + val comparison = DistributionComparator.Builder(baseline, experiment).build().compare() + + // then + plotQuantiles(baseline, experiment) + assertSoftly { + it.assertThat(comparison.experimentAbsoluteChange).`as`("absolute change").isPositive + it.assertThat(comparison.experimentRelativeChange).`as`("relative change").isPositive + it.assertThat(comparison.isExperimentImproved).`as`("improvement").isFalse + it.assertThat(comparison.isExperimentRegressed).`as`("regressed").isTrue + } + } + + @Test + fun shouldDetectImprovementWhenEveryPercentileBetter() { + // given + val baseline = + this.javaClass.getResource("/real-results/view issue 9.17.0 vs 10.0.0/baseline.csv").readText().lines() + .map { it.toDouble() }.toDoubleArray() + val experiment = + this.javaClass.getResource("/real-results/view issue 9.17.0 vs 10.0.0/experiment.csv").readText().lines() + .map { it.toDouble() }.toDoubleArray() + // when + val comparison = DistributionComparator.Builder(baseline, experiment).build().compare() + // then + plotQuantiles(baseline, experiment) + assertThat(comparison.isExperimentImproved).isTrue() + assertThat(comparison.isExperimentRegressed).isFalse() + assertThat(comparison.experimentRelativeChange).isEqualTo(-0.03941908713692943) + } + + + private fun plotQuantiles( + baseline: DoubleArray, + experiment: DoubleArray + ) { + val chart = Chart( + listOf( + chartLine(baseline, "baseline"), + chartLine(experiment, "experiment") + ) + ) + val htmlFile = Files.createTempFile("distribution-quantiles", ".html") + .also { println("Distribution comparison at $it") } + DistributionComparison(GitRepo.findFromCurrentDirectory()).render(chart, htmlFile) + } + + private fun chartLine(data: DoubleArray, label: String) = ChartLine( + data = QuantileFunction().plot(data.toList()), + label = label, + type = "line", + yAxisId = "latency-axis" + ) + + @Test + fun shouldSeeNoShiftAcrossTheSameResult() { + // given + val result = FakeResults.fastResult + .actionMetrics.map { it.duration.toMillis() } + .map { it.toDouble() }.toDoubleArray() + + // when + val comparison = DistributionComparator.Builder(result, result).build().compare() + + // then + assertThat(comparison.experimentAbsoluteChange).isEqualTo(0.0) + assertThat(comparison.experimentRelativeChange).isEqualTo(0.0) + assertThat(comparison.hasImpact()).isFalse() + } +} diff --git a/src/test/resources/real-results/view issue 9.17.0 vs 10.0.0/baseline.csv b/src/test/resources/real-results/view issue 9.17.0 vs 10.0.0/baseline.csv new file mode 100644 index 00000000..77984c5f --- /dev/null +++ b/src/test/resources/real-results/view issue 9.17.0 vs 10.0.0/baseline.csv @@ -0,0 +1,480 @@ +693 +423 +450 +430 +639 +433 +477 +702 +490 +420 +479 +418 +521 +451 +803 +449 +480 +689 +442 +421 +437 +381 +490 +520 +574 +455 +431 +917 +784 +807 +450 +1020 +451 +726 +755 +523 +510 +632 +754 +660 +472 +731 +540 +757 +636 +440 +515 +716 +797 +678 +442 +1052 +494 +625 +785 +633 +463 +728 +405 +763 +567 +396 +442 +516 +818 +590 +414 +1275 +412 +1024 +441 +408 +429 +465 +679 +640 +655 +410 +421 +423 +643 +416 +410 +666 +403 +545 +461 +403 +423 +462 +669 +462 +438 +891 +411 +416 +419 +423 +840 +477 +807 +701 +665 +647 +731 +842 +911 +620 +811 +621 +657 +665 +1260 +709 +726 +708 +887 +869 +772 +505 +708 +938 +715 +721 +2331 +1036 +638 +715 +602 +610 +608 +900 +832 +802 +761 +752 +555 +719 +479 +691 +494 +719 +643 +742 +671 +686 +683 +752 +506 +782 +462 +461 +456 +1146 +466 +472 +882 +443 +434 +435 +421 +523 +472 +434 +468 +438 +776 +440 +594 +468 +423 +443 +527 +487 +2312 +625 +757 +468 +792 +522 +606 +702 +389 +448 +703 +628 +697 +438 +583 +880 +711 +705 +402 +554 +743 +789 +643 +447 +1381 +780 +908 +623 +423 +903 +404 +685 +481 +366 +430 +704 +724 +656 +439 +761 +417 +769 +554 +394 +558 +413 +820 +400 +469 +728 +393 +565 +426 +399 +410 +400 +842 +388 +452 +406 +395 +428 +467 +407 +425 +414 +731 +467 +602 +431 +516 +417 +432 +584 +841 +750 +702 +445 +681 +491 +1248 +606 +381 +401 +596 +684 +634 +425 +870 +478 +735 +629 +660 +437 +560 +829 +601 +424 +662 +434 +594 +487 +747 +523 +467 +1071 +478 +458 +465 +431 +468 +496 +458 +468 +454 +667 +438 +658 +483 +426 +499 +464 +734 +414 +429 +435 +709 +448 +456 +831 +435 +398 +434 +418 +556 +514 +438 +439 +430 +657 +488 +426 +434 +460 +514 +761 +841 +648 +417 +709 +400 +752 +462 +453 +419 +490 +678 +572 +400 +772 +392 +717 +475 +402 +416 +483 +680 +642 +409 +670 +419 +463 +439 +666 +468 +444 +622 +686 +456 +456 +429 +488 +479 +505 +502 +445 +666 +749 +537 +444 +405 +967 +839 +974 +458 +747 +518 +878 +754 +618 +457 +664 +781 +654 +470 +819 +490 +843 +786 +438 +545 +748 +861 +616 +453 +687 +428 +487 +480 +988 +507 +475 +774 +764 +482 +529 +453 +520 +513 +1232 +491 +451 +821 +751 +453 +555 +461 +1085 +751 +776 +466 +822 +752 +784 +1032 +445 +609 +831 +752 +790 +492 +989 +658 +1052 +782 +602 +490 +806 +815 +671 +506 +661 +395 +417 +427 +593 +452 +409 +693 +471 +430 +423 +388 +449 +468 +747 +541 +442 +780 +431 +394 +433 +397 +754 +695 +668 +515 +888 +411 +819 +568 +376 +433 +694 +716 +639 +424 +773 +448 +754 +594 +618 +426 +456 +654 +525 \ No newline at end of file diff --git a/src/test/resources/real-results/view issue 9.17.0 vs 10.0.0/experiment.csv b/src/test/resources/real-results/view issue 9.17.0 vs 10.0.0/experiment.csv new file mode 100644 index 00000000..19152912 --- /dev/null +++ b/src/test/resources/real-results/view issue 9.17.0 vs 10.0.0/experiment.csv @@ -0,0 +1,480 @@ +619 +427 +435 +452 +881 +556 +413 +769 +397 +485 +426 +387 +414 +422 +696 +454 +422 +572 +419 +518 +430 +383 +530 +447 +390 +445 +426 +835 +702 +841 +433 +494 +503 +670 +667 +460 +426 +643 +750 +603 +412 +636 +480 +783 +662 +372 +428 +609 +832 +620 +406 +665 +520 +711 +390 +403 +457 +665 +449 +411 +754 +436 +426 +413 +379 +396 +460 +388 +470 +405 +689 +437 +443 +429 +436 +714 +662 +652 +454 +725 +394 +760 +566 +389 +477 +515 +677 +709 +400 +724 +390 +761 +522 +411 +527 +416 +805 +599 +1200 +506 +666 +731 +633 +691 +817 +683 +657 +718 +781 +515 +794 +915 +611 +666 +677 +627 +770 +620 +783 +457 +607 +710 +578 +689 +2178 +622 +463 +742 +544 +620 +477 +672 +582 +625 +621 +681 +594 +668 +513 +646 +504 +716 +610 +625 +628 +830 +611 +741 +550 +2242 +622 +828 +449 +720 +504 +713 +778 +444 +581 +722 +717 +675 +445 +875 +486 +720 +725 +426 +463 +701 +721 +764 +459 +704 +679 +412 +427 +448 +805 +536 +422 +608 +550 +485 +537 +411 +410 +427 +713 +512 +418 +572 +606 +465 +456 +413 +464 +454 +463 +709 +408 +414 +422 +715 +446 +394 +729 +539 +558 +421 +481 +424 +434 +388 +471 +398 +707 +434 +400 +427 +390 +407 +416 +660 +752 +850 +680 +428 +729 +519 +739 +539 +391 +446 +481 +712 +624 +443 +686 +438 +780 +464 +378 +571 +476 +707 +748 +439 +706 +411 +421 +457 +763 +474 +416 +653 +658 +419 +435 +408 +413 +427 +767 +437 +409 +681 +407 +582 +450 +380 +433 +417 +721 +945 +712 +437 +924 +405 +656 +453 +424 +462 +500 +659 +602 +448 +867 +406 +713 +464 +464 +547 +460 +653 +426 +442 +683 +371 +432 +545 +590 +429 +438 +1010 +403 +451 +426 +375 +387 +444 +550 +414 +387 +657 +392 +414 +471 +364 +563 +673 +712 +550 +430 +689 +512 +659 +501 +563 +394 +507 +727 +378 +428 +729 +385 +734 +411 +370 +402 +423 +793 +533 +396 +660 +467 +490 +446 +423 +437 +407 +817 +900 +432 +476 +427 +446 +457 +587 +486 +408 +757 +596 +440 +450 +453 +663 +683 +901 +427 +787 +394 +854 +584 +399 +431 +504 +764 +585 +430 +738 +458 +768 +458 +386 +419 +565 +676 +591 +412 +751 +750 +661 +462 +810 +429 +798 +638 +463 +447 +577 +714 +693 +567 +748 +415 +755 +484 +405 +482 +555 +689 +717 +457 +702 +405 +442 +444 +823 +481 +426 +860 +839 +440 +451 +426 +416 +458 +554 +459 +407 +669 +638 +437 +452 +473 +694 +750 +634 +442 +879 +451 +768 +725 +428 +440 +612 +786 +616 +414 +870 +619 +824 +544 +403 +629 +446 +822 +441 +656 +416 +474 +445 +862 +462 +446 +796 +725 +530 +456 +478 +439 +458 +692 +484 +442 +723 +481 +433 +436 +407 \ No newline at end of file