diff --git a/performance/build.gradle b/performance/build.gradle index e89f6fe0f40..ab218d2efd1 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -88,6 +88,7 @@ MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) { workingDir = project.provider { kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile } + depends("build") args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}") } diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt index 4d9a18cf198..19b25871dab 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt @@ -40,7 +40,18 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc other.varianceBenchmark.score >= 0 && other.meanBenchmark.score - other.varianceBenchmark.score != 0.0, { "Mean and variance should be positive and not equal!" }) - val mean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score + val exactMean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score + // Analyze intervals. Calculate difference between border points. + val (bigValue, smallValue) = if (meanBenchmark.score > other.meanBenchmark.score) Pair(this, other) else Pair(other, this) + val bigValueIntervalStart = bigValue.meanBenchmark.score - bigValue.varianceBenchmark.score + val smallValueIntervalEnd = smallValue.meanBenchmark.score + smallValue.varianceBenchmark.score + if (smallValueIntervalEnd > bigValueIntervalStart) { + // Interval intersect. + return MeanVariance(0.0, 0.0) + } + val mean = ((smallValueIntervalEnd - bigValueIntervalStart) / bigValueIntervalStart) * + (if (meanBenchmark.score > other.meanBenchmark.score) -1 else 1) + val maxValueChange = abs(meanBenchmark.score + varianceBenchmark.score - other.meanBenchmark.score + other.varianceBenchmark.score) / abs(other.meanBenchmark.score + other.varianceBenchmark.score) @@ -49,7 +60,7 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc other.meanBenchmark.score - other.varianceBenchmark.score) / abs(other.meanBenchmark.score - other.varianceBenchmark.score) - val variance = abs(abs(mean) - max(minValueChange, maxValueChange)) + val variance = abs(abs(exactMean) - max(minValueChange, maxValueChange)) return MeanVariance(mean * 100, variance * 100) } diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt index 7a42e465962..bcfc258435d 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt @@ -145,9 +145,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, geometricMean(percentsList, benchmarksNumber) } else { // Geometric mean can be counted on positive numbers. - val precision = abs(getMaximumChange(bucket)) + 1 - percentsList = percentsList.map { it + precision } - geometricMean(percentsList, benchmarksNumber) - precision + percentsList = percentsList.map { abs(it) } + -geometricMean(percentsList, benchmarksNumber) } } diff --git a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt index 5bad651c192..100ad104809 100644 --- a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt +++ b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt @@ -49,7 +49,7 @@ class AnalyzerTests { val numbers = listOf(10.1, 10.2, 10.3) val value = computeMeanVariance(numbers) val expectedMean = 10.2 - val expectedVariance = 0.12539360253 + val expectedVariance = 0.07872455 assertTrue(abs(value.mean - expectedMean) < eps) assertTrue(abs(value.variance - expectedVariance) < eps) } @@ -59,7 +59,7 @@ class AnalyzerTests { val inputs = createMeanVarianceBenchmarks() val percent = inputs.first.calcPercentageDiff(inputs.second) - val expectedMean = -10.0 + val expectedMean = -9.99809998 val expectedVariance = 0.0021 assertTrue(abs(percent.mean - expectedMean) < eps) assertTrue(abs(percent.variance - expectedVariance) < eps)