Calculate percentage difference based on border values of intervals. (#2636)

performance/build.gradle

@@ -88,6 +88,7 @@ MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) {
     workingDir = project.provider {
         kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile
     }
+    depends("build")
     args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}")
 }
 

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt

@@ -40,7 +40,18 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc
                 other.varianceBenchmark.score >= 0 &&
                 other.meanBenchmark.score - other.varianceBenchmark.score != 0.0,
                 { "Mean and variance should be positive and not equal!" })
-        val mean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score
+        val exactMean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score
+        // Analyze intervals. Calculate difference between border points.
+        val (bigValue, smallValue) = if (meanBenchmark.score > other.meanBenchmark.score) Pair(this, other) else Pair(other, this)
+        val bigValueIntervalStart = bigValue.meanBenchmark.score - bigValue.varianceBenchmark.score
+        val smallValueIntervalEnd = smallValue.meanBenchmark.score + smallValue.varianceBenchmark.score
+        if (smallValueIntervalEnd > bigValueIntervalStart) {
+            // Interval intersect.
+            return MeanVariance(0.0, 0.0)
+        }
+        val mean = ((smallValueIntervalEnd - bigValueIntervalStart) / bigValueIntervalStart) *
+                (if (meanBenchmark.score > other.meanBenchmark.score) -1 else 1)
+
         val maxValueChange = abs(meanBenchmark.score + varianceBenchmark.score -
                         other.meanBenchmark.score + other.varianceBenchmark.score) /
                         abs(other.meanBenchmark.score + other.varianceBenchmark.score)
@@ -49,7 +60,7 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc
                         other.meanBenchmark.score - other.varianceBenchmark.score) /
                         abs(other.meanBenchmark.score - other.varianceBenchmark.score)
 
-        val variance = abs(abs(mean) - max(minValueChange, maxValueChange))
+        val variance = abs(abs(exactMean) - max(minValueChange, maxValueChange))
         return MeanVariance(mean * 100, variance * 100)
     }
 

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -145,9 +145,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
             geometricMean(percentsList, benchmarksNumber)
         } else {
             // Geometric mean can be counted on positive numbers.
-            val precision = abs(getMaximumChange(bucket)) + 1
-            percentsList = percentsList.map { it + precision }
-            geometricMean(percentsList, benchmarksNumber) - precision
+            percentsList = percentsList.map { abs(it) }
+            -geometricMean(percentsList, benchmarksNumber)
         }
     }
 

tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt

@@ -49,7 +49,7 @@ class AnalyzerTests {
         val numbers = listOf(10.1, 10.2, 10.3)
         val value = computeMeanVariance(numbers)
         val expectedMean = 10.2
-        val expectedVariance = 0.12539360253
+        val expectedVariance = 0.07872455
         assertTrue(abs(value.mean - expectedMean) < eps)
         assertTrue(abs(value.variance - expectedVariance) < eps)
     }
@@ -59,7 +59,7 @@ class AnalyzerTests {
         val inputs = createMeanVarianceBenchmarks()
 
         val percent = inputs.first.calcPercentageDiff(inputs.second)
-        val expectedMean = -10.0
+        val expectedMean = -9.99809998
         val expectedVariance = 0.0021
         assertTrue(abs(percent.mean - expectedMean) < eps)
         assertTrue(abs(percent.variance - expectedVariance) < eps)