From 88b2955bb34a766c99171b0f965c8fb03c66baf4 Mon Sep 17 00:00:00 2001
From: Elena Lepilkina <Elena.Lepilkina@jetbrains.com>
Date: Thu, 17 Sep 2020 10:17:16 +0300
Subject: [PATCH] Changed calculation autoEvaluatedNumberOfMeasureIteration to
 reduce duration

---
 performance/gradle.properties                     |  4 ++--
 .../org/jetbrains/benchmarksLauncher/launcher.kt  | 11 +++++------
 .../shared/src/main/kotlin/analyzer/Statistics.kt | 15 +++++++++++++--
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/performance/gradle.properties b/performance/gradle.properties
index ce27a88fbd4..6eec7e72c67 100644
--- a/performance/gradle.properties
+++ b/performance/gradle.properties
@@ -1,8 +1,8 @@
 kotlin.native.home=../dist
 org.jetbrains.kotlin.native.jvmArgs=-Xmx6G
 jvmWarmup = 1000
-nativeWarmup = 20
-attempts = 40
+nativeWarmup = 10
+attempts = 30
 jvmBenchResults = jvmBenchResults.json
 nativeBenchResults = nativeBenchResults.json
 nativeTextReport = nativeReport.txt
diff --git a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt
index 6d6650da7ab..f1f678bf329 100644
--- a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt
+++ b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt
@@ -76,13 +76,12 @@ abstract class Launcher {
         val benchmarkInstance = (benchmark as? BenchmarkEntryWithInit)?.ctor?.invoke()
         logger.log("Warm up iterations for benchmark $name\n")
         runBenchmark(benchmarkInstance, benchmark, numWarmIterations)
+        val expectedDuration = 2000L * 1_000_000 // 2s
         var autoEvaluatedNumberOfMeasureIteration = 1
-        while (benchmark.useAutoEvaluatedNumberOfMeasure) {
-            var j = autoEvaluatedNumberOfMeasureIteration
-            val time = runBenchmark(benchmarkInstance, benchmark, j)
-            if (time >= 2000L * 1_000_000) // 2s
-                break
-            autoEvaluatedNumberOfMeasureIteration *= 2
+        if (benchmark.useAutoEvaluatedNumberOfMeasure) {
+            val time = runBenchmark(benchmarkInstance, benchmark, 1)
+            if (time < expectedDuration)
+                autoEvaluatedNumberOfMeasureIteration = (expectedDuration / time).toInt() / 4 * 4
         }
         logger.log("Running benchmark $name ")
         for (k in 0.until(numberOfAttempts)) {
diff --git a/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt b/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt
index adb1d226f6e..fcde19c32d3 100644
--- a/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt
+++ b/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt
@@ -71,9 +71,20 @@ fun geometricMean(values: Collection<Double>, totalNumber: Int = values.size) =
         }
 
 fun computeMeanVariance(samples: List<Double>): MeanVariance {
+    val removedBroadSamples = 0.2
     val zStar = 1.67    // Critical point for 90% confidence of normal distribution.
-    val mean = samples.sum() / samples.size
-    val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size
+    // Skip several minimal and maximum values.
+    val filteredSamples = if (samples.size >= 1/removedBroadSamples) {
+         samples.sorted().subList((samples.size * removedBroadSamples).toInt(),
+                samples.size - (samples.size * removedBroadSamples).toInt())
+    } else {
+        samples
+    }
+
+    val mean = filteredSamples.sum() / filteredSamples.size
+    val variance = samples.indices.sumByDouble {
+        (samples[it] - mean) * (samples[it] - mean)
+    } / samples.size
     val confidenceInterval = sqrt(variance / samples.size) * zStar
     return MeanVariance(mean, confidenceInterval)
 }