Change constants for getting more stable benchmarks. (#2631)

performance/gradle.properties

@@ -1,8 +1,8 @@
 org.jetbrains.kotlin.native.home=../dist
 org.jetbrains.kotlin.native.jvmArgs=-Xmx6G
 jvmWarmup = 10000
-nativeWarmup = 10
-attempts = 10
+nativeWarmup = 20
+attempts = 60
 jvmBenchResults = jvmBenchResults.json
 nativeBenchResults = nativeBenchResults.json
 nativeTextReport = nativeReport.txt
@@ -14,4 +14,4 @@ analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin
 outputReport = ../report/report.html
 bintrayUrl = https://api.bintray.com/content/lepilkinaelena
 bintrayRepo = KotlinNativePerformance
-bintrayPackage = jsonReports
+bintrayPackage = jsonReports

performance/src/main/kotlin/org/jetbrains/ring/launcher.kt

@@ -20,7 +20,7 @@ import octoTest
 import kotlin.math.sqrt
 import org.jetbrains.report.BenchmarkResult
 
-val BENCHMARK_SIZE = 100
+const val BENCHMARK_SIZE = 10000
 
 //-----------------------------------------------------------------------------//
 
@@ -455,4 +455,4 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
     fun runOctoTest() {
         launch(::octoTest, "OctoTest")
     }
-}
+}

tools/benchmarksAnalyzer/src/main/kotlin/main.kt

@@ -81,7 +81,7 @@ fun main(args: Array<String>) {
 
     val options = listOf(
             OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
-            OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "0.5"),
+            OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "1.0"),
             OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"),
             OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")),
                     "renders", "r", "Renders for showing information", "text", isMultiple = true),

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt

@@ -71,10 +71,11 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc
 
 }
 
-fun geometricMean(values: List<Double>) = values.map { it.pow(1.0 / values.size) }.reduce { a, b -> a * b }
+fun geometricMean(values: List<Double>, totalNumber: Int? = null) = 
+    values.map { it.pow(1.0 / (totalNumber ?: values.size)) }.reduce { a, b -> a * b }
 
 fun computeMeanVariance(samples: List<Double>): MeanVariance {
-    val zStar = 1.96    // Critical point for 90% confidence of normal distribution.
+    val zStar = 1.67    // Critical point for 90% confidence of normal distribution.
     val mean = samples.sum() / samples.size
     val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size
     val confidenceInterval = sqrt(variance / samples.size) * zStar

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -142,12 +142,12 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
             return 0.0
         var percentsList = bucket.values.map { it.first.mean }
         return if (percentsList.first() > 0.0) {
-            geometricMean(percentsList)
+            geometricMean(percentsList, benchmarksNumber)
         } else {
             // Geometric mean can be counted on positive numbers.
             val precision = abs(getMaximumChange(bucket)) + 1
             percentsList = percentsList.map { it + precision }
-            geometricMean(percentsList) - precision
+            geometricMean(percentsList, benchmarksNumber) - precision
         }
     }
 

tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt

@@ -49,7 +49,7 @@ class AnalyzerTests {
         val numbers = listOf(10.1, 10.2, 10.3)
         val value = computeMeanVariance(numbers)
         val expectedMean = 10.2
-        val expectedVariance = 0.092395
+        val expectedVariance = 0.12539360253
         assertTrue(abs(value.mean - expectedMean) < eps)
         assertTrue(abs(value.variance - expectedVariance) < eps)
     }
@@ -76,4 +76,4 @@ class AnalyzerTests {
         assertTrue(abs(ratio.mean - expectedMean) < eps)
         assertTrue(abs(ratio.variance - expectedVariance) < eps)
     }
-}
+}