From 54fa7468138eb6b76ac8686e6bc3718f045645bd Mon Sep 17 00:00:00 2001
From: Igor Chevdar <igor.chevdar@jetbrains.com>
Date: Tue, 15 May 2018 18:53:14 +0300
Subject: [PATCH] Perf: implemented many attempts computation Mean and standard
 deviation are computed as result

---
 performance/build.gradle                      | 32 ++++++++++----
 .../kotlin/org/jetbrains/ring/launcher.kt     | 42 ++++++++++++-------
 2 files changed, 49 insertions(+), 25 deletions(-)

diff --git a/performance/build.gradle b/performance/build.gradle
index 9b7a76ccd95..ea78c31e9ae 100644
--- a/performance/build.gradle
+++ b/performance/build.gradle
@@ -96,33 +96,47 @@ task bench(type:DefaultTask) {
         def average = "none"
         def absoluteAverage = "none"
         jvmReport.report
-                .sort { konanReport.report[it.key] / it.value }
+                .sort { konanReport.report[it.key].mean / it.value.mean }
                 .each { k, v ->
             def konanValue = konanReport.report[k]
-            def ratio = String.format('%.2f', konanValue / v)
-            def formattedKonanValue = String.format('%.2f', konanValue / 1000)
+            def ratio = konanValue.mean / v.mean
+            def minRatio = (konanValue.mean - konanValue.stdDev) / (v.mean + v.stdDev)
+            def maxRatio = (konanValue.mean + konanValue.stdDev) / (v.mean - v.stdDev)
+            def ratioStdDev = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio))
+            def formattedKonanValue = String.format('%.4f us +- %.4f us', konanValue.mean / 1000, konanValue.stdDev / 1000)
+            def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioStdDev)
             if (k == 'RingAverage') {
-                average = ratio
+                average = formattedRatio
                 absoluteAverage = formattedKonanValue
+            } else {
+                println("$k : absolute = $formattedKonanValue, ratio = $formattedRatio")
             }
-            println("$k : absolute = $formattedKonanValue us, ratio = $ratio")
             if (System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") != null)
                 println("##teamcity[buildStatisticValue key='$k' value='$ratio']")
         }
 
         println()
-        println("Average Ring score: absolute = $absoluteAverage us, ratio = $average")
+        println("Average Ring score: absolute = $absoluteAverage, ratio = $average")
     }
 }
 
+class Results {
+    def Double mean
+    def Double stdDev
+
+    Results(Double mean, Double stdDev) {
+        this.mean = mean
+        this.stdDev = stdDev
+    }
+}
 
 class Report {
-    def Map<String, Double> report = new TreeMap()
+    def Map<String, Results> report = new TreeMap()
 
     Report(File path) {
-        path.readLines().drop(3).findAll { it.split(':').length == 2 }.each {
+        path.readLines().drop(3).findAll { it.split(':').length == 3 }.each {
             def p = it.split(':')
-            report.put(p[0].trim(), Double.parseDouble(p[1].trim()))
+            report.put(p[0].trim(), new Results(Double.parseDouble(p[1].trim()), Double.parseDouble(p[2].trim())))
         }
     }
 }
diff --git a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt
index adef1fbaaba..1f52c759ce4 100644
--- a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt
+++ b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt
@@ -24,9 +24,11 @@ val BENCHMARK_SIZE = 100
 //-----------------------------------------------------------------------------//
 
 class Launcher(val numWarmIterations: Int) {
-    val results = mutableMapOf<String, Long>()
+    class Results(val mean: Double, val variance: Double)
 
-    fun launch(benchmark: () -> Any?): Long {                          // If benchmark runs too long - use coeff to speed it up.
+    val results = mutableMapOf<String, Results>()
+
+    fun launch(benchmark: () -> Any?): Results {                          // If benchmark runs too long - use coeff to speed it up.
         var i = numWarmIterations
 
         while (i-- > 0) benchmark()
@@ -41,20 +43,27 @@ class Launcher(val numWarmIterations: Int) {
                 }
                 cleanup()
             }
-            if (time >= 200L * 1_000_000) // 200ms
+            if (time >= 100L * 1_000_000) // 100ms
                 break
             autoEvaluatedNumberOfMeasureIteration *= 2
         }
 
-        i = autoEvaluatedNumberOfMeasureIteration
-        val time = measureNanoTime {
-            while (i-- > 0) {
-                benchmark()
+        val attempts = 10
+        val samples = DoubleArray(attempts)
+        for (k in samples.indices) {
+            i = autoEvaluatedNumberOfMeasureIteration
+            val time = measureNanoTime {
+                while (i-- > 0) {
+                    benchmark()
+                }
+                cleanup()
             }
-            cleanup()
+            samples[k] = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
         }
+        val mean = samples.sum() / attempts
+        val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / attempts
 
-        return time / autoEvaluatedNumberOfMeasureIteration
+        return Results(mean, variance)
     }
 
     //-------------------------------------------------------------------------//
@@ -101,17 +110,18 @@ class Launcher(val numWarmIterations: Int) {
     //-------------------------------------------------------------------------//
 
     fun printResultsNormalized() {
-        var total = 0.0
+        var totalMean = 0.0
+        var totalVariance = 0.0
         results.asSequence().sortedBy { it.key }.forEach {
-            val norma     = it.value.toDouble()
             val niceName  = it.key.padEnd(50, ' ')
-            val niceNorma = norma.toString(9)
-            println("$niceName : $niceNorma")
+            println("$niceName : ${it.value.mean.toString(9)} : ${kotlin.math.sqrt(it.value.variance).toString(9)}")
 
-            total += norma
+            totalMean += it.value.mean
+            totalVariance += it.value.variance
         }
-        val average = total / results.size
-        println("\nRingAverage: ${average.toString(9)}")
+        val averageMean = totalMean / results.size
+        val averageStdDev = kotlin.math.sqrt(totalVariance) / results.size
+        println("\nRingAverage: ${averageMean.toString(9)} : ${averageStdDev.toString(9)}")
     }
 
     //-------------------------------------------------------------------------//