Some fixes for <perf>

- Extracted number of attempts to a parameter
- Show 95% confidence intervals instead of standard deviation

performance/build.gradle

@@ -26,6 +26,7 @@ repositories {
 //TODO: property
 def jvmWarmup = 10000
 def nativeWarmup = 10
+def attempts = 10
 
 ext."konan.home" = distDir
 
@@ -64,7 +65,7 @@ task jvmRun(type: JavaExec) {
     def output = new ByteArrayOutputStream()
     classpath sourceSets.main.runtimeClasspath
     main = "MainKt"
-    args "$jvmWarmup"
+    args "$jvmWarmup", "$attempts"
     standardOutput = output
     doLast {
         dumpReport('jvmReport', output)
@@ -80,7 +81,7 @@ private void dumpReport(String name, ByteArrayOutputStream output) {
 task konanRun(type: Exec) {
     dependsOn 'build'
     def output = new ByteArrayOutputStream()
-    commandLine konanArtifacts.Ring.getByTarget('host').artifact.absolutePath, "$nativeWarmup"
+    commandLine konanArtifacts.Ring.getByTarget('host').artifact.absolutePath, "$nativeWarmup", "$attempts"
     standardOutput = output
     doLast {
         dumpReport('konanReport', output)
@@ -105,11 +106,12 @@ task bench(type:DefaultTask) {
                 .each { k, v ->
             def konanValue = konanReport.report[k]
             def ratio = konanValue.mean / v.mean
+            // This is a hack since neither mean nor variance of ratio of two distributions is known.
             def minRatio = (konanValue.mean - konanValue.stdDev) / (v.mean + v.stdDev)
             def maxRatio = (konanValue.mean + konanValue.stdDev) / (v.mean - v.stdDev)
-            def ratioStdDev = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio))
+            def ratioConfInt = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio))
             def formattedKonanValue = String.format('%.4f us +- %.4f us', konanValue.mean / 1000, konanValue.stdDev / 1000)
-            def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioStdDev)
+            def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioConfInt)
             if (k == 'RingAverage') {
                 average = formattedRatio
                 absoluteAverage = formattedKonanValue

performance/src/main/kotlin/main.kt

@@ -17,12 +17,23 @@
 import org.jetbrains.ring.Launcher
 
 fun main(args: Array<String>) {
-    var numWarmIterations    =  0       // Should be 100000 for jdk based run
+    var numWarmIterations = 0       // Should be 100000 for jdk based run
+    var numberOfAttempts = 10
 
-    if (args.size == 1)
-        numWarmIterations    = args[0].toInt()
+    when (args.size) {
+        0 -> { }
+        1 -> numWarmIterations = args[0].toInt()
+        2 -> {
+            numWarmIterations = args[0].toInt()
+            numberOfAttempts = args[1].toInt()
+        }
+        else -> {
+            println("Usage: perf [# warmup iterations] [# attempts]")
+            return
+        }
+    }
 
     println("Ring starting")
     println("  warmup  iterations count: $numWarmIterations")
-    Launcher(numWarmIterations).runBenchmarks()
+    Launcher(numWarmIterations, numberOfAttempts).runBenchmarks()
 }

performance/src/main/kotlin/org/jetbrains/ring/launcher.kt

@@ -17,13 +17,14 @@
 package org.jetbrains.ring
 
 import octoTest
+import kotlin.math.sqrt
 import kotlin.system.measureNanoTime
 
 val BENCHMARK_SIZE = 100
 
 //-----------------------------------------------------------------------------//
 
-class Launcher(val numWarmIterations: Int) {
+class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
     class Results(val mean: Double, val variance: Double)
 
     val results = mutableMapOf<String, Results>()
@@ -48,8 +49,7 @@ class Launcher(val numWarmIterations: Int) {
             autoEvaluatedNumberOfMeasureIteration *= 2
         }
 
-        val attempts = 10
-        val samples = DoubleArray(attempts)
+        val samples = DoubleArray(numberOfAttempts)
         for (k in samples.indices) {
             i = autoEvaluatedNumberOfMeasureIteration
             val time = measureNanoTime {
@@ -60,8 +60,8 @@ class Launcher(val numWarmIterations: Int) {
             }
             samples[k] = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
         }
-        val mean = samples.sum() / attempts
-        val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / attempts
+        val mean = samples.sum() / numberOfAttempts
+        val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / numberOfAttempts
 
         return Results(mean, variance)
     }
@@ -110,19 +110,24 @@ class Launcher(val numWarmIterations: Int) {
 
     //-------------------------------------------------------------------------//
 
+    private val zStar = 1.96 // For 95% confidence interval.
+
     fun printResultsNormalized() {
         var totalMean = 0.0
         var totalVariance = 0.0
         results.asSequence().sortedBy { it.key }.forEach {
             val niceName  = it.key.padEnd(50, ' ')
-            println("$niceName : ${it.value.mean.toString(9)} : ${kotlin.math.sqrt(it.value.variance).toString(9)}")
+            val mean = it.value.mean
+            val variance = it.value.variance
+            val confidenceInterval = sqrt(variance / numberOfAttempts) * zStar
+            println("$niceName : ${mean.toString(9)} : ${confidenceInterval.toString(9)}")
 
-            totalMean += it.value.mean
-            totalVariance += it.value.variance
+            totalMean += mean
+            totalVariance += variance
         }
         val averageMean = totalMean / results.size
-        val averageStdDev = kotlin.math.sqrt(totalVariance) / results.size
-        println("\nRingAverage: ${averageMean.toString(9)} : ${averageStdDev.toString(9)}")
+        val averageConfidenceInterval = sqrt(totalVariance / numberOfAttempts) * zStar / results.size
+        println("\nRingAverage: ${averageMean.toString(9)} : ${averageConfidenceInterval.toString(9)}")
     }
 
     //-------------------------------------------------------------------------//