Some fixes in <perf> to make it stable

performance/build.gradle

@@ -19,8 +19,8 @@ repositories {
 }
 
 //TODO: property
-def ringWarmup=1000
-def iterations=2000
+def jvmWarmup = 10000
+def nativeWarmup = 10
 
 apply plugin: 'kotlin'
 apply plugin: 'application'
@@ -31,10 +31,17 @@ checkKonanCompiler.dependsOn ':dist'
 
 konanArtifacts {
     program('Ring') {
+        srcDir 'src/main/kotlin'
+        srcDir 'src/main/kotlin-native'
         enableOptimizations true
     }
 }
 
+sourceSets {
+    main.kotlin.srcDir 'src/main/kotlin'
+    main.kotlin.srcDir 'src/main/kotlin-jvm'
+}
+
 compileKotlin {
     kotlinOptions.suppressWarnings = true
     kotlinOptions {
@@ -51,7 +58,7 @@ task jvmRun(type: JavaExec) {
     def output = new ByteArrayOutputStream()
     classpath sourceSets.main.runtimeClasspath
     main = "MainKt"
-    args "$ringWarmup", "$iterations"
+    args "$jvmWarmup"
     standardOutput = output
     doLast {
         dumpReport('jvmReport', output)
@@ -67,7 +74,7 @@ private void dumpReport(String name, ByteArrayOutputStream output) {
 task konanRun(type: Exec) {
     dependsOn 'build'
     def output = new ByteArrayOutputStream()
-    commandLine konanArtifacts.Ring.getByTarget('host').artifact.absolutePath, "$ringWarmup", "$iterations"
+    commandLine konanArtifacts.Ring.getByTarget('host').artifact.absolutePath, "$nativeWarmup"
     standardOutput = output
     doLast {
         dumpReport('konanReport', output)
@@ -86,24 +93,28 @@ task bench(type:DefaultTask) {
         def jvmReport = new Report(project.file("build/jvmReport.txt"))
         def konanReport = new Report(project.file("build/konanReport.txt"))
         def average = "none"
+        def absoluteAverage = "none"
         jvmReport.report.each { k, v ->
-            def ratio = String.format('%.2f', konanReport.report[k]/v * 100)
+            def konanValue = konanReport.report[k]
+            def ratio = String.format('%.2f', konanValue/v)
+            def formattedKonanValue = String.format('%.2f', konanValue/1000)
             if (k == 'RingAverage') {
                 average = ratio
+                absoluteAverage = formattedKonanValue
             }
-            println("$k : $ratio %")
+            println("$k : absolute = $formattedKonanValue us, ratio = $ratio")
             if (System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") != null)
                 println("##teamcity[buildStatisticValue key='$k' value='$ratio']")
         }
 
         println()
-        println("Average Ring score: $average %")
+        println("Average Ring score: absolute = $absoluteAverage us, ratio = $average")
     }
 }
 
 
 class Report {
-    def Map<String, Double> report = new HashMap()
+    def Map<String, Double> report = new TreeMap()
 
     Report(File path) {
         path.readLines().drop(3).findAll { it.split(':').length == 2 }.each {

performance/src/main/kotlin-jvm/cleanup.kt

@@ -0,0 +1,3 @@
+package org.jetbrains.ring
+
+fun cleanup() { }

performance/src/main/kotlin-native/cleanup.kt

@@ -0,0 +1,5 @@
+package org.jetbrains.ring
+
+import konan.internal.GC
+
+fun cleanup() { GC.collect() }

performance/src/main/kotlin/main.kt

@@ -18,15 +18,11 @@ import org.jetbrains.ring.Launcher
 
 fun main(args: Array<String>) {
     var numWarmIterations    =  0       // Should be 100000 for jdk based run
-    var numMeasureIterations =  2000
 
-    if (args.size == 2) {
+    if (args.size == 1)
         numWarmIterations    = args[0].toInt()
-        numMeasureIterations = args[1].toInt()
-    }
 
     println("Ring starting")
     println("  warmup  iterations count: $numWarmIterations")
-    println("  measure iterations count: $numMeasureIterations")
-    Launcher(numWarmIterations, numMeasureIterations).runBenchmarks()
+    Launcher(numWarmIterations).runBenchmarks()
 }

performance/src/main/kotlin/org/jetbrains/ring/launcher.kt

@@ -23,21 +23,38 @@ val BENCHMARK_SIZE = 100
 
 //-----------------------------------------------------------------------------//
 
-class Launcher(val numWarmIterations: Int, val numMeasureIterations: Int) {
+class Launcher(val numWarmIterations: Int) {
     val results = mutableMapOf<String, Long>()
 
-    fun launch(benchmark: () -> Any?, coeff: Double = 1.0): Long {                          // If benchmark runs too long - use coeff to speed it up.
-        var i = (numWarmIterations    * coeff).toInt()
-        var j = (numMeasureIterations * coeff).toInt()
+    fun launch(benchmark: () -> Any?): Long {                          // If benchmark runs too long - use coeff to speed it up.
+        var i = numWarmIterations
 
         while (i-- > 0) benchmark()
-        val time = measureNanoTime {
-            while (j-- > 0) {
-                benchmark()
+        cleanup()
+
+        var autoEvaluatedNumberOfMeasureIteration = 1
+        while (true) {
+            var j = autoEvaluatedNumberOfMeasureIteration
+            val time = measureNanoTime {
+                while (j-- > 0) {
+                    benchmark()
+                }
+                cleanup()
             }
+            if (time >= 100L * 1_000_000) // 100ms
+                break
+            autoEvaluatedNumberOfMeasureIteration *= 2
         }
 
-        return (time / numMeasureIterations / coeff).toLong()
+        i = autoEvaluatedNumberOfMeasureIteration
+        val time = measureNanoTime {
+            while (i-- > 0) {
+                benchmark()
+            }
+            cleanup()
+        }
+
+        return time / autoEvaluatedNumberOfMeasureIteration
     }
 
     //-------------------------------------------------------------------------//
@@ -85,7 +102,7 @@ class Launcher(val numWarmIterations: Int, val numMeasureIterations: Int) {
 
     fun printResultsNormalized() {
         var total = 0.0
-        results.forEach {
+        results.asSequence().sortedBy { it.key }.forEach {
             val normaTime = NormaResults[it.key]
             if (normaTime == null) println("ERROR: no norma for benchmark ${it.key}")
 
@@ -436,7 +453,7 @@ class Launcher(val numWarmIterations: Int, val numMeasureIterations: Int) {
     //-------------------------------------------------------------------------//
 
     fun runOctoTest() {
-        results["OctoTest"] = launch(::octoTest, 0.1)
+        results["OctoTest"] = launch(::octoTest)
     }
 
     //-------------------------------------------------------------------------//