Added usage of benchmarksAnalyzer for analysis of performance, so removed extra code from benchmark application and gradle. (#2501)

performance/build.gradle

@@ -32,6 +32,8 @@ repositories {
 
 }
 
+defaultTasks 'bench'
+
 private def determinePreset() {
     def preset = MPPTools.defaultHostPreset(project)
     println("$project has been configured for ${preset.name} platform.")
@@ -63,7 +65,6 @@ kotlin {
 
     targets {
         fromPreset(presets.jvm, 'jvm') {
-            def mainOutput = compilations.main.output
             compilations.all {
                 tasks[compileKotlinTaskName].kotlinOptions {
                     jvmTarget = '1.8'
@@ -80,12 +81,13 @@ kotlin {
     }
 }
 
+MPPTools.addTimeListener(project)
+
 MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) {
     workingDir = project.provider {
         kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile
     }
     args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}")
-    outputFileName("${buildDir.absolutePath}/${nativeTextReport}")
 }
 
 task jvmRun(type: JavaExec)  {
@@ -98,19 +100,17 @@ task jvmRun(type: JavaExec)  {
     classpath runtimeClasspath
     main = "MainKt"
     args "$jvmWarmup", "$attempts",  "${buildDir.absolutePath}/${jvmBenchResults}"
-    standardOutput = output
-    doLast {
-        dumpReport("${buildDir.absolutePath}/${jvmTextReport}", output)
-    }
 }
 
 task konanJsonReport {
     doLast {
+        def nativeCompileTime = MPPTools.getNativeCompileTime('Ring')
         String benchContents = new File("${buildDir.absolutePath}/${nativeBenchResults}").text
         def properties = getCommonProperties() + ['type'           : 'native',
                                                   'compilerVersion': "${konanVersion}".toString(),
                                                   'flags'          : kotlin.targets.native.compilations.main.extraOpts.collect{ '"' + it + '"'},
-                                                  'benchmarks'     : benchContents]
+                                                  'benchmarks'     : benchContents,
+                                                  'compileTime'    : nativeCompileTime]
         def output = MPPTools.createJsonReport(properties)
         new File("${buildDir.absolutePath}/${nativeJson}").write(output)
     }
@@ -118,11 +118,12 @@ task konanJsonReport {
 
 task jvmJsonReport {
     doLast {
+        def jvmCompileTime = MPPTools.getJvmCompileTime('Ring')
         String benchContents = new File("${buildDir.absolutePath}/${jvmBenchResults}").text
         def properties = getCommonProperties() + ['type'           : 'jvm',
                                                   'compilerVersion': "${buildKotlinVersion}".toString(),
-                                                  'benchmarks'     : benchContents]
+                                                  'benchmarks'     : benchContents,
-        println(properties['compilerVersion'])
+                                                  'compileTime'    : jvmCompileTime]
         def output = MPPTools.createJsonReport(properties)
         new File("${buildDir.absolutePath}/${jvmJson}").write(output)
     }
@@ -145,58 +146,14 @@ private def getCommonProperties() {
             'kotlinVersion': "${kotlinVersion}".toString()]
 }
 
-task bench(type:DefaultTask) {
+task bench(type:Exec) {
     dependsOn jvmRun
     dependsOn konanRun
-
+    def extension = MPPTools.getNativeProgramExtension()
-    doLast {
+    def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}")
-        def jvmReport = new Report(project.file("${buildDir.absolutePath}/${jvmTextReport}"))
+    if (analyzer != null) {
-        def konanReport = new Report(project.file("${buildDir.absolutePath}/${nativeTextReport}"))
+        commandLine "${analyzer}", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}"
-        def average = "none"
+    } else {
-        def absoluteAverage = "none"
+        println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}")
-        jvmReport.report
-                .sort { konanReport.report[it.key].mean / it.value.mean }
-                .each { k, v ->
-            def konanValue = konanReport.report[k]
-            def ratio = konanValue.mean / v.mean
-            // This is a hack since neither mean nor variance of ratio of two distributions is known.
-            def minRatio = (konanValue.mean - konanValue.stdDev) / (v.mean + v.stdDev)
-            def maxRatio = (konanValue.mean + konanValue.stdDev) / (v.mean - v.stdDev)
-            def ratioConfInt = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio))
-            def formattedKonanValue = String.format('%.4f us +- %.4f us', konanValue.mean / 1000, konanValue.stdDev / 1000)
-            def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioConfInt)
-            if (k == 'RingAverage') {
-                average = formattedRatio
-                absoluteAverage = formattedKonanValue
-            } else {
-                println("$k : absolute = $formattedKonanValue, ratio = $formattedRatio")
-            }
-            if (System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") != null)
-                println("##teamcity[buildStatisticValue key='$k' value='$ratio']")
-        }
-
-        println()
-        println("Average Ring score: absolute = $absoluteAverage, ratio = $average")
     }
-}
+}
-
-class Results {
-    def Double mean
-    def Double stdDev
-
-    Results(Double mean, Double stdDev) {
-        this.mean = mean
-        this.stdDev = stdDev
-    }
-}
-
-class Report {
-    Map<String, Results> report = new TreeMap()
-
-    Report(File path) {
-        path.readLines().drop(3).findAll { it.split(':').length == 3 }.each {
-            def p = it.split(':')
-            report.put(p[0].trim(), new Results(Double.parseDouble(p[1].trim()), Double.parseDouble(p[2].trim())))
-        }
-    }
-}

performance/buildSrc/src/main/kotlin/MPPTools.kt

@@ -8,11 +8,14 @@
 import groovy.lang.Closure
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.tasks.TaskState
+import org.gradle.api.execution.TaskExecutionListener
 import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
 import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
 import org.jetbrains.report.*
 import org.jetbrains.report.json.*
 import java.nio.file.Paths
+import java.io.File
 
 /*
  * This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future.
@@ -60,9 +63,16 @@ fun defaultHostPreset(
         throw Exception("Host OS '$hostOs' is not supported in Kotlin/Native ${subproject.displayName}.")
 }
 
+fun getNativeProgramExtension(): String = when {
+    isMacos -> ".kexe"
+    isLinux -> ".kexe"
+    isWindows -> ".exe"
+    else -> error("Unknown host")
+}
+
 // Create benchmarks json report based on information get from gradle project
 fun createJsonReport(projectProperties: Map<String, Any>): String {
-    fun getValue(key: String): String = projectProperties[key] as? String ?: "uknown"
+    fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown"
     val machine = Environment.Machine(getValue("cpu"), getValue("os"))
     val jdk = Environment.JDKInstance(getValue("jdkVersion"), getValue("jdkVendor"))
     val env = Environment(machine, jdk)
@@ -73,10 +83,15 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
     val benchDesc = getValue("benchmarks")
     val benchmarksArray = JsonTreeParser.parse(benchDesc)
     val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray)
+            .union(listOf<BenchmarkResult>(projectProperties["compileTime"] as BenchmarkResult)).toList()
     val report = BenchmarksReport(env, benchmarks, kotlin)
     return report.toJson()
 }
 
+// Find file with set name in directory.
+fun findFile(fileName: String, directory: String): String? =
+    File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()
+
 // A short-cut to add a Kotlin/Native run task.
 @JvmOverloads
 fun createRunTask(
@@ -89,3 +104,40 @@ fun createRunTask(
     task.configure(configureClosure ?: task.emptyConfigureClosure())
     return task
 }
+
+fun getJvmCompileTime(programName: String): BenchmarkResult =
+        TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
+
+fun getNativeCompileTime(programName: String): BenchmarkResult =
+        TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkReleaseExecutableNative"))
+
+// Class time tracker for all tasks.
+class TaskTimerListener: TaskExecutionListener {
+    companion object {
+        val tasksTimes = mutableMapOf<String, Double>()
+
+        fun getBenchmarkResult(programName: String, tasksNames: List<String>): BenchmarkResult {
+            val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum()
+            // TODO get this info from gradle plugin with exit code end stacktrace.
+            val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b }
+            return BenchmarkResult("$programName.compileTime",
+                                    if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
+                                    time, time, 1, 0)
+        }
+
+    }
+
+    private var startTime = System.currentTimeMillis()
+
+    override fun beforeExecute(task: Task) {
+        startTime = System.nanoTime()
+    }
+
+     override fun afterExecute(task: Task, taskState: TaskState) {
+         tasksTimes[task.name] = (System.nanoTime() - startTime) / 1000.0
+     }
+}
+
+fun addTimeListener(subproject: Project) {
+    subproject.gradle.addListener(TaskTimerListener())
+}

performance/gradle.properties

@@ -8,4 +8,5 @@ nativeTextReport = nativeReport.txt
 jvmTextReport = jvmReport.txt
 nativeJson = nativeReport.json
 jvmJson = jvmReport.json
-
+analyzerTool = benchmarksAnalyzer
+analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin

performance/src/main/kotlin/org/jetbrains/ring/launcher.kt

@@ -27,7 +27,6 @@ val BENCHMARK_SIZE = 100
 class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
     class Results(val mean: Double, val variance: Double)
 
-    val results = mutableMapOf<String, Results>()
     val benchmarkResults = mutableListOf<BenchmarkResult>()
 
     fun launch(benchmark: () -> Any?, name: String) {                          // If benchmark runs too long - use coeff to speed it up.
@@ -66,10 +65,6 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
                                 scaledTime / 1000, scaledTime / 1000,
                                 k + 1, numWarmIterations))
         }
-        val mean = samples.sum() / numberOfAttempts
-        val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / numberOfAttempts
-
-        results[name] = Results(mean, variance)
     }
 
     //-------------------------------------------------------------------------//
@@ -101,44 +96,11 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
         runWithIndiciesBenchmark()
         runOctoTest()
 
-        printResultsNormalized()
         return benchmarkResults
     }
 
     //-------------------------------------------------------------------------//
 
-    fun printResultsAsTime() {
-        results.forEach {
-            val niceName = "\"${it.key}\"".padEnd(51)
-            val niceTime = "${it.value}".padStart(10)
-            println("    $niceName to ${niceTime}L,")
-        }
-    }
-
-    //-------------------------------------------------------------------------//
-
-    private val zStar = 1.96 // For 95% confidence interval.
-
-    fun printResultsNormalized() {
-        var totalMean = 0.0
-        var totalVariance = 0.0
-        results.asSequence().sortedBy { it.key }.forEach {
-            val niceName  = it.key.padEnd(50, ' ')
-            val mean = it.value.mean
-            val variance = it.value.variance
-            val confidenceInterval = sqrt(variance / numberOfAttempts) * zStar
-            println("$niceName : ${mean.toString(9)} : ${confidenceInterval.toString(9)}")
-
-            totalMean += mean
-            totalVariance += variance
-        }
-        val averageMean = totalMean / results.size
-        val averageConfidenceInterval = sqrt(totalVariance / numberOfAttempts) * zStar / results.size
-        println("\nRingAverage: ${averageMean.toString(9)} : ${averageConfidenceInterval.toString(9)}")
-    }
-
-    //-------------------------------------------------------------------------//
-
     fun runAbstractMethodBenchmark() {
         val benchmark = AbstractMethodBenchmark()
 
@@ -493,17 +455,4 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
     fun runOctoTest() {
         launch(::octoTest, "OctoTest")
     }
-
-    //-------------------------------------------------------------------------//
-
-    fun Double.toString(n: Int): String {
-        val str = this.toString()
-        if (str.contains('e', ignoreCase = true)) return str
-
-        val len      = str.length
-        val pointIdx = str.indexOf('.')
-        val dropCnt  = len - pointIdx - n - 1
-        if (dropCnt < 1) return str
-        return str.dropLast(dropCnt)
-    }
 }

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt

@@ -122,4 +122,9 @@ fun collectMeanResults(benchmarks: Map<String, List<BenchmarkResult>>): Benchmar
                 currentWarmup)
         name to MeanVarianceBenchmark(meanBenchmark, varianceBenchmark)
     }.toMap()
-}
+}
+
+fun collectBenchmarksDurations(benchmarks: Map<String, List<BenchmarkResult>>): Map<String, Double> =
+        benchmarks.map { (name, resultsSet) ->
+            name to resultsSet.sumByDouble { it.runtimeInUs }
+        }.toMap()

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -34,6 +34,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
                                val meaningfulChangesValue: Double = 0.5) {
     // Report created by joining comparing reports.
     val mergedReport: Map<String, SummaryBenchmark>
+    val benchmarksDurations: Map<String, Pair<Double?, Double?>>
 
     // Lists of benchmarks in different status.
     private val benchmarksWithChangedStatus = mutableListOf<FieldChange<BenchmarkResult.Status>>()
@@ -70,6 +71,9 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
     val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
         get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! }
 
+    val currentBenchmarksDuration: Map<String, Double>
+        get() = benchmarksDurations.filter{ it.value.first != null }.map { it.key to it.value.first!! }.toMap()
+
     init {
         // Count avarage values for each benchmark.
         val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks)
@@ -77,6 +81,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
             collectMeanResults(previousReport.benchmarks)
         }
         mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable)
+        benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
         geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable)
         if (previousReport != null) {
             // Check changes in environment and tools.
@@ -101,6 +106,17 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
         return MeanVarianceBenchmark(meanBenchmark, varianceBenchmark)
     }
 
+    // Generate map with summary durations of each benchmark.
+    private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?):
+            Map<String, Pair<Double?, Double?>> {
+        val currentDurations = collectBenchmarksDurations(currentReport.benchmarks)
+        val previousDurations = previousReport?.let {
+            collectBenchmarksDurations(previousReport.benchmarks)
+        } ?: mapOf<String, Double>()
+        return currentDurations.keys.union(previousDurations.keys)
+                .map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap()
+    }
+
     // Merge current and compare to report.
     private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?):
             Map<String, SummaryBenchmark> {

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt

@@ -25,20 +25,40 @@ class TeamCityStatisticsRender: Render {
     private var content = StringBuilder()
 
     override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
+        val currentDurations = report.currentBenchmarksDuration
+
+        content.append("##teamcity[testSuiteStarted name='Benchmarks']\n")
+
         // For current benchmarks print score as TeamCity Test Metadata
         report.currentMeanVarianceBenchmarks.forEach { benchmark ->
-            content.append(renderBenchmark(benchmark.meanBenchmark, "Mean"))
+            renderBenchmark(benchmark.meanBenchmark, currentDurations[benchmark.meanBenchmark.name]!!)
-            content.append(renderBenchmark(benchmark.varianceBenchmark, "Variance"))
+            renderSummaryBecnhmarkValue(benchmark.meanBenchmark, "Mean")
+            renderSummaryBecnhmarkValue(benchmark.varianceBenchmark, "Variance")
         }
+        content.append("##teamcity[testSuiteFinished name='Benchmarks']\n")
+
         // Report geometric mean as build statistic value
-        content.append(renderGeometricMean(report.geoMeanBenchmark.first!!))
+        renderGeometricMean(report.geoMeanBenchmark.first!!)
+
         return content.toString()
     }
 
-    private fun renderBenchmark(benchmark: BenchmarkResult, metric: String): String =
+    private fun renderSummaryBecnhmarkValue(benchmark: BenchmarkResult, metric: String) =
-        "##teamcity[testMetadata testName='${benchmark.name}' name='$metric' type='number' value='${benchmark.score}']\n"
+            content.append("##teamcity[testMetadata testName='${benchmark.name}' name='$metric'" +
+                    " type='number' value='${benchmark.score}']\n")
 
-    private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark): String =
+    // Produce benchmark as test in TeamCity
-        "##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n" +
+    private fun renderBenchmark(benchmark: BenchmarkResult , duration: Double) {
-                "##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n"
+        content.append("##teamcity[testStarted name='${benchmark.name}']\n")
+        if (benchmark.status == BenchmarkResult.Status.FAILED) {
+            content.append("##teamcity[testFailed name='${benchmark.name}']\n")
+        }
+        // test_duration_in_milliseconds is set for TeamCity
+        content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n")
+    }
+
+    private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) {
+        content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n")
+        content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n")
+    }
 }