External benchmarks (#3335)

build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt

@@ -12,9 +12,7 @@ import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.tasks.TaskState
 import org.gradle.api.execution.TaskExecutionListener
-import org.gradle.api.tasks.AbstractExecTask
 import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
-import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
 import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink
 import org.jetbrains.report.*
 import org.jetbrains.report.json.*
@@ -77,11 +75,21 @@ fun getFileSize(filePath: String): Long? {
 
 fun getCodeSizeBenchmark(programName: String, filePath: String): BenchmarkResult {
     val codeSize = getFileSize(filePath)
-    return BenchmarkResult("$programName",
+    return BenchmarkResult(programName,
             codeSize?. let { BenchmarkResult.Status.PASSED } ?: run { BenchmarkResult.Status.FAILED },
             codeSize?.toDouble() ?: 0.0, BenchmarkResult.Metric.CODE_SIZE, codeSize?.toDouble() ?: 0.0, 1, 0)
 }
 
+fun toCodeSizeBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult {
+    if (!metricDescription.startsWith("CODE_SIZE")) {
+        error("Wrong metric is used as code size.")
+    }
+    val codeSize = metricDescription.split(' ')[1].toDouble()
+    return BenchmarkResult(programName,
+            if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
+            codeSize, BenchmarkResult.Metric.CODE_SIZE, codeSize, 1, 0)
+}
+
 // Create benchmarks json report based on information get from gradle project
 fun createJsonReport(projectProperties: Map<String, Any>): String {
     fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown"
@@ -96,17 +104,16 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
     val benchmarksArray = JsonTreeParser.parse(benchDesc)
     val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray)
             .union(projectProperties["compileTime"] as List<BenchmarkResult>).union(
-                    listOf(projectProperties["codeSize"] as BenchmarkResult)).toList()
+                    listOf(projectProperties["codeSize"] as? BenchmarkResult).filterNotNull()).toList()
     val report = BenchmarksReport(env, benchmarks, kotlin)
     return report.toJson()
 }
 
 fun mergeReports(reports: List<File>): String {
-    val reportsToMerge = reports.map {
+    val reportsToMerge = reports.filter { it.exists() }.map {
         val json = it.inputStream().bufferedReader().use { it.readText() }
         val reportElement = JsonTreeParser.parse(json)
         BenchmarksReport.create(reportElement)
-
     }
     return if (reportsToMerge.isEmpty()) "" else reportsToMerge.reduce { result, it -> result + it }.toJson()
 }
@@ -186,9 +193,18 @@ fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable<String>, r
             status = if (exitCodes["$it$number"] != 0) BenchmarkResult.Status.FAILED else status
         }
 
-        BenchmarkResult("$programName", status, time, BenchmarkResult.Metric.COMPILE_TIME, time, number, 0)
+        BenchmarkResult(programName, status, time, BenchmarkResult.Metric.COMPILE_TIME, time, number, 0)
     }.toList()
 
+fun toCompileBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult {
+    if (!metricDescription.startsWith("COMPILE_TIME")) {
+        error("Wrong metric is used as compile time.")
+    }
+    val time = metricDescription.split(' ')[1].toDouble()
+    return BenchmarkResult(programName,
+            if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
+            time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0)
+}
 
 // Class time tracker for all tasks.
 class TaskTimerListener: TaskExecutionListener {
@@ -199,7 +215,7 @@ class TaskTimerListener: TaskExecutionListener {
             val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum()
             // TODO get this info from gradle plugin with exit code end stacktrace.
             val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b }
-            return BenchmarkResult("$programName",
+            return BenchmarkResult(programName,
                                     if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
                                     time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0)
         }

performance/build.gradle

@@ -79,6 +79,58 @@ private def uploadBenchmarkResultToBintray(String fileName) {
     }
 }
 
+// Register external benchmarks reports (e.g. results of libraries benchmarks)
+task registerExternalBenchmarks {
+    doLast {
+        def reports = externalReports.split(';')
+        def jsonReports = []
+        reports.each {
+            def reportFile = new File(buildDir, it)
+            if (!reportFile.exists())
+                return
+
+            def lines = reportFile.readLines()
+            if (lines.size < 3) {
+                println("Couldn't use report to register benchmarks. Wrong format.")
+                return
+            }
+            def name = lines[0]
+            def status = lines[1]
+            def compileTime = lines[2]
+            def codeSize = null
+            if (lines.size > 3) {
+                codeSize = lines[3]
+            }
+            // Create benchmarks report.
+            def compileBenchmark = MPPTools.toCompileBenchmark(compileTime, status, name)
+            def properties = [
+                    "cpu": System.getProperty("os.arch"),
+                    "os": System.getProperty("os.name"),
+                    "jdkVersion": System.getProperty("java.version"),
+                    "jdkVendor": System.getProperty("java.vendor"),
+                    "kotlinVersion": kotlinVersion,
+                    "type": "native",
+                    "compilerVersion": konanVersion,
+                    "flags": ["-g"],
+                    "benchmarks": "[]",
+                    "compileTime": [compileBenchmark]]
+            if (codeSize != null) {
+                properties += ["codeSize": MPPTools.toCodeSizeBenchmark(codeSize, status, name)]
+            }
+            def output = MPPTools.createJsonReport(properties)
+            def jsonFile = new File(buildDir, it.replace(".txt", ".json"))
+            jsonFile.write(output)
+            jsonReports.add(jsonFile)
+        }
+        def merged = MPPTools.mergeReports(jsonReports)
+        if (!merged.isEmpty()) {
+            mkdir buildDir.absolutePath
+            new File(buildDir, externalBenchmarksReport).write(merged)
+            uploadBenchmarkResultToBintray(externalBenchmarksReport)
+        }
+    }
+}
+
 task registerBuild(type: BuildRegister) {
     def analyzerBinary = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
             "${rootBuildDirectory}/${analyzerToolDirectory}")

performance/gradle.properties

@@ -15,3 +15,5 @@ outputReport = ../report/report.html
 bintrayUrl = https://api.bintray.com/content/lepilkinaelena
 bintrayRepo = KotlinNativePerformance
 bintrayPackage = jsonReports
+externalReports = coroutinesReport.txt
+externalBenchmarksReport = externalReport.json