From 454400a4a20cd205803808cebc4d2037d7445eb9 Mon Sep 17 00:00:00 2001 From: LepilkinaElena Date: Fri, 13 Sep 2019 09:49:45 +0300 Subject: [PATCH] External benchmarks (#3335) --- .../kotlin/org/jetbrains/kotlin/MPPTools.kt | 32 +++++++++--- performance/build.gradle | 52 +++++++++++++++++++ performance/gradle.properties | 2 + 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt index 55c8963d77c..0fccf73f774 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt @@ -12,9 +12,7 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.tasks.TaskState import org.gradle.api.execution.TaskExecutionListener -import org.gradle.api.tasks.AbstractExecTask import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink import org.jetbrains.report.* import org.jetbrains.report.json.* @@ -77,11 +75,21 @@ fun getFileSize(filePath: String): Long? { fun getCodeSizeBenchmark(programName: String, filePath: String): BenchmarkResult { val codeSize = getFileSize(filePath) - return BenchmarkResult("$programName", + return BenchmarkResult(programName, codeSize?. let { BenchmarkResult.Status.PASSED } ?: run { BenchmarkResult.Status.FAILED }, codeSize?.toDouble() ?: 0.0, BenchmarkResult.Metric.CODE_SIZE, codeSize?.toDouble() ?: 0.0, 1, 0) } +fun toCodeSizeBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult { + if (!metricDescription.startsWith("CODE_SIZE")) { + error("Wrong metric is used as code size.") + } + val codeSize = metricDescription.split(' ')[1].toDouble() + return BenchmarkResult(programName, + if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED, + codeSize, BenchmarkResult.Metric.CODE_SIZE, codeSize, 1, 0) +} + // Create benchmarks json report based on information get from gradle project fun createJsonReport(projectProperties: Map): String { fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown" @@ -96,17 +104,16 @@ fun createJsonReport(projectProperties: Map): String { val benchmarksArray = JsonTreeParser.parse(benchDesc) val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray) .union(projectProperties["compileTime"] as List).union( - listOf(projectProperties["codeSize"] as BenchmarkResult)).toList() + listOf(projectProperties["codeSize"] as? BenchmarkResult).filterNotNull()).toList() val report = BenchmarksReport(env, benchmarks, kotlin) return report.toJson() } fun mergeReports(reports: List): String { - val reportsToMerge = reports.map { + val reportsToMerge = reports.filter { it.exists() }.map { val json = it.inputStream().bufferedReader().use { it.readText() } val reportElement = JsonTreeParser.parse(json) BenchmarksReport.create(reportElement) - } return if (reportsToMerge.isEmpty()) "" else reportsToMerge.reduce { result, it -> result + it }.toJson() } @@ -186,9 +193,18 @@ fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable, r status = if (exitCodes["$it$number"] != 0) BenchmarkResult.Status.FAILED else status } - BenchmarkResult("$programName", status, time, BenchmarkResult.Metric.COMPILE_TIME, time, number, 0) + BenchmarkResult(programName, status, time, BenchmarkResult.Metric.COMPILE_TIME, time, number, 0) }.toList() +fun toCompileBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult { + if (!metricDescription.startsWith("COMPILE_TIME")) { + error("Wrong metric is used as compile time.") + } + val time = metricDescription.split(' ')[1].toDouble() + return BenchmarkResult(programName, + if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED, + time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0) +} // Class time tracker for all tasks. class TaskTimerListener: TaskExecutionListener { @@ -199,7 +215,7 @@ class TaskTimerListener: TaskExecutionListener { val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum() // TODO get this info from gradle plugin with exit code end stacktrace. val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b } - return BenchmarkResult("$programName", + return BenchmarkResult(programName, if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED, time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0) } diff --git a/performance/build.gradle b/performance/build.gradle index e4b0a50ca5d..79724be50fb 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -79,6 +79,58 @@ private def uploadBenchmarkResultToBintray(String fileName) { } } +// Register external benchmarks reports (e.g. results of libraries benchmarks) +task registerExternalBenchmarks { + doLast { + def reports = externalReports.split(';') + def jsonReports = [] + reports.each { + def reportFile = new File(buildDir, it) + if (!reportFile.exists()) + return + + def lines = reportFile.readLines() + if (lines.size < 3) { + println("Couldn't use report to register benchmarks. Wrong format.") + return + } + def name = lines[0] + def status = lines[1] + def compileTime = lines[2] + def codeSize = null + if (lines.size > 3) { + codeSize = lines[3] + } + // Create benchmarks report. + def compileBenchmark = MPPTools.toCompileBenchmark(compileTime, status, name) + def properties = [ + "cpu": System.getProperty("os.arch"), + "os": System.getProperty("os.name"), + "jdkVersion": System.getProperty("java.version"), + "jdkVendor": System.getProperty("java.vendor"), + "kotlinVersion": kotlinVersion, + "type": "native", + "compilerVersion": konanVersion, + "flags": ["-g"], + "benchmarks": "[]", + "compileTime": [compileBenchmark]] + if (codeSize != null) { + properties += ["codeSize": MPPTools.toCodeSizeBenchmark(codeSize, status, name)] + } + def output = MPPTools.createJsonReport(properties) + def jsonFile = new File(buildDir, it.replace(".txt", ".json")) + jsonFile.write(output) + jsonReports.add(jsonFile) + } + def merged = MPPTools.mergeReports(jsonReports) + if (!merged.isEmpty()) { + mkdir buildDir.absolutePath + new File(buildDir, externalBenchmarksReport).write(merged) + uploadBenchmarkResultToBintray(externalBenchmarksReport) + } + } +} + task registerBuild(type: BuildRegister) { def analyzerBinary = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}", "${rootBuildDirectory}/${analyzerToolDirectory}") diff --git a/performance/gradle.properties b/performance/gradle.properties index b2a436c56eb..6d6894a6b2a 100644 --- a/performance/gradle.properties +++ b/performance/gradle.properties @@ -15,3 +15,5 @@ outputReport = ../report/report.html bintrayUrl = https://api.bintray.com/content/lepilkinaelena bintrayRepo = KotlinNativePerformance bintrayPackage = jsonReports +externalReports = coroutinesReport.txt +externalBenchmarksReport = externalReport.json