diff --git a/performance/build.gradle b/performance/build.gradle index 647b16376ef..e89f6fe0f40 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -185,9 +185,18 @@ task slackReport(type: RegressionsReporter) { // Create folder for report (root Kotlin project settings make create report in separate folder). def reportDirectory = outputReport.substring(0, outputReport.lastIndexOf("/")) mkdir reportDirectory + mkdir "../targetsResults" currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}" analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}", "${rootBuildDirectory}/${analyzerToolDirectory}") htmlReport = outputReport defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master" + def target = System.getProperty("os.name") + summaryFile = "../targetsResults/${target}.txt" +} + +task slackSummary(type: RegressionsSummaryReporter) { + targetsResultFiles = ["Linux": "../targetsResults/Linux.txt", + "MacOSX": "../targetsResults/Mac OS X.txt", + "Windows": "../targetsResults/Windows 10.txt"] } \ No newline at end of file diff --git a/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt b/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt index eee36ce7bc6..36aed86fbd8 100644 --- a/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt +++ b/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt @@ -76,7 +76,8 @@ class CommitsList(data: JsonElement): ConvertedFromJson { * @property currentBenchmarksReportFile path to file with becnhmarks result * @property analyzer path to analyzer tool * @property htmlReport name of result html report - * @property defaultBranch name of defaukt branch + * @property defaultBranch name of default branch + * @property summaryFile name of file with short summary */ open class RegressionsReporter : DefaultTask() { @@ -110,6 +111,9 @@ open class RegressionsReporter : DefaultTask() { @Input lateinit var defaultBranch: String + @Input + lateinit var summaryFile: String + private fun tabUrl(buildId: String, buildTypeId: String, tab: String) = "$teamCityUrl/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab" @@ -215,6 +219,8 @@ open class RegressionsReporter : DefaultTask() { "bintray:$compareToBuildNumber:$target:$bintrayFileName with $analyzer! " + "Please check files existance and their correctness.") } + "$analyzer -r statistics $currentBenchmarksReportFile bintray:$compareToBuildNumber:$target:$bintrayFileName -o $summaryFile" + .runCommand() val reportLink = "http://kotlin-native-performance.labs.jb.gg/?" + "report=bintray:$buildNumber:$target:$bintrayFileName&" + diff --git a/performance/buildSrc/src/main/kotlin/RegressionsSummaryReporter.kt b/performance/buildSrc/src/main/kotlin/RegressionsSummaryReporter.kt new file mode 100644 index 00000000000..7cf4a40cba4 --- /dev/null +++ b/performance/buildSrc/src/main/kotlin/RegressionsSummaryReporter.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +import groovy.lang.Closure +import org.gradle.api.Action +import org.gradle.api.DefaultTask +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction + +import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory +import com.ullink.slack.simpleslackapi.SlackAttachment +import com.ullink.slack.simpleslackapi.SlackPreparedMessage + +import java.io.FileInputStream +import java.io.File +import java.util.Properties + +/** + * Task to produce regressions report and send it to slack. Requires a report with current benchmarks result + * and path to analyzer tool + * + * @property targetsResultFiles map with pathes to results of each target + */ +open class RegressionsSummaryReporter : DefaultTask() { + @Input + lateinit var targetsResultFiles: Map + + @TaskAction + fun run() { + // Get TeamCity properties. + val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?: + error("Can't load teamcity config!") + + val buildProperties = Properties() + buildProperties.load(FileInputStream(teamcityConfig)) + val buildId = buildProperties.getProperty("teamcity.build.id") + val buildTypeId = buildProperties.getProperty("teamcity.buildType.id") + val buildNumber = buildProperties.getProperty("build.number") + val results = mutableMapOf>() + + // Parse and merge results from all targets. + targetsResultFiles.forEach { (key, value) -> + val file = File(value) + if (file.exists()) { + file.forEachLine { + val matchResult = "(\\w+)\\s*:\\s*(\\w+)".toRegex().find(it) + val propertyName = matchResult?.groups?.get(1)?.value + val propertyValue = matchResult?.groups?.get(2)?.value + if (propertyName != null && propertyValue != null) { + results[propertyName]?.let { it[key] = propertyValue } + ?: run { results.put(propertyName, mutableMapOf(key to propertyValue)) } + } + } + } + } + + val message = buildString { + results.forEach { (property, targets) -> + append("$property: ${targets.map {(target, value) -> "$value ($target)"}.joinToString(" | ")}\n") + } + } + val summaryStatus = results["status"]?.values?.fold("STABLE") {summary, element -> + when { + summary == "FAILED" || element == "FAILED" -> "FAILED" + summary == "FIXED" || element == "FIXED" -> "FIXED" + summary == "STABLE" -> element + summary == element -> summary + else -> "UNSTABLE" + } + } + + val attachement = SlackAttachment() + with (attachement) { + setTitle("Performance Summary (build $buildNumber)") + setTitleLink("https://buildserver.labs.intellij.net/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId") + setText(message) + if (summaryStatus == "FIXED" || summaryStatus == "IMPROVED") { + setColor("#36a64f") + } else if (summaryStatus == "FAILED" || summaryStatus == "REGRESSED") { + setColor("#ff0000") + } + } + + // Send to channel or user directly. + val session = SlackSessionFactory.createWebSocketSlackSession(buildProperties.getProperty("konan-reporter-token")) + session.connect() + + val channel = session.findChannelByName(buildProperties.getProperty("konan-channel-name")) + val preparedMessage = SlackPreparedMessage.Builder() + .addAttachment(attachement) + .build() + session.sendMessage(channel, preparedMessage) + + session.disconnect() + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index 3443763d894..eaf6ee58512 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -83,7 +83,7 @@ fun main(args: Array) { OptionDescriptor(ArgType.String(), "output", "o", "Output file"), OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "0.5"), OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"), - OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity")), + OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")), "renders", "r", "Renders for showing information", "text", isMultiple = true), OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization") ) diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt index 4c5617cc436..d6bd4eb939f 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt @@ -468,6 +468,26 @@ class HTMLRender: Render() { } } } + + if (!report.addedBenchmarks.isEmpty()) { + renderCollapsedData("Added", true) { + table { + attributes["class"] = "table table-sm table-striped table-hover" + attributes["style"] = "width:initial; font-size: 11pt;" + renderTableFromList(report.addedBenchmarks, "Added benchmarks") + } + } + } + + if (!report.removedBenchmarks.isEmpty()) { + renderCollapsedData("Removed", true) { + table { + attributes["class"] = "table table-sm table-striped table-hover" + attributes["style"] = "width:initial; font-size: 11pt;" + renderTableFromList(report.removedBenchmarks, "Removed benchmarks") + } + } + } } private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) { diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt index 0ed2d4ddead..91eecea433b 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt @@ -31,6 +31,7 @@ abstract class Render { "text" -> TextRender() "html" -> HTMLRender() "teamcity" -> TeamCityStatisticsRender() + "statistics" -> StatisticsRender() else -> error("Unknown render $name") } } diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt new file mode 100644 index 00000000000..a4a73252b56 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.renders + +import org.jetbrains.analyzer.* +import org.jetbrains.report.BenchmarkResult + +enum class Status { + FAILED, FIXED, IMPROVED, REGRESSED, STABLE, UNSTABLE +} + +// Report render to short summary statistics. +class StatisticsRender: Render() { + override val name: String + get() = "statistics" + + private var content = StringBuilder() + + override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { + val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus() + val newPasses = benchmarksWithChangedStatus + .filter { it.current == BenchmarkResult.Status.PASSED } + val newFailures = benchmarksWithChangedStatus + .filter { it.current == BenchmarkResult.Status.FAILED } + if (report.failedBenchmarks.isNotEmpty()) { + content.append("failed: ${report.failedBenchmarks.size}\n") + } + val status = when { + newFailures.isNotEmpty() -> { + content.append("new failures: ${newFailures.size}\n") + Status.FAILED + } + newPasses.isNotEmpty() -> { + content.append("new passes: ${newPasses.size}\n") + Status.FIXED + } + report.improvements.isNotEmpty() && report.regressions.isNotEmpty() -> { + content.append("regressions: ${report.regressions.size}\nimprovements: ${report.improvements.size}") + Status.UNSTABLE + } + report.improvements.isNotEmpty() && report.regressions.isEmpty() -> { + content.append("improvements: ${report.improvements.size}") + Status.IMPROVED + } + report.improvements.isEmpty() && report.regressions.isNotEmpty() -> { + content.append("regressions: ${report.regressions.size}") + Status.REGRESSED + } + else -> Status.STABLE + } + return """ + status: $status + total: ${report.benchmarksNumber} + """.trimIndent() + "\n$content" + } +} \ No newline at end of file