azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added short summary report for all targets. (#2622)

Browse Source
This commit is contained in:
LepilkinaElena
2019-02-05 10:17:04 +03:00
committed by GitHub
parent e50f64bae2
commit a9d86b0247
7 changed files with 209 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
performance/build.gradle
+9
View File
@@ -185,9 +185,18 @@ task slackReport(type: RegressionsReporter) {
// Create folder for report (root Kotlin project settings make create report in separate folder).
def reportDirectory = outputReport.substring(0, outputReport.lastIndexOf("/"))
mkdir reportDirectory
mkdir "../targetsResults"
currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
"${rootBuildDirectory}/${analyzerToolDirectory}")
htmlReport = outputReport
defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
def target = System.getProperty("os.name")
summaryFile = "../targetsResults/${target}.txt"
}
task slackSummary(type: RegressionsSummaryReporter) {
targetsResultFiles = ["Linux": "../targetsResults/Linux.txt",
"MacOSX": "../targetsResults/Mac OS X.txt",
"Windows": "../targetsResults/Windows 10.txt"]
}
performance/buildSrc/src/main/kotlin/RegressionsReporter.kt
+7 -1
View File
@@ -76,7 +76,8 @@ class CommitsList(data: JsonElement): ConvertedFromJson {
* @property currentBenchmarksReportFile path to file with becnhmarks result
* @property analyzer path to analyzer tool
* @property htmlReport name of result html report
* @property defaultBranch name of defaukt branch
* @property defaultBranch name of default branch
* @property summaryFile name of file with short summary
*/
open class RegressionsReporter : DefaultTask() {
@@ -110,6 +111,9 @@ open class RegressionsReporter : DefaultTask() {
@Input
lateinit var defaultBranch: String
@Input
lateinit var summaryFile: String
private fun tabUrl(buildId: String, buildTypeId: String, tab: String) =
"$teamCityUrl/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab"
@@ -215,6 +219,8 @@ open class RegressionsReporter : DefaultTask() {
"bintray:$compareToBuildNumber:$target:$bintrayFileName with $analyzer! " +
"Please check files existance and their correctness.")
}
"$analyzer -r statistics $currentBenchmarksReportFile bintray:$compareToBuildNumber:$target:$bintrayFileName -o $summaryFile"
.runCommand()
val reportLink = "http://kotlin-native-performance.labs.jb.gg/?" +
"report=bintray:$buildNumber:$target:$bintrayFileName&" +
performance/buildSrc/src/main/kotlin/RegressionsSummaryReporter.kt
+100
View File
@@ -0,0 +1,100 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.DefaultTask
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory
import com.ullink.slack.simpleslackapi.SlackAttachment
import com.ullink.slack.simpleslackapi.SlackPreparedMessage
import java.io.FileInputStream
import java.io.File
import java.util.Properties
/**
* Task to produce regressions report and send it to slack. Requires a report with current benchmarks result
* and path to analyzer tool
*
* @property targetsResultFiles map with pathes to results of each target
*/
open class RegressionsSummaryReporter : DefaultTask() {
@Input
lateinit var targetsResultFiles: Map<String, String>
@TaskAction
fun run() {
// Get TeamCity properties.
val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?:
error("Can't load teamcity config!")
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val buildTypeId = buildProperties.getProperty("teamcity.buildType.id")
val buildNumber = buildProperties.getProperty("build.number")
val results = mutableMapOf<String, MutableMap<String, String>>()
// Parse and merge results from all targets.
targetsResultFiles.forEach { (key, value) ->
val file = File(value)
if (file.exists()) {
file.forEachLine {
val matchResult = "(\\w+)\\s*:\\s*(\\w+)".toRegex().find(it)
val propertyName = matchResult?.groups?.get(1)?.value
val propertyValue = matchResult?.groups?.get(2)?.value
if (propertyName != null && propertyValue != null) {
results[propertyName]?.let { it[key] = propertyValue }
?: run { results.put(propertyName, mutableMapOf(key to propertyValue)) }
}
}
}
}
val message = buildString {
results.forEach { (property, targets) ->
append("$property: ${targets.map {(target, value) -> "$value ($target)"}.joinToString(" | ")}\n")
}
}
val summaryStatus = results["status"]?.values?.fold("STABLE") {summary, element ->
when {
summary == "FAILED" || element == "FAILED" -> "FAILED"
summary == "FIXED" || element == "FIXED" -> "FIXED"
summary == "STABLE" -> element
summary == element -> summary
else -> "UNSTABLE"
}
}
val attachement = SlackAttachment()
with (attachement) {
setTitle("Performance Summary (build $buildNumber)")
setTitleLink("https://buildserver.labs.intellij.net/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId")
setText(message)
if (summaryStatus == "FIXED" || summaryStatus == "IMPROVED") {
setColor("#36a64f")
} else if (summaryStatus == "FAILED" || summaryStatus == "REGRESSED") {
setColor("#ff0000")
}
}
// Send to channel or user directly.
val session = SlackSessionFactory.createWebSocketSlackSession(buildProperties.getProperty("konan-reporter-token"))
session.connect()
val channel = session.findChannelByName(buildProperties.getProperty("konan-channel-name"))
val preparedMessage = SlackPreparedMessage.Builder()
.addAttachment(attachement)
.build()
session.sendMessage(channel, preparedMessage)
session.disconnect()
}
}
tools/benchmarksAnalyzer/src/main/kotlin/main.kt
+1 -1
View File
@@ -83,7 +83,7 @@ fun main(args: Array<String>) {
OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "0.5"),
OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"),
OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity")),
OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")),
"renders", "r", "Renders for showing information", "text", isMultiple = true),
OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization")
)
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt
+20
View File
@@ -468,6 +468,26 @@ class HTMLRender: Render() {
}
}
}
if (!report.addedBenchmarks.isEmpty()) {
renderCollapsedData("Added", true) {
table {
attributes["class"] = "table table-sm table-striped table-hover"
attributes["style"] = "width:initial; font-size: 11pt;"
renderTableFromList(report.addedBenchmarks, "Added benchmarks")
}
}
}
if (!report.removedBenchmarks.isEmpty()) {
renderCollapsedData("Removed", true) {
table {
attributes["class"] = "table table-sm table-striped table-hover"
attributes["style"] = "width:initial; font-size: 11pt;"
renderTableFromList(report.removedBenchmarks, "Removed benchmarks")
}
}
}
}
private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) {
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt
+1
View File
@@ -31,6 +31,7 @@ abstract class Render {
"text" -> TextRender()
"html" -> HTMLRender()
"teamcity" -> TeamCityStatisticsRender()
"statistics" -> StatisticsRender()
else -> error("Unknown render $name")
}
}
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.renders
import org.jetbrains.analyzer.*
import org.jetbrains.report.BenchmarkResult
enum class Status {
FAILED, FIXED, IMPROVED, REGRESSED, STABLE, UNSTABLE
}
// Report render to short summary statistics.
class StatisticsRender: Render() {
override val name: String
get() = "statistics"
private var content = StringBuilder()
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
val newPasses = benchmarksWithChangedStatus
.filter { it.current == BenchmarkResult.Status.PASSED }
val newFailures = benchmarksWithChangedStatus
.filter { it.current == BenchmarkResult.Status.FAILED }
if (report.failedBenchmarks.isNotEmpty()) {
content.append("failed: ${report.failedBenchmarks.size}\n")
}
val status = when {
newFailures.isNotEmpty() -> {
content.append("new failures: ${newFailures.size}\n")
Status.FAILED
}
newPasses.isNotEmpty() -> {
content.append("new passes: ${newPasses.size}\n")
Status.FIXED
}
report.improvements.isNotEmpty() && report.regressions.isNotEmpty() -> {
content.append("regressions: ${report.regressions.size}\nimprovements: ${report.improvements.size}")
Status.UNSTABLE
}
report.improvements.isNotEmpty() && report.regressions.isEmpty() -> {
content.append("improvements: ${report.improvements.size}")
Status.IMPROVED
}
report.improvements.isEmpty() && report.regressions.isNotEmpty() -> {
content.append("regressions: ${report.regressions.size}")
Status.REGRESSED
}
else -> Status.STABLE
}
return """
status: $status
total: ${report.benchmarksNumber}
""".trimIndent() + "\n$content"
}
}