azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added framework benchmark collecting code size and compile time (#2839)

Browse Source
This commit is contained in:
LepilkinaElena
2019-04-05 12:20:23 +03:00
committed by GitHub
parent 258f2bc0de
commit 87c852fb1d
9 changed files with 181 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
performance/buildSrc/src/main/kotlin/BuildRegister.kt
+27 -6
View File
@@ -46,7 +46,8 @@ open class BuildRegister : DefaultTask() {
var bundleSize: Int? = null
val buildInfoToken: Int = 4
val buildInfoTokens: Int = 4
val frameworkInfoTokens: Int = 3
val compileTimeSamplesNumber: Int = 2
val buildNumberTokens: Int = 3
val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg"
@@ -96,30 +97,50 @@ open class BuildRegister : DefaultTask() {
val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword)
val branch = getBuildProperty(currentBuild,"branchName")
val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
// Get summary information.
val output = arrayOf("$analyzer", "summary", "-exec-samples", "all", "-compile", "samples",
"-compile-samples", "HelloWorld,Videoplayer", "-codesize-samples", "all",
"-exec-normalize", "bintray:goldenResults.csv",
"-codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
.runCommand()
// Postprocess information.
val buildInfoParts = output.split(',')
if (buildInfoParts.size != buildInfoToken) {
error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile.")
if (buildInfoParts.size != buildInfoTokens) {
error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $output")
}
val (failures, executionTime, compileTime, codeSize) = buildInfoParts.map { it.trim() }
// Add legends.
val geometricMean = "Geometric Mean-"
val executionTimeInfo = "$geometricMean$executionTime"
val codeSizeInfo = "$geometricMean$codeSize"
var codeSizeInfo = "$geometricMean$codeSize"
val compileTimeSamples = compileTime.split(';')
if (compileTimeSamples.size != compileTimeSamplesNumber) {
error("Problems with getting compile time samples value. Expected at least $compileTimeSamplesNumber samples, got ${compileTimeSamples.size}")
}
val (helloWorldCompile, videoplayerCompile) = compileTimeSamples
val compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile"
val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
var compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile"
// Collect framework run details.
if (target == "MacOSX") {
val frameworkOutput = arrayOf("$analyzer", "summary", "-compile", "samples",
"-compile-samples", "FrameworkBenchmarksAnalyzer", "-codesize-samples", "FrameworkBenchmarksAnalyzer",
"-codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
.runCommand()
val buildInfoPartsFramework = frameworkOutput.split(',')
if (buildInfoPartsFramework.size != frameworkInfoTokens) {
error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $frameworkOutput")
}
val (_, frameworkCompileTime, frameworkCodeSize) = buildInfoPartsFramework.map { it.trim() }
codeSizeInfo += ";FrameworkBenchmarksAnalyzer-$frameworkCodeSize"
compileTimeInfo += ";FrameworkBenchmarksAnalyzer-$frameworkCompileTime"
}
val buildNumberParts = buildNumber.split("-")
if (buildNumberParts.size != buildNumberTokens) {
error("Wrong format of build number $buildNumber.")
performance/buildSrc/src/main/kotlin/MPPTools.kt
+5 -3
View File
@@ -117,7 +117,7 @@ fun mergeReports(reports: List<File>): String {
BenchmarksReport.create(reportElement)
}
return reportsToMerge.reduce { result, it -> result + it }.toJson()
return if (reportsToMerge.isEmpty()) "" else reportsToMerge.reduce { result, it -> result + it }.toJson()
}
// Find file with set name in directory.
@@ -172,8 +172,10 @@ fun createRunTask(
fun getJvmCompileTime(programName: String): BenchmarkResult =
TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
fun getNativeCompileTime(programName: String): BenchmarkResult =
TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative"))
@JvmOverloads
fun getNativeCompileTime(programName: String,
tasks: List<String> = listOf("compileKotlinNative", "linkMainReleaseExecutableNative")): BenchmarkResult =
TaskTimerListener.getBenchmarkResult(programName, tasks)
fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable<String>, repeats: Int, exitCodes: Map<String, Int>) =
(1..repeats).map { number ->
performance/framework/build.gradle
+115
View File
@@ -0,0 +1,115 @@
buildscript {
ext.rootBuildDirectory = file('../..')
apply from: "$rootBuildDirectory/gradle/loadRootProperties.gradle"
apply from: "$rootBuildDirectory/gradle/kotlinGradlePlugin.gradle"
repositories {
maven {
url 'https://cache-redirector.jetbrains.com/jcenter'
}
maven {
url kotlinCompilerRepo
}
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
}
}
apply plugin: 'kotlin-multiplatform'
repositories {
maven {
url 'https://cache-redirector.jetbrains.com/jcenter'
}
maven {
url kotlinCompilerRepo
}
maven {
url buildKotlinCompilerRepo
}
}
def toolsPath = '../../tools'
def frameworkName = 'benchmarksAnalyzer'
kotlin {
sourceSets {
macosMain {
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
}
kotlin.srcDir "$toolsPath/benchmarks/shared/src"
kotlin.srcDir "$toolsPath/benchmarksAnalyzer/src/main/kotlin"
kotlin.srcDir "$toolsPath/kliopt"
kotlin.srcDir "$toolsPath/benchmarksAnalyzer/src/main/kotlin-native"
}
}
configure([macosX64("macos")]) {
compilations.main.cinterops {
libcurl {
defFile "$toolsPath/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def"
}
}
}
macosX64("macos").binaries {
framework(frameworkName, [RELEASE])
}
}
MPPTools.addTimeListener(project)
task konanRun {
if (MPPTools.isMacos()) {
dependsOn 'build'
}
}
task jvmRun(type: RunJvmTask) {
doLast {
println("JVM run is unsupported")
}
}
task konanJsonReport {
doLast {
if (MPPTools.isMacos()) {
def applicationName = "FrameworkBenchmarksAnalyzer"
def frameworkPath = kotlin.macosX64("macos").binaries.
getFramework(frameworkName, kotlin.macosX64("macos").binaries.RELEASE).outputFile.absolutePath
def nativeExecutable = new File("$frameworkPath/$frameworkName").canonicalPath
def nativeCompileTime = MPPTools.getNativeCompileTime(applicationName, ['compileKotlinMacos',
'linkBenchmarksAnalyzerReleaseFrameworkMacos',
'cinteropLibcurlMacos'])
def properties = getCommonProperties() + ['type' : 'native',
'compilerVersion': "${konanVersion}".toString(),
'flags' : [],
'benchmarks' : '[]',
'compileTime' : [nativeCompileTime],
'codeSize' : MPPTools.getCodeSizeBenchmark(applicationName, nativeExecutable)]
def output = MPPTools.createJsonReport(properties)
new File("${buildDir.absolutePath}/${nativeJson}").write(output)
}
}
}
task jvmJsonReport {
doLast {
println("JVM run is unsupported")
}
}
jvmRun.finalizedBy jvmJsonReport
konanRun.finalizedBy konanJsonReport
private def getCommonProperties() {
return ['cpu': System.getProperty("os.arch"),
'os': System.getProperty("os.name"), // OperatingSystem.current().getName()
'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion
'jdkVendor': System.getProperty("java.vendor"),
'kotlinVersion': "${kotlinVersion}".toString()]
}
performance/framework/gradle.properties
+1
View File
@@ -0,0 +1 @@
org.jetbrains.kotlin.native.home=../../dist
performance/settings.gradle
+2 -1
View File
@@ -1,4 +1,5 @@
include ':ring'
include ':cinterop'
include ':helloworld'
include ':videoplayer'
include ':videoplayer'
include ':framework'
performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt
+2 -2
View File
@@ -72,8 +72,8 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v
object BenchmarksRunner {
fun parse(args: Array<String>): ArgParser {
val options = listOf(
OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "0"),
OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "10"),
OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "20"),
OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "60"),
OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""),
OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true)
tools/performance-server/ui/css/style.css
+18 -12
View File
@@ -45,14 +45,18 @@
border-color: gold;
}
.ct-legend .ct-series-2:before {
background-color: limegreen;
border-color: limegreen;
}
.ct-legend .ct-series-3:before {
background-color: mediumorchid;
border-color: mediumorchid;
}
.ct-legend .ct-series-3:before {
background-color: #d17905;
border-color: #d17905;
}
.ct-legend .ct-series-4:before {
background-color: navy;
border-color: navy;
}
.ct-legend .ct-series-5:before {
background-color: #453d3f;
border-color: #453d3f;
}
@@ -66,16 +70,18 @@
.ct-series-c .ct-line,
.ct-series-c .ct-point {
/* Set the colour of this series line */
stroke: mediumorchid;
}
.ct-series-d .ct-line {
/* Set the colour of this series line */
stroke: mediumorchid;
stroke: limegreen;
}
.ct-series-d .ct-line,
.ct-series-d .ct-point {
/* Set the colour of this series line */
stroke: mediumorchid;
stroke-width: 20;
}
}
.ct-series-e .ct-line,
.ct-series-e .ct-point {
/* Set the colour of this series line */
stroke: navy;
}
.tooltip { pointer-events: none; }
tools/performance-server/ui/index.ejs
+1 -1
View File
@@ -11,7 +11,7 @@
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js">
</script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js">
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js">
</script>
</head>
<body>
tools/performance-server/ui/src/main/kotlin/main.kt
+10 -10
View File
@@ -69,13 +69,13 @@ fun <T : Any> separateValues(values: String, valuesContainer: MutableMap<String,
}
}
fun getChartData(labels: List<String>, valuesList: Collection<List<*>>, className: String? = null): dynamic {
fun getChartData(labels: List<String>, valuesList: Collection<List<*>>, classNames: Array<String>? = null): dynamic {
val chartData: dynamic = object{}
chartData["labels"] = labels.toTypedArray()
chartData["series"] = valuesList.map {
chartData["series"] = valuesList.mapIndexed { index, it ->
val series: dynamic = object{}
series["data"] = it.toTypedArray()
className?.let { series["className"] = className }
classNames?.let { series["className"] = classNames[index] }
series
}.toTypedArray()
return chartData
@@ -142,7 +142,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
"L", data.x + pointSize, data.y + pointSize/2, "z").joinToString(" ")
svgParameters["style"] = "fill:rgb(255,0,0);stroke-width:0"
val triangle = Chartist.Svg("path", svgParameters, chartContainer)
element = data.element._node.replace(triangle)
element = data.element.replace(triangle)
} else if (currentBuild.buildNumber == parameters["build"]) {
// Higlight choosen build.
val svgParameters: dynamic = object{}
@@ -228,7 +228,7 @@ fun main(args: Array<String>) {
val branchesUrl = "$serverUrl/branches/${parameters["target"]}"
val branches: Array<String> = JSON.parse(sendGetRequest(branchesUrl))
val releaseBranches = branches.filter { "v\\d+\\.\\d+\\.\\d+-fixes".toRegex().find(it) != null }
val releaseBranches = branches.filter { "^v\\d+\\.\\d+\\.\\d+-fixes$".toRegex().find(it) != null }
// Fill autocomplete list.
val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}"
@@ -312,17 +312,17 @@ fun main(args: Array<String>) {
bundleSize.add(it.bundleSize?.toInt()?. let { it / 1024 / 1024 })
}
val sizeClassName = "ct-series-c"
val sizeClassNames = arrayOf("ct-series-d", "ct-series-e")
// Draw charts.
val execChart = Chartist.Line("#exec_chart", getChartData(labels, executionTime.values),
getChartOptions(executionTime.keys.toTypedArray(), "Normalized time"))
val compileChart = Chartist.Line("#compile_chart", getChartData(labels, compileTime.values),
getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds"))
val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassName),
getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-2")))
val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassName),
getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-2")))
val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassNames),
getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-3", "ct-series-4")))
val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassNames),
getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-3")))
// Tooltips and higlights.
customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)