Added framework benchmark collecting code size and compile time (#2839)

performance/buildSrc/src/main/kotlin/BuildRegister.kt

@@ -46,7 +46,8 @@ open class BuildRegister : DefaultTask() {
 
     var bundleSize: Int? = null
 
-    val buildInfoToken: Int = 4
+    val buildInfoTokens: Int = 4
+    val frameworkInfoTokens: Int = 3
     val compileTimeSamplesNumber: Int = 2
     val buildNumberTokens: Int = 3
     val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg"
@@ -96,30 +97,50 @@ open class BuildRegister : DefaultTask() {
         val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword)
         val branch = getBuildProperty(currentBuild,"branchName")
 
+        val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
+
         // Get summary information.
         val output = arrayOf("$analyzer", "summary", "-exec-samples", "all", "-compile", "samples",
                 "-compile-samples", "HelloWorld,Videoplayer", "-codesize-samples", "all",
                 "-exec-normalize", "bintray:goldenResults.csv",
                 "-codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
                 .runCommand()
+
         // Postprocess information.
         val buildInfoParts = output.split(',')
-        if (buildInfoParts.size != buildInfoToken) {
+        if (buildInfoParts.size != buildInfoTokens) {
-            error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile.")
+            error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $output")
         }
 
         val (failures, executionTime, compileTime, codeSize) = buildInfoParts.map { it.trim() }
         // Add legends.
         val geometricMean = "Geometric Mean-"
         val executionTimeInfo = "$geometricMean$executionTime"
-        val codeSizeInfo = "$geometricMean$codeSize"
+        var codeSizeInfo = "$geometricMean$codeSize"
         val compileTimeSamples = compileTime.split(';')
         if (compileTimeSamples.size != compileTimeSamplesNumber) {
             error("Problems with getting compile time samples value. Expected at least $compileTimeSamplesNumber samples, got ${compileTimeSamples.size}")
         }
         val (helloWorldCompile, videoplayerCompile) = compileTimeSamples
-        val compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile"
+        var compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile"
-        val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
+
+        // Collect framework run details.
+        if (target == "MacOSX") {
+
+            val frameworkOutput = arrayOf("$analyzer", "summary", "-compile", "samples",
+                    "-compile-samples", "FrameworkBenchmarksAnalyzer", "-codesize-samples", "FrameworkBenchmarksAnalyzer",
+                    "-codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
+                    .runCommand()
+
+            val buildInfoPartsFramework = frameworkOutput.split(',')
+            if (buildInfoPartsFramework.size != frameworkInfoTokens) {
+                error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $frameworkOutput")
+            }
+            val (_, frameworkCompileTime, frameworkCodeSize) = buildInfoPartsFramework.map { it.trim() }
+            codeSizeInfo += ";FrameworkBenchmarksAnalyzer-$frameworkCodeSize"
+            compileTimeInfo += ";FrameworkBenchmarksAnalyzer-$frameworkCompileTime"
+        }
+
         val buildNumberParts = buildNumber.split("-")
         if (buildNumberParts.size != buildNumberTokens) {
             error("Wrong format of build number $buildNumber.")

performance/buildSrc/src/main/kotlin/MPPTools.kt

@@ -117,7 +117,7 @@ fun mergeReports(reports: List<File>): String {
         BenchmarksReport.create(reportElement)
 
     }
-    return reportsToMerge.reduce { result, it -> result + it }.toJson()
+    return if (reportsToMerge.isEmpty()) "" else reportsToMerge.reduce { result, it -> result + it }.toJson()
 }
 
 // Find file with set name in directory.
@@ -172,8 +172,10 @@ fun createRunTask(
 fun getJvmCompileTime(programName: String): BenchmarkResult =
         TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
 
-fun getNativeCompileTime(programName: String): BenchmarkResult =
+@JvmOverloads
-        TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative"))
+fun getNativeCompileTime(programName: String,
+                         tasks: List<String> = listOf("compileKotlinNative", "linkMainReleaseExecutableNative")): BenchmarkResult =
+        TaskTimerListener.getBenchmarkResult(programName, tasks)
 
 fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable<String>, repeats: Int, exitCodes: Map<String, Int>) =
     (1..repeats).map { number ->

performance/framework/build.gradle

@@ -0,0 +1,115 @@
+buildscript {
+    ext.rootBuildDirectory = file('../..')
+
+    apply from: "$rootBuildDirectory/gradle/loadRootProperties.gradle"
+    apply from: "$rootBuildDirectory/gradle/kotlinGradlePlugin.gradle"
+
+    repositories {
+        maven {
+            url 'https://cache-redirector.jetbrains.com/jcenter'
+        }
+        maven {
+            url kotlinCompilerRepo
+        }
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
+    }
+}
+
+apply plugin: 'kotlin-multiplatform'
+
+repositories {
+    maven {
+        url 'https://cache-redirector.jetbrains.com/jcenter'
+    }
+    maven {
+        url kotlinCompilerRepo
+    }
+    maven {
+        url buildKotlinCompilerRepo
+    }
+}
+
+def toolsPath = '../../tools'
+def frameworkName = 'benchmarksAnalyzer'
+
+kotlin {
+    sourceSets {
+        macosMain {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
+            }
+            kotlin.srcDir "$toolsPath/benchmarks/shared/src"
+            kotlin.srcDir "$toolsPath/benchmarksAnalyzer/src/main/kotlin"
+            kotlin.srcDir "$toolsPath/kliopt"
+            kotlin.srcDir "$toolsPath/benchmarksAnalyzer/src/main/kotlin-native"
+        }
+    }
+
+    configure([macosX64("macos")]) {
+        compilations.main.cinterops {
+            libcurl {
+                defFile "$toolsPath/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def"
+            }
+        }
+    }
+
+    macosX64("macos").binaries {
+        framework(frameworkName, [RELEASE])
+    }
+}
+
+MPPTools.addTimeListener(project)
+
+task konanRun {
+    if (MPPTools.isMacos()) {
+        dependsOn 'build'
+    }
+}
+
+task jvmRun(type: RunJvmTask)  {
+    doLast {
+        println("JVM run is unsupported")
+    }
+}
+
+task konanJsonReport {
+    doLast {
+        if (MPPTools.isMacos()) {
+            def applicationName = "FrameworkBenchmarksAnalyzer"
+            def frameworkPath = kotlin.macosX64("macos").binaries.
+                    getFramework(frameworkName, kotlin.macosX64("macos").binaries.RELEASE).outputFile.absolutePath
+            def nativeExecutable = new File("$frameworkPath/$frameworkName").canonicalPath
+            def nativeCompileTime = MPPTools.getNativeCompileTime(applicationName, ['compileKotlinMacos',
+                                                                                    'linkBenchmarksAnalyzerReleaseFrameworkMacos',
+                                                                                    'cinteropLibcurlMacos'])
+            def properties = getCommonProperties() + ['type'           : 'native',
+                                                      'compilerVersion': "${konanVersion}".toString(),
+                                                      'flags'          : [],
+                                                      'benchmarks'     : '[]',
+                                                      'compileTime'    : [nativeCompileTime],
+                                                      'codeSize'       : MPPTools.getCodeSizeBenchmark(applicationName, nativeExecutable)]
+            def output = MPPTools.createJsonReport(properties)
+            new File("${buildDir.absolutePath}/${nativeJson}").write(output)
+        }
+    }
+}
+
+task jvmJsonReport {
+    doLast {
+        println("JVM run is unsupported")
+    }
+}
+
+jvmRun.finalizedBy jvmJsonReport
+konanRun.finalizedBy konanJsonReport
+
+private def getCommonProperties() {
+    return ['cpu': System.getProperty("os.arch"),
+            'os': System.getProperty("os.name"), // OperatingSystem.current().getName()
+            'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion
+            'jdkVendor': System.getProperty("java.vendor"),
+            'kotlinVersion': "${kotlinVersion}".toString()]
+}

performance/framework/gradle.properties

@@ -0,0 +1 @@
+org.jetbrains.kotlin.native.home=../../dist

performance/settings.gradle

@@ -2,3 +2,4 @@ include ':ring'
 include ':cinterop'
 include ':helloworld'
 include ':videoplayer'
+include ':framework'

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt

@@ -72,8 +72,8 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v
 object BenchmarksRunner {
     fun parse(args: Array<String>): ArgParser {
         val options = listOf(
-                OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "0"),
+                OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "20"),
-                OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "10"),
+                OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "60"),
                 OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""),
                 OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
                 OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true)

tools/performance-server/ui/css/style.css

@@ -45,14 +45,18 @@
   border-color: gold;
 }
 .ct-legend .ct-series-2:before {
+  background-color: limegreen;
+  border-color: limegreen;
+}
+.ct-legend .ct-series-3:before {
   background-color: mediumorchid;
   border-color: mediumorchid;
 }
-.ct-legend .ct-series-3:before {
-  background-color: #d17905;
-  border-color: #d17905;
-}
 .ct-legend .ct-series-4:before {
+  background-color: navy;
+  border-color: navy;
+}
+.ct-legend .ct-series-5:before {
   background-color: #453d3f;
   border-color: #453d3f;
 }
@@ -66,16 +70,18 @@
 .ct-series-c .ct-line,
 .ct-series-c .ct-point {
   /* Set the colour of this series line */
-  stroke: mediumorchid;
+  stroke: limegreen;
-}
-
-.ct-series-d .ct-line {
-  /* Set the colour of this series line */
-  stroke: mediumorchid;
 }
 
+.ct-series-d .ct-line,
 .ct-series-d .ct-point {
   /* Set the colour of this series line */
   stroke: mediumorchid;
-  stroke-width: 20;
 }
+
+.ct-series-e .ct-line,
+.ct-series-e .ct-point {
+  /* Set the colour of this series line */
+  stroke: navy;
+}
+.tooltip { pointer-events: none; }

tools/performance-server/ui/index.ejs

@@ -11,7 +11,7 @@
     </script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js">
     </script>
-    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js">
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js">
     </script>
 </head>
 <body>

tools/performance-server/ui/src/main/kotlin/main.kt

@@ -69,13 +69,13 @@ fun <T : Any> separateValues(values: String, valuesContainer: MutableMap<String,
     }
 }
 
-fun getChartData(labels: List<String>, valuesList: Collection<List<*>>, className: String? = null): dynamic {
+fun getChartData(labels: List<String>, valuesList: Collection<List<*>>, classNames: Array<String>? = null): dynamic {
     val chartData: dynamic = object{}
     chartData["labels"] = labels.toTypedArray()
-    chartData["series"] = valuesList.map {
+    chartData["series"] = valuesList.mapIndexed { index, it ->
         val series: dynamic = object{}
         series["data"] = it.toTypedArray()
-        className?.let { series["className"] = className }
+        classNames?.let { series["className"] = classNames[index] }
         series
     }.toTypedArray()
     return chartData
@@ -142,7 +142,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
                         "L", data.x + pointSize, data.y + pointSize/2, "z").joinToString(" ")
                 svgParameters["style"] = "fill:rgb(255,0,0);stroke-width:0"
                 val triangle = Chartist.Svg("path", svgParameters, chartContainer)
-                element = data.element._node.replace(triangle)
+                element = data.element.replace(triangle)
             } else if (currentBuild.buildNumber == parameters["build"]) {
                 // Higlight choosen build.
                 val svgParameters: dynamic = object{}
@@ -228,7 +228,7 @@ fun main(args: Array<String>) {
     val branchesUrl = "$serverUrl/branches/${parameters["target"]}"
 
     val branches: Array<String> = JSON.parse(sendGetRequest(branchesUrl))
-    val releaseBranches = branches.filter { "v\\d+\\.\\d+\\.\\d+-fixes".toRegex().find(it) != null }
+    val releaseBranches = branches.filter { "^v\\d+\\.\\d+\\.\\d+-fixes$".toRegex().find(it) != null }
 
     // Fill autocomplete list.
     val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}"
@@ -312,17 +312,17 @@ fun main(args: Array<String>) {
         bundleSize.add(it.bundleSize?.toInt()?. let { it / 1024 / 1024 })
     }
 
-    val sizeClassName = "ct-series-c"
+    val sizeClassNames = arrayOf("ct-series-d", "ct-series-e")
 
     // Draw charts.
     val execChart = Chartist.Line("#exec_chart", getChartData(labels, executionTime.values),
             getChartOptions(executionTime.keys.toTypedArray(), "Normalized time"))
     val compileChart = Chartist.Line("#compile_chart", getChartData(labels, compileTime.values),
             getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds"))
-    val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassName),
+    val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassNames),
-            getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-2")))
+            getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-3", "ct-series-4")))
-    val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassName),
+    val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassNames),
-            getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-2")))
+            getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-3")))
 
     // Tooltips and higlights.
     customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)