Compile only benchmarks: helloWorld and videoplayer (#2703)

performance/build.gradle

@@ -42,24 +42,27 @@ defaultTasks 'konanRun'
 
 // Produce and send slack report.
 task slackReport(type: RegressionsReporter) {
-    // Create folder for report (root Kotlin project settings make create report in separate folder).
+    def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")
-    def reportDirectory = new File(outputReport).parentFile
+    if (teamcityConfig) {
-    mkdir reportDirectory
+        // Create folder for report (root Kotlin project settings make create report in separate folder).
-    def targetsResults = new File(new File("${rootBuildDirectory}"), "targetsResults").toString()
+        def reportDirectory = new File(outputReport).parentFile
-    mkdir targetsResults
+        mkdir reportDirectory
-    currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
+        def targetsResults = new File(new File("${rootBuildDirectory}"), "targetsResults").toString()
-    analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
+        mkdir targetsResults
-            "${rootBuildDirectory}/${analyzerToolDirectory}")
+        currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
-    htmlReport = outputReport
+        analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
-    defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
+                "${rootBuildDirectory}/${analyzerToolDirectory}")
-    def target = System.getProperty("os.name")
+        htmlReport = outputReport
-    summaryFile = "${targetsResults}/${target}.txt"
+        defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
+        def target = System.getProperty("os.name")
+        summaryFile = "${targetsResults}/${target}.txt"
+    }
 }
 
 task slackSummary(type: RegressionsSummaryReporter) {
-    targetsResultFiles = ["Linux": "${rootBuildDirectory}/targetsResults/Linux.txt",
+    targetsResultFiles = ['Linux': "${rootBuildDirectory}/targetsResults/Linux.txt".toString(),
-                          "MacOSX": "${rootBuildDirectory}/targetsResults/Mac OS X.txt",
+                          'MacOSX': "${rootBuildDirectory}/targetsResults/Mac OS X.txt".toString(),
-                          "Windows": "${rootBuildDirectory}/targetsResults/Windows 10.txt"]
+                          'Windows': "${rootBuildDirectory}/targetsResults/Windows 10.txt".toString()]
 }
 
 private def uploadBenchmarkResultToBintray(String fileName) {

performance/buildSrc/src/main/kotlin/MPPTools.kt

@@ -104,8 +104,8 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
     val benchDesc = getValue("benchmarks")
     val benchmarksArray = JsonTreeParser.parse(benchDesc)
     val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray)
-            .union(listOf<BenchmarkResult>(projectProperties["compileTime"] as BenchmarkResult,
+            .union(projectProperties["compileTime"] as List<BenchmarkResult>).union(
-                                            projectProperties["codeSize"] as BenchmarkResult)).toList()
+                    listOf(projectProperties["codeSize"] as BenchmarkResult)).toList()
     val report = BenchmarksReport(env, benchmarks, kotlin)
     return report.toJson()
 }
@@ -175,6 +175,19 @@ fun getJvmCompileTime(programName: String): BenchmarkResult =
 fun getNativeCompileTime(programName: String): BenchmarkResult =
         TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative"))
 
+fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable<String>, repeats: Int, exitCodes: Map<String, Int>) =
+    (1..repeats).map { number ->
+        var time = 0.0
+        var status = BenchmarkResult.Status.PASSED
+        tasksNames.forEach {
+            time += TaskTimerListener.getTime("$it$number")
+            status = if (exitCodes["$it$number"] != 0) BenchmarkResult.Status.FAILED else status
+        }
+
+        BenchmarkResult("$programName.compileTime", status, time, time, number, 0)
+    }.toList()
+
+
 // Class time tracker for all tasks.
 class TaskTimerListener: TaskExecutionListener {
     companion object {
@@ -189,6 +202,7 @@ class TaskTimerListener: TaskExecutionListener {
                                     time, time, 1, 0)
         }
 
+        fun getTime(taskName: String) = tasksTimes[taskName] ?: 0.0
     }
 
     private var startTime = System.currentTimeMillis()

performance/cinterop/build.gradle

@@ -5,7 +5,7 @@ project.ext {
     nativeSrcDirs = ['src/main/kotlin-native', '../shared/src/main/kotlin-native']
 
 }
-apply from: rootProject.file('gradle/benchmark.gradle')
+apply from: rootProject.file("${rootProject.rootDir}/gradle/benchmark.gradle")
 kotlin.targets.native.compilations.main.cinterops { 
     macros
     struct

performance/gradle/benchmark.gradle

@@ -96,7 +96,7 @@ task konanJsonReport {
                                                   'compilerVersion': "${konanVersion}".toString(),
                                                   'flags': kotlin.targets.native.compilations.main.extraOpts.collect{ "\"$it\"" },
                                                   'benchmarks': benchContents,
-                                                  'compileTime': nativeCompileTime,
+                                                  'compileTime': [nativeCompileTime],
                                                   'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, nativeExecutable) ]
         def output = MPPTools.createJsonReport(properties)
         new File("${buildDir.absolutePath}/${nativeJson}").write(output)
@@ -111,7 +111,7 @@ task jvmJsonReport {
         def properties = getCommonProperties() + ['type': 'jvm',
                                                   'compilerVersion': "${buildKotlinVersion}".toString(),
                                                   'benchmarks': benchContents,
-                                                  'compileTime': jvmCompileTime,
+                                                  'compileTime': [jvmCompileTime],
                                                   'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, "${jarPath}") ]
         def output = MPPTools.createJsonReport(properties)
         new File("${buildDir.absolutePath}/${jvmJson}").write(output)

performance/gradle/compileBenchmark.gradle

@@ -0,0 +1,67 @@
+def exitCodes = [:]
+
+MPPTools.addTimeListener(project)
+
+
+task konanRun {
+    mkdir buildDir.absolutePath
+    (1..project.ext.repeatNumber).each { number ->
+        project.ext.buildSteps.keySet().each {
+            dependsOn "$it$number"
+        }
+    }
+}
+
+(1..project.ext.repeatNumber).each { number ->
+    project.ext.buildSteps.each { taskName, command ->
+        tasks.create(name: "$taskName$number", type: Exec) {
+            commandLine command
+            ignoreExitValue true
+            doLast {
+                exitCodes[name] = execResult.exitValue
+            }
+        }
+    }
+}
+
+task clean {
+    doLast {
+        delete "${buildDir.absolutePath}"
+    }
+}
+
+task konanJsonReport {
+    doLast {
+        def nativeCompileTime = MPPTools.getCompileBenchmarkTime(project.ext.applicationName, project.ext.buildSteps.keySet(),
+                project.ext.repeatNumber, exitCodes)
+        def nativeExecutable = "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}"
+        def properties = getCommonProperties() + ['type': 'native',
+                                                  'compilerVersion': "${konanVersion}".toString(),
+                                                  'benchmarks': "[]",
+                                                  'compileTime': nativeCompileTime,
+                                                  'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, nativeExecutable) ]
+        def output = MPPTools.createJsonReport(properties)
+        new File("${buildDir.absolutePath}/${nativeJson}").write(output)
+    }
+}
+task jvmRun {
+    println("JVM run isn't supported")
+}
+
+task jvmJsonReport {
+    doLast {
+        println("JVM run isn't supported")
+    }
+}
+
+jvmRun.finalizedBy jvmJsonReport
+konanRun.finalizedBy konanJsonReport
+
+private def getCommonProperties() {
+    return ['cpu': System.getProperty("os.arch"),
+            'os': System.getProperty("os.name"), // OperatingSystem.current().getName()
+            'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion
+            'jdkVendor': System.getProperty("java.vendor"),
+            'kotlinVersion': "${kotlinVersion}".toString()]
+}
+

performance/helloworld/build.gradle

@@ -0,0 +1,8 @@
+def dist = findProperty('org.jetbrains.kotlin.native.home') ?: 'dist'
+project.ext {
+    applicationName = 'HelloWorld'
+    repeatNumber = 10
+    buildSteps = ["runKonanc": ["${project.getProjectDir()}/$dist/bin/konanc", "${project.getProjectDir()}/src/main/kotlin/main.kt", "-o",
+                                "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}"]]
+}
+apply from: rootProject.file("${rootProject.rootDir}/gradle/compileBenchmark.gradle")

performance/helloworld/gradle.properties

@@ -0,0 +1 @@
+org.jetbrains.kotlin.native.home=../../dist

performance/helloworld/src/main/kotlin/main.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+fun main() {
+    println("Hello, World!")
+}

performance/ring/build.gradle

@@ -5,4 +5,4 @@ project.ext {
     nativeSrcDirs = ['src/main/kotlin-native', '../shared/src/main/kotlin-native']
 
 }
-apply from: rootProject.file('gradle/benchmark.gradle')
+apply from: rootProject.file("${rootProject.rootDir}/gradle/benchmark.gradle")

performance/settings.gradle

@@ -1,2 +1,4 @@
 include ':ring'
 include ':cinterop'
+include ':helloworld'
+include ':videoplayer'

performance/videoplayer/build.gradle

@@ -0,0 +1,48 @@
+def dist = findProperty('org.jetbrains.kotlin.native.home') ?: 'dist'
+
+def linkerOpts = []
+if (MPPTools.isMacos()) {
+    linkerOpts += ['-linker-options','-L/opt/local/lib', '-linker-options', '-L/usr/local/lib']
+} else if (MPPTools.isLinux()) {
+    linkerOpts += ['-linker-options', '-L/usr/lib/x86_64-linux-gnu', '-linker-options', '-L/usr/lib64']
+} else if (MPPTools.isWindows()) {
+    linkerOpts += ['-linker-options', "-L${MPPTools.mingwPath()}/lib"]
+}
+
+def includeDirsFfmpeg = []
+def filterDirsFfmpeg = []
+if (MPPTools.isMacos()) {
+    filterDirsFfmpeg += ['-headerFilterAdditionalSearchPrefix', '/opt/local/include',
+                          '-headerFilterAdditionalSearchPrefix', '/usr/local/include']
+} else if (MPPTools.isLinux()) {
+    filterDirsFfmpeg += ['-headerFilterAdditionalSearchPrefix', '/usr/include',
+                          '-headerFilterAdditionalSearchPrefix', '/usr/include/x86_64-linux-gnu',
+                          '-headerFilterAdditionalSearchPrefix', '/usr/include/ffmpeg']
+} else if (MPPTools.isWindows()) {
+    includeDirsFfmpeg += ['-copt', "-I${MPPTools.mingwPath()}/include"]
+}
+
+def includeDirsSdl = []
+if (MPPTools.isMacos()) {
+    includeDirsSdl += ['-copt', '-I/opt/local/include/SDL2',
+                       '-copt', '-I/usr/local/include/SDL2']
+} else if (MPPTools.isLinux()) {
+    includeDirsSdl += ['-copt', '-I/usr/include/SDL2']
+} else if (MPPTools.isWindows()) {
+    includeDirsSdl += ['-copt', "-I${MPPTools.mingwPath()}/include/SDL2"]
+}
+
+project.ext {
+    applicationName = 'Videoplayer'
+    repeatNumber = 2
+    buildSteps = ["runCinteropFfmpeg": ["$dist/bin/cinterop", "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib",
+                                "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/ffmpeg.def"] + filterDirsFfmpeg + includeDirsFfmpeg,
+                  "runCinteropSdl": ["$dist/bin/cinterop", "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib",
+                                     "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/sdl.def"] + includeDirsSdl,
+                  "runKonanProgram": ["$dist/bin/konanc", "-ea", "-p", "program", "-o", "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}",
+                       "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib",
+                       "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib",
+                       "-Xmulti-platform", "$dist/../samples/videoplayer/src/videoPlayerMain/kotlin",
+                       "-entry", "sample.videoplayer.main"] + linkerOpts]
+}
+apply from: rootProject.file("${rootProject.rootDir}/gradle/compileBenchmark.gradle")

performance/videoplayer/gradle.properties

@@ -0,0 +1 @@
+org.jetbrains.kotlin.native.home=../../dist

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt

@@ -555,6 +555,8 @@ class HTMLRender: Render() {
     private fun TableBlock.renderBenchmarksDetails(fullSet: Map<String, SummaryBenchmark>,
                                                    bucket: Map<String, ScoreChange>? = null, rowStyle: String? = null) {
         if (bucket != null) {
+            // Find max ratio.
+            val maxRatio = bucket.values.map { it.second.mean }.max()!!
             // There are changes in performance.
             // Output changed benchmarks.
             for ((name, change) in bucket) {
@@ -571,9 +573,7 @@ class HTMLRender: Render() {
                         +"${change.first.toString() + " %"}"
                     }
                     td {
-                        val scaledRatio = if (change.first.mean < 0)
+                        val scaledRatio = change.second.mean / maxRatio
-                                            (1 - change.second.mean) / (-1 + bucket.values.first().second.mean)
-                                            else (change.second.mean / bucket.values.first().second.mean)
                         attributes["bgcolor"] = ColoredCell(scaledRatio).backgroundStyle
                         +"${change.second}"
                     }