diff --git a/performance/build.gradle b/performance/build.gradle index 906f5d89247..9a792ab15c9 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -42,24 +42,27 @@ defaultTasks 'konanRun' // Produce and send slack report. task slackReport(type: RegressionsReporter) { - // Create folder for report (root Kotlin project settings make create report in separate folder). - def reportDirectory = new File(outputReport).parentFile - mkdir reportDirectory - def targetsResults = new File(new File("${rootBuildDirectory}"), "targetsResults").toString() - mkdir targetsResults - currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}" - analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}", - "${rootBuildDirectory}/${analyzerToolDirectory}") - htmlReport = outputReport - defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master" - def target = System.getProperty("os.name") - summaryFile = "${targetsResults}/${target}.txt" + def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") + if (teamcityConfig) { + // Create folder for report (root Kotlin project settings make create report in separate folder). + def reportDirectory = new File(outputReport).parentFile + mkdir reportDirectory + def targetsResults = new File(new File("${rootBuildDirectory}"), "targetsResults").toString() + mkdir targetsResults + currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}" + analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}", + "${rootBuildDirectory}/${analyzerToolDirectory}") + htmlReport = outputReport + defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master" + def target = System.getProperty("os.name") + summaryFile = "${targetsResults}/${target}.txt" + } } task slackSummary(type: RegressionsSummaryReporter) { - targetsResultFiles = ["Linux": "${rootBuildDirectory}/targetsResults/Linux.txt", - "MacOSX": "${rootBuildDirectory}/targetsResults/Mac OS X.txt", - "Windows": "${rootBuildDirectory}/targetsResults/Windows 10.txt"] + targetsResultFiles = ['Linux': "${rootBuildDirectory}/targetsResults/Linux.txt".toString(), + 'MacOSX': "${rootBuildDirectory}/targetsResults/Mac OS X.txt".toString(), + 'Windows': "${rootBuildDirectory}/targetsResults/Windows 10.txt".toString()] } private def uploadBenchmarkResultToBintray(String fileName) { diff --git a/performance/buildSrc/src/main/kotlin/MPPTools.kt b/performance/buildSrc/src/main/kotlin/MPPTools.kt index 38eff12965e..6050190d188 100644 --- a/performance/buildSrc/src/main/kotlin/MPPTools.kt +++ b/performance/buildSrc/src/main/kotlin/MPPTools.kt @@ -104,8 +104,8 @@ fun createJsonReport(projectProperties: Map): String { val benchDesc = getValue("benchmarks") val benchmarksArray = JsonTreeParser.parse(benchDesc) val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray) - .union(listOf(projectProperties["compileTime"] as BenchmarkResult, - projectProperties["codeSize"] as BenchmarkResult)).toList() + .union(projectProperties["compileTime"] as List).union( + listOf(projectProperties["codeSize"] as BenchmarkResult)).toList() val report = BenchmarksReport(env, benchmarks, kotlin) return report.toJson() } @@ -175,6 +175,19 @@ fun getJvmCompileTime(programName: String): BenchmarkResult = fun getNativeCompileTime(programName: String): BenchmarkResult = TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative")) +fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable, repeats: Int, exitCodes: Map) = + (1..repeats).map { number -> + var time = 0.0 + var status = BenchmarkResult.Status.PASSED + tasksNames.forEach { + time += TaskTimerListener.getTime("$it$number") + status = if (exitCodes["$it$number"] != 0) BenchmarkResult.Status.FAILED else status + } + + BenchmarkResult("$programName.compileTime", status, time, time, number, 0) + }.toList() + + // Class time tracker for all tasks. class TaskTimerListener: TaskExecutionListener { companion object { @@ -189,6 +202,7 @@ class TaskTimerListener: TaskExecutionListener { time, time, 1, 0) } + fun getTime(taskName: String) = tasksTimes[taskName] ?: 0.0 } private var startTime = System.currentTimeMillis() diff --git a/performance/cinterop/build.gradle b/performance/cinterop/build.gradle index ec6b88e2243..e804c59d6cf 100644 --- a/performance/cinterop/build.gradle +++ b/performance/cinterop/build.gradle @@ -5,7 +5,7 @@ project.ext { nativeSrcDirs = ['src/main/kotlin-native', '../shared/src/main/kotlin-native'] } -apply from: rootProject.file('gradle/benchmark.gradle') +apply from: rootProject.file("${rootProject.rootDir}/gradle/benchmark.gradle") kotlin.targets.native.compilations.main.cinterops { macros struct diff --git a/performance/gradle/benchmark.gradle b/performance/gradle/benchmark.gradle index 89ddd1b1229..00616a0ea46 100644 --- a/performance/gradle/benchmark.gradle +++ b/performance/gradle/benchmark.gradle @@ -96,7 +96,7 @@ task konanJsonReport { 'compilerVersion': "${konanVersion}".toString(), 'flags': kotlin.targets.native.compilations.main.extraOpts.collect{ "\"$it\"" }, 'benchmarks': benchContents, - 'compileTime': nativeCompileTime, + 'compileTime': [nativeCompileTime], 'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, nativeExecutable) ] def output = MPPTools.createJsonReport(properties) new File("${buildDir.absolutePath}/${nativeJson}").write(output) @@ -111,7 +111,7 @@ task jvmJsonReport { def properties = getCommonProperties() + ['type': 'jvm', 'compilerVersion': "${buildKotlinVersion}".toString(), 'benchmarks': benchContents, - 'compileTime': jvmCompileTime, + 'compileTime': [jvmCompileTime], 'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, "${jarPath}") ] def output = MPPTools.createJsonReport(properties) new File("${buildDir.absolutePath}/${jvmJson}").write(output) diff --git a/performance/gradle/compileBenchmark.gradle b/performance/gradle/compileBenchmark.gradle new file mode 100644 index 00000000000..cbc7500ea8a --- /dev/null +++ b/performance/gradle/compileBenchmark.gradle @@ -0,0 +1,67 @@ +def exitCodes = [:] + +MPPTools.addTimeListener(project) + + +task konanRun { + mkdir buildDir.absolutePath + (1..project.ext.repeatNumber).each { number -> + project.ext.buildSteps.keySet().each { + dependsOn "$it$number" + } + } +} + +(1..project.ext.repeatNumber).each { number -> + project.ext.buildSteps.each { taskName, command -> + tasks.create(name: "$taskName$number", type: Exec) { + commandLine command + ignoreExitValue true + doLast { + exitCodes[name] = execResult.exitValue + } + } + } +} + +task clean { + doLast { + delete "${buildDir.absolutePath}" + } +} + +task konanJsonReport { + doLast { + def nativeCompileTime = MPPTools.getCompileBenchmarkTime(project.ext.applicationName, project.ext.buildSteps.keySet(), + project.ext.repeatNumber, exitCodes) + def nativeExecutable = "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}" + def properties = getCommonProperties() + ['type': 'native', + 'compilerVersion': "${konanVersion}".toString(), + 'benchmarks': "[]", + 'compileTime': nativeCompileTime, + 'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, nativeExecutable) ] + def output = MPPTools.createJsonReport(properties) + new File("${buildDir.absolutePath}/${nativeJson}").write(output) + } +} +task jvmRun { + println("JVM run isn't supported") +} + +task jvmJsonReport { + doLast { + println("JVM run isn't supported") + } +} + +jvmRun.finalizedBy jvmJsonReport +konanRun.finalizedBy konanJsonReport + +private def getCommonProperties() { + return ['cpu': System.getProperty("os.arch"), + 'os': System.getProperty("os.name"), // OperatingSystem.current().getName() + 'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion + 'jdkVendor': System.getProperty("java.vendor"), + 'kotlinVersion': "${kotlinVersion}".toString()] +} + diff --git a/performance/helloworld/build.gradle b/performance/helloworld/build.gradle new file mode 100644 index 00000000000..982a98f36b3 --- /dev/null +++ b/performance/helloworld/build.gradle @@ -0,0 +1,8 @@ +def dist = findProperty('org.jetbrains.kotlin.native.home') ?: 'dist' +project.ext { + applicationName = 'HelloWorld' + repeatNumber = 10 + buildSteps = ["runKonanc": ["${project.getProjectDir()}/$dist/bin/konanc", "${project.getProjectDir()}/src/main/kotlin/main.kt", "-o", + "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}"]] +} +apply from: rootProject.file("${rootProject.rootDir}/gradle/compileBenchmark.gradle") \ No newline at end of file diff --git a/performance/helloworld/gradle.properties b/performance/helloworld/gradle.properties new file mode 100644 index 00000000000..87803bed88a --- /dev/null +++ b/performance/helloworld/gradle.properties @@ -0,0 +1 @@ +org.jetbrains.kotlin.native.home=../../dist \ No newline at end of file diff --git a/performance/helloworld/src/main/kotlin/main.kt b/performance/helloworld/src/main/kotlin/main.kt new file mode 100644 index 00000000000..d41a74f9a74 --- /dev/null +++ b/performance/helloworld/src/main/kotlin/main.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +fun main() { + println("Hello, World!") +} \ No newline at end of file diff --git a/performance/ring/build.gradle b/performance/ring/build.gradle index 4c90f483e75..2af6e891f53 100644 --- a/performance/ring/build.gradle +++ b/performance/ring/build.gradle @@ -5,4 +5,4 @@ project.ext { nativeSrcDirs = ['src/main/kotlin-native', '../shared/src/main/kotlin-native'] } -apply from: rootProject.file('gradle/benchmark.gradle') \ No newline at end of file +apply from: rootProject.file("${rootProject.rootDir}/gradle/benchmark.gradle") \ No newline at end of file diff --git a/performance/settings.gradle b/performance/settings.gradle index 97fe54cc991..89a03c33563 100644 --- a/performance/settings.gradle +++ b/performance/settings.gradle @@ -1,2 +1,4 @@ include ':ring' -include ':cinterop' \ No newline at end of file +include ':cinterop' +include ':helloworld' +include ':videoplayer' \ No newline at end of file diff --git a/performance/videoplayer/build.gradle b/performance/videoplayer/build.gradle new file mode 100644 index 00000000000..e4228d9f72c --- /dev/null +++ b/performance/videoplayer/build.gradle @@ -0,0 +1,48 @@ +def dist = findProperty('org.jetbrains.kotlin.native.home') ?: 'dist' + +def linkerOpts = [] +if (MPPTools.isMacos()) { + linkerOpts += ['-linker-options','-L/opt/local/lib', '-linker-options', '-L/usr/local/lib'] +} else if (MPPTools.isLinux()) { + linkerOpts += ['-linker-options', '-L/usr/lib/x86_64-linux-gnu', '-linker-options', '-L/usr/lib64'] +} else if (MPPTools.isWindows()) { + linkerOpts += ['-linker-options', "-L${MPPTools.mingwPath()}/lib"] +} + +def includeDirsFfmpeg = [] +def filterDirsFfmpeg = [] +if (MPPTools.isMacos()) { + filterDirsFfmpeg += ['-headerFilterAdditionalSearchPrefix', '/opt/local/include', + '-headerFilterAdditionalSearchPrefix', '/usr/local/include'] +} else if (MPPTools.isLinux()) { + filterDirsFfmpeg += ['-headerFilterAdditionalSearchPrefix', '/usr/include', + '-headerFilterAdditionalSearchPrefix', '/usr/include/x86_64-linux-gnu', + '-headerFilterAdditionalSearchPrefix', '/usr/include/ffmpeg'] +} else if (MPPTools.isWindows()) { + includeDirsFfmpeg += ['-copt', "-I${MPPTools.mingwPath()}/include"] +} + +def includeDirsSdl = [] +if (MPPTools.isMacos()) { + includeDirsSdl += ['-copt', '-I/opt/local/include/SDL2', + '-copt', '-I/usr/local/include/SDL2'] +} else if (MPPTools.isLinux()) { + includeDirsSdl += ['-copt', '-I/usr/include/SDL2'] +} else if (MPPTools.isWindows()) { + includeDirsSdl += ['-copt', "-I${MPPTools.mingwPath()}/include/SDL2"] +} + +project.ext { + applicationName = 'Videoplayer' + repeatNumber = 2 + buildSteps = ["runCinteropFfmpeg": ["$dist/bin/cinterop", "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib", + "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/ffmpeg.def"] + filterDirsFfmpeg + includeDirsFfmpeg, + "runCinteropSdl": ["$dist/bin/cinterop", "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib", + "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/sdl.def"] + includeDirsSdl, + "runKonanProgram": ["$dist/bin/konanc", "-ea", "-p", "program", "-o", "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}", + "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib", + "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib", + "-Xmulti-platform", "$dist/../samples/videoplayer/src/videoPlayerMain/kotlin", + "-entry", "sample.videoplayer.main"] + linkerOpts] +} +apply from: rootProject.file("${rootProject.rootDir}/gradle/compileBenchmark.gradle") \ No newline at end of file diff --git a/performance/videoplayer/gradle.properties b/performance/videoplayer/gradle.properties new file mode 100644 index 00000000000..87803bed88a --- /dev/null +++ b/performance/videoplayer/gradle.properties @@ -0,0 +1 @@ +org.jetbrains.kotlin.native.home=../../dist \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt index d6bd4eb939f..e45ff4c0d05 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt @@ -555,6 +555,8 @@ class HTMLRender: Render() { private fun TableBlock.renderBenchmarksDetails(fullSet: Map, bucket: Map? = null, rowStyle: String? = null) { if (bucket != null) { + // Find max ratio. + val maxRatio = bucket.values.map { it.second.mean }.max()!! // There are changes in performance. // Output changed benchmarks. for ((name, change) in bucket) { @@ -571,9 +573,7 @@ class HTMLRender: Render() { +"${change.first.toString() + " %"}" } td { - val scaledRatio = if (change.first.mean < 0) - (1 - change.second.mean) / (-1 + bucket.values.first().second.mean) - else (change.second.mean / bucket.values.first().second.mean) + val scaledRatio = change.second.mean / maxRatio attributes["bgcolor"] = ColoredCell(scaledRatio).backgroundStyle +"${change.second}" }