From f1c7d9a728ab809a7734648a222e56b569d82b4c Mon Sep 17 00:00:00 2001 From: LepilkinaElena Date: Tue, 8 Oct 2019 10:01:14 +0300 Subject: [PATCH] Returned compiler options and made system possible to get reports with different flags (#3424) --- .../kotlin/org/jetbrains/kotlin/MPPTools.kt | 21 ++++++++- .../jetbrains/kotlin/RunKotlinNativeTask.kt | 2 - .../kotlin/benchmark/BenchmarkingPlugin.kt | 26 +++++++++-- .../benchmark/CompileBenchmarkingPlugin.kt | 5 ++ .../KotlinNativeBenchmarkingPlugin.kt | 4 +- .../benchmark/SwiftBenchmarkingPlugin.kt | 10 ++-- performance/cinterop/build.gradle.kts | 5 +- performance/framework/build.gradle | 18 +++++++- performance/helloworld/build.gradle.kts | 6 ++- performance/numerical/build.gradle.kts | 4 ++ performance/objcinterop/build.gradle.kts | 4 ++ performance/ring/build.gradle.kts | 5 ++ performance/swiftinterop/build.gradle.kts | 4 ++ performance/videoplayer/build.gradle.kts | 9 +++- .../org/jetbrains/report/BenchmarksReport.kt | 22 +++++++-- .../src/main/kotlin/main.kt | 46 +++++++++++++++---- 16 files changed, 157 insertions(+), 34 deletions(-) diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt index a5a9c6ccfbb..131c160967c 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt @@ -13,6 +13,7 @@ import org.gradle.api.Task import org.gradle.api.tasks.TaskState import org.gradle.api.execution.TaskExecutionListener import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink import org.jetbrains.report.* import org.jetbrains.report.json.* @@ -115,9 +116,27 @@ fun mergeReports(reports: List): String { val reportElement = JsonTreeParser.parse(json) BenchmarksReport.create(reportElement) } - return if (reportsToMerge.isEmpty()) "" else reportsToMerge.reduce { result, it -> result + it }.toJson() + val structuredReports = mutableMapOf>() + reportsToMerge.map { it.compiler.backend.flags.joinToString() to it }.forEach { + structuredReports.getOrPut(it.first) { mutableListOf() }.add(it.second) + } + val jsons = structuredReports.map { (_, value) -> value.reduce { result, it -> result + it }.toJson() } + return when(jsons.size) { + 0 -> "" + 1 -> jsons[0] + else -> jsons.joinToString(prefix = "[", postfix = "]") + } } +fun getCompileOnlyBenchmarksOpts(project: Project, defaultCompilerOpts: List) = + (project.findProperty("nativeBuildType") as String?)?.let { + if (it.equals("RELEASE", true)) + listOf("-opt") + else if (it.equals("DEBUG", true)) + listOf("-g") + else listOf() + } ?: defaultCompilerOpts + // Find file with set name in directory. fun findFile(fileName: String, directory: String): String? = File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath() diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt index 980cb6cc72a..77683e4b269 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt @@ -19,8 +19,6 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task, private val executable: String, private val outputFileName: String ) : DefaultTask() { - @Input - var buildType = "RELEASE" @Input @Option(option = "filter", description = "Benchmarks to run (comma-separated)") var filter: String = "" diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt index 65091dc1aa5..9b092ca02c1 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt @@ -11,6 +11,7 @@ import org.jetbrains.kotlin.* import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinNativeTargetPreset +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.HostManager import javax.inject.Inject import kotlin.reflect.KClass @@ -30,6 +31,7 @@ internal val Project.attempts: Int internal val Project.nativeBenchResults: String get() = property("nativeBenchResults") as String +// Gradle property to add flags to benchmarks run from command line. internal val Project.compilerArgs: List get() = (findProperty("compilerArgs") as String?)?.split("\\s").orEmpty() @@ -66,6 +68,8 @@ open class BenchmarkExtension @Inject constructor(val project: Project) { var nativeSrcDirs: Collection = emptyList() var compileTasks: List = emptyList() var linkerOpts: Collection = emptyList() + var compilerOpts: List = emptyList() + var buildType: NativeBuildType = NativeBuildType.RELEASE val dependencies: BenchmarkDependencies = BenchmarkDependencies() @@ -141,7 +145,7 @@ abstract class BenchmarkingPlugin: Plugin { } protected open fun KotlinNativeTarget.configureNativeOutput(project: Project) { - binaries.executable(NATIVE_EXECUTABLE_NAME, listOf(RELEASE)) { + binaries.executable(NATIVE_EXECUTABLE_NAME, listOf(project.benchmark.buildType)) { if (HostManager.hostIsMingw) { linkerOpts.add("-L${mingwPath}/lib") } @@ -154,13 +158,14 @@ abstract class BenchmarkingPlugin: Plugin { // Specify settings configured by a user in the benchmark extension. project.afterEvaluate { linkerOpts.addAll(project.benchmark.linkerOpts) + freeCompilerArgs = project.benchmark.compilerOpts + project.compilerArgs } } } protected fun Project.configureNativeTarget(hostPreset: AbstractKotlinNativeTargetPreset<*>) { kotlin.targetFromPreset(hostPreset, NATIVE_TARGET_NAME) { - compilations.getByName("main").kotlinOptions.freeCompilerArgs = project.compilerArgs + compilations.getByName("main").kotlinOptions.freeCompilerArgs = benchmark.compilerOpts + project.compilerArgs compilations.getByName("main").enableEndorsedLibs = true configureNativeOutput(this@configureNativeTarget) } @@ -189,8 +194,19 @@ abstract class BenchmarkingPlugin: Plugin { protected abstract fun Project.configureJvmTask(): Task - protected open fun Project.getCompilerFlags(nativeTarget: KotlinNativeTarget) = - nativeTarget.compilations.main.kotlinOptions.freeCompilerArgs.map { "\"$it\"" } + protected fun compilerFlagsFromBinary(project: Project): List { + val result = mutableListOf() + if (project.benchmark.buildType.optimized) { + result.add("-opt") + } + if (project.benchmark.buildType.debuggable) { + result.add("-g") + } + return result + } + + protected open fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) = + compilerFlagsFromBinary(project) + nativeTarget.compilations.main.kotlinOptions.freeCompilerArgs.map { "\"$it\"" } protected open fun Project.collectCodeSize(applicationName: String) = getCodeSizeBenchmark(applicationName, nativeExecutable) @@ -209,7 +225,7 @@ abstract class BenchmarkingPlugin: Plugin { val properties = commonBenchmarkProperties + mapOf( "type" to "native", "compilerVersion" to konanVersion, - "flags" to getCompilerFlags(nativeTarget), + "flags" to getCompilerFlags(project, nativeTarget).sorted(), "benchmarks" to benchContents, "compileTime" to listOf(nativeCompileTime), "codeSize" to collectCodeSize(applicationName) diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/CompileBenchmarkingPlugin.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/CompileBenchmarkingPlugin.kt index bd10b9481bb..3df6e444b7a 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/CompileBenchmarkingPlugin.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/CompileBenchmarkingPlugin.kt @@ -29,6 +29,7 @@ open class CompileBenchmarkExtension @Inject constructor(val project: Project) { var applicationName = project.name var repeatNumber: Int = 1 var buildSteps: BuildStepContainer = BuildStepContainer(project) + var compilerOpts: List = emptyList() fun buildSteps(configure: Action): Unit = buildSteps.let { configure.execute(it) } fun buildSteps(configure: Closure): Unit = buildSteps(ConfigureUtil.configureUsing(configure)) @@ -94,6 +95,7 @@ open class CompileBenchmarkingPlugin : Plugin { "type" to "native", "compilerVersion" to konanVersion, "benchmarks" to "[]", + "flags" to getCompilerFlags(benchmarkExtension).sorted(), "compileTime" to nativeCompileTime, "codeSize" to getCodeSizeBenchmark(applicationName, nativeExecutable.absolutePath) ) @@ -104,6 +106,9 @@ open class CompileBenchmarkingPlugin : Plugin { } } + private fun getCompilerFlags(benchmarkExtension: CompileBenchmarkExtension) = + benchmarkExtension.compilerOpts + private fun Project.configureJvmRun( benchmarkExtension: CompileBenchmarkExtension ) { diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/KotlinNativeBenchmarkingPlugin.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/KotlinNativeBenchmarkingPlugin.kt index 18876c80b74..dc21ff8fb65 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/KotlinNativeBenchmarkingPlugin.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/KotlinNativeBenchmarkingPlugin.kt @@ -96,7 +96,7 @@ open class KotlinNativeBenchmarkingPlugin: BenchmarkingPlugin() { private val Project.nativeBinary: Executable get() = (kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget) - .binaries.getExecutable(NATIVE_EXECUTABLE_NAME, NativeBuildType.RELEASE) + .binaries.getExecutable(NATIVE_EXECUTABLE_NAME, benchmark.buildType) override val Project.nativeExecutable: String get() = nativeBinary.outputFile.absolutePath @@ -133,7 +133,7 @@ open class KotlinNativeBenchmarkingPlugin: BenchmarkingPlugin() { it.compileKotlinTask.kotlinOptions { jvmTarget = "1.8" suppressWarnings = true - freeCompilerArgs = project.compilerArgs + freeCompilerArgs = project.benchmark.compilerOpts + project.compilerArgs } } } diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/SwiftBenchmarkingPlugin.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/SwiftBenchmarkingPlugin.kt index 5a727e91bf9..3604e0a9386 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/SwiftBenchmarkingPlugin.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/SwiftBenchmarkingPlugin.kt @@ -67,7 +67,7 @@ open class SwiftBenchmarkingPlugin : BenchmarkingPlugin() { override fun Project.determinePreset(): AbstractKotlinNativeTargetPreset<*> = kotlin.presets.macosX64 as AbstractKotlinNativeTargetPreset<*> override fun KotlinNativeTarget.configureNativeOutput(project: Project) { - binaries.framework(nativeFrameworkName, listOf(RELEASE)) { + binaries.framework(nativeFrameworkName, listOf(project.benchmark.buildType)) { // Specify settings configured by a user in the benchmark extension. project.afterEvaluate { linkerOpts.addAll(project.benchmark.linkerOpts) @@ -78,7 +78,7 @@ open class SwiftBenchmarkingPlugin : BenchmarkingPlugin() { override fun Project.configureExtraTasks() { val nativeTarget = kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget // Build executable from swift code. - framework = nativeTarget.binaries.getFramework(nativeFrameworkName, NativeBuildType.RELEASE) + framework = nativeTarget.binaries.getFramework(nativeFrameworkName, benchmark.buildType) val buildSwift = tasks.create("buildSwift") { task -> task.dependsOn(framework.linkTaskName) task.doLast { @@ -98,9 +98,9 @@ open class SwiftBenchmarkingPlugin : BenchmarkingPlugin() { nativeExecutable ) - override fun Project.getCompilerFlags(nativeTarget: KotlinNativeTarget) = - if (benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK) { - nativeTarget.compilations.main.kotlinOptions.freeCompilerArgs.map { "\"$it\"" } + override fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) = + if (project.benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK) { + super.getCompilerFlags(project, nativeTarget) } else { listOf("-O", "-wmo") } diff --git a/performance/cinterop/build.gradle.kts b/performance/cinterop/build.gradle.kts index 031ba1f2725..f325edf12fc 100644 --- a/performance/cinterop/build.gradle.kts +++ b/performance/cinterop/build.gradle.kts @@ -4,11 +4,14 @@ */ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType plugins { id("benchmarking") } +val defaultBuildType = NativeBuildType.RELEASE + benchmark { applicationName = "Cinterop" commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") @@ -16,7 +19,7 @@ benchmark { nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix") - + buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType dependencies.common(project(":endorsedLibraries:kotlinx.cli")) } diff --git a/performance/framework/build.gradle b/performance/framework/build.gradle index 5783a56f418..c9a71d5318e 100644 --- a/performance/framework/build.gradle +++ b/performance/framework/build.gradle @@ -1,6 +1,7 @@ import org.jetbrains.kotlin.MPPTools import org.jetbrains.kotlin.PlatformInfo import org.jetbrains.kotlin.RunJvmTask +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType buildscript { apply from: "$rootProject.projectDir/gradle/kotlinGradlePlugin.gradle" @@ -29,6 +30,8 @@ repositories { def toolsPath = '../../tools' def frameworkName = 'benchmarksAnalyzer' +def buildType = NativeBuildType.valueOf(findProperty('nativeBuildType') ?: 'DEBUG') + kotlin { sourceSets { macosMain { @@ -52,7 +55,7 @@ kotlin { } macosX64("macos").binaries { - framework(frameworkName, [DEBUG]) + framework(frameworkName, [buildType]) } } @@ -70,6 +73,17 @@ task jvmRun(type: RunJvmTask) { } } +def compilerFlags(def buildType) { + def result = [] + if (buildType.optimized) { + result.add("-opt") + } + if (buildType.debuggable) { + result.add("-g") + } + return result +} + task konanJsonReport { doLast { if (PlatformInfo.isMac()) { @@ -82,7 +96,7 @@ task konanJsonReport { 'cinteropLibcurlMacos']) def properties = getCommonProperties() + ['type' : 'native', 'compilerVersion': "${konanVersion}".toString(), - 'flags' : [], + 'flags' : compilerFlags(buildType).sort(), 'benchmarks' : '[]', 'compileTime' : [nativeCompileTime], 'codeSize' : MPPTools.getCodeSizeBenchmark(applicationName, nativeExecutable)] diff --git a/performance/helloworld/build.gradle.kts b/performance/helloworld/build.gradle.kts index 76fca5c3bda..17341c935f2 100644 --- a/performance/helloworld/build.gradle.kts +++ b/performance/helloworld/build.gradle.kts @@ -4,6 +4,7 @@ */ import org.jetbrains.kotlin.getNativeProgramExtension +import org.jetbrains.kotlin.getCompileOnlyBenchmarksOpts plugins { id("compile-benchmarking") @@ -12,13 +13,16 @@ plugins { val dist = file(findProperty("org.jetbrains.kotlin.native.home") ?: "dist") val toolSuffix = if (System.getProperty("os.name").startsWith("Windows")) ".bat" else "" val binarySuffix = getNativeProgramExtension() +val defaultCompilerOpts = listOf("-g") +val buildOpts = getCompileOnlyBenchmarksOpts(project, defaultCompilerOpts) compileBenchmark { applicationName = "HelloWorld" repeatNumber = 10 + compilerOpts = buildOpts buildSteps { step("runKonanc") { - command("$dist/bin/konanc$toolSuffix", "$projectDir/src/main/kotlin/main.kt", "-g", "-o", "$buildDir/program$binarySuffix") + command("$dist/bin/konanc$toolSuffix", "$projectDir/src/main/kotlin/main.kt", "-o", "$buildDir/program$binarySuffix", *(buildOpts.toTypedArray())) } } } diff --git a/performance/numerical/build.gradle.kts b/performance/numerical/build.gradle.kts index 781eb33dd16..9fd84a31c68 100644 --- a/performance/numerical/build.gradle.kts +++ b/performance/numerical/build.gradle.kts @@ -6,12 +6,15 @@ import org.jetbrains.kotlin.benchmark.BenchmarkingPlugin import org.jetbrains.kotlin.ExecClang import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.HostManager plugins { id("benchmarking") } +val defaultBuildType = NativeBuildType.RELEASE + benchmark { applicationName = "Numerical" commonSrcDirs = listOf("src/main/kotlin", "../../tools/benchmarks/shared/src", "../shared/src/main/kotlin") @@ -20,6 +23,7 @@ benchmark { mingwSrcDirs = listOf("../shared/src/main/kotlin-native/mingw") posixSrcDirs = listOf("../shared/src/main/kotlin-native/posix") linkerOpts = listOf("$buildDir/pi.o") + buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType dependencies.common(project(":endorsedLibraries:kotlinx.cli")) } diff --git a/performance/objcinterop/build.gradle.kts b/performance/objcinterop/build.gradle.kts index f01dc398c84..62bea4337c8 100644 --- a/performance/objcinterop/build.gradle.kts +++ b/performance/objcinterop/build.gradle.kts @@ -6,12 +6,15 @@ import org.jetbrains.kotlin.benchmark.BenchmarkingPlugin import org.jetbrains.kotlin.ExecClang import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.HostManager plugins { id("benchmarking") } +val defaultBuildType = NativeBuildType.RELEASE + benchmark { applicationName = "ObjCInterop" commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") @@ -20,6 +23,7 @@ benchmark { mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix") linkerOpts = listOf("-L$buildDir", "-lcomplexnumbers") + buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType dependencies.common(project(":endorsedLibraries:kotlinx.cli")) } diff --git a/performance/ring/build.gradle.kts b/performance/ring/build.gradle.kts index f866db108f2..f73da76453e 100644 --- a/performance/ring/build.gradle.kts +++ b/performance/ring/build.gradle.kts @@ -1,3 +1,5 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType + /* * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. @@ -7,6 +9,8 @@ plugins { id("benchmarking") } +val defaultBuildType = NativeBuildType.RELEASE + benchmark { applicationName = "Ring" commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") @@ -14,6 +18,7 @@ benchmark { nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix") + buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType dependencies.common(project(":endorsedLibraries:kotlinx.cli")) } diff --git a/performance/swiftinterop/build.gradle.kts b/performance/swiftinterop/build.gradle.kts index f9961be47a0..70bc0ac11e6 100644 --- a/performance/swiftinterop/build.gradle.kts +++ b/performance/swiftinterop/build.gradle.kts @@ -1,3 +1,5 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType + /* * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. @@ -9,6 +11,7 @@ plugins { val toolsPath = "../../tools" val targetExtension = "Macos" +val defaultBuildType = NativeBuildType.RELEASE swiftBenchmark { applicationName = "swiftInterop" @@ -16,6 +19,7 @@ swiftBenchmark { nativeSrcDirs = listOf("../shared/src/main/kotlin-native/common", "../shared/src/main/kotlin-native/posix") swiftSources = listOf("$projectDir/swiftSrc/benchmarks.swift", "$projectDir/swiftSrc/main.swift") compileTasks = listOf("compileKotlinNative", "linkBenchmarkReleaseFrameworkNative") + buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType dependencies.common(project(":endorsedLibraries:kotlinx.cli")) } \ No newline at end of file diff --git a/performance/videoplayer/build.gradle.kts b/performance/videoplayer/build.gradle.kts index cb3c59a3326..9badadb7d6e 100644 --- a/performance/videoplayer/build.gradle.kts +++ b/performance/videoplayer/build.gradle.kts @@ -4,6 +4,7 @@ */ import org.jetbrains.kotlin.PlatformInfo +import org.jetbrains.kotlin.getCompileOnlyBenchmarksOpts import org.jetbrains.kotlin.getNativeProgramExtension import org.jetbrains.kotlin.mingwPath @@ -47,9 +48,13 @@ var includeDirsSdl = when { else -> error("Unsupported platform") } +val defaultCompilerOpts = listOf("-g") +val buildOpts = getCompileOnlyBenchmarksOpts(project, defaultCompilerOpts) + compileBenchmark { applicationName = "Videoplayer" repeatNumber = 10 + compilerOpts = buildOpts buildSteps { step("runCinteropFfmpeg") { command = listOf( @@ -68,13 +73,13 @@ compileBenchmark { step("runKonanProgram") { command = listOf( "$dist/bin/konanc$toolSuffix", - "-ea", "-p", "program", "-g", + "-ea", "-p", "program", "-o", "${buildDir.absolutePath}/program$binarySuffix", "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib", "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib", "-Xmulti-platform", "$dist/../samples/videoplayer/src/videoPlayerMain/kotlin", "-entry", "sample.videoplayer.main" - ) + linkerOpts + ) + buildOpts + linkerOpts } } } diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt index 626af1dfdfd..9c9e2eccb63 100644 --- a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt +++ b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt @@ -77,15 +77,24 @@ class BenchmarksReport(val env: Environment, benchmarksList: List>(benchmarks) mergedBenchmarks.putAll(other.benchmarks) return BenchmarksReport(env, mergedBenchmarks.flatMap{it.value}, compiler) } + + // Concatenate benchmarks report if they have same environment and compiler. + operator fun plus(other: BenchmarksReport): BenchmarksReport { + if (compiler != other.compiler || env != other.env) { + error ("It's impossible to concat reports from different machines!") + } + return merge(other) + } } // Class for kotlin compiler @@ -304,4 +313,7 @@ class BenchmarkResult(val name: String, val status: Status, } """ } + + val shortName: String + get() = name.removeSuffix(metric.suffix) } \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index 8ead8141b2e..94017efefc5 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -19,9 +19,8 @@ import org.jetbrains.analyzer.readFile import org.jetbrains.analyzer.SummaryBenchmarksReport import kotlinx.cli.* import org.jetbrains.renders.* -import org.jetbrains.report.BenchmarksReport -import org.jetbrains.report.BenchmarkResult -import org.jetbrains.report.json.JsonTreeParser +import org.jetbrains.report.* +import org.jetbrains.report.json.* abstract class Connector { abstract val connectorPrefix: String @@ -85,8 +84,14 @@ fun getFileContent(fileName: String, user: String? = null): String { } } -fun getBenchmarkReport(fileName: String, user: String? = null) = - BenchmarksReport.create(JsonTreeParser.parse(getFileContent(fileName, user))) +fun getBenchmarkReport(fileName: String, user: String? = null): List { + val jsonEntity = JsonTreeParser.parse(getFileContent(fileName, user)) + return when (jsonEntity) { + is JsonObject -> listOf(BenchmarksReport.create(jsonEntity)) + is JsonArray -> jsonEntity.map { BenchmarksReport.create(it) } + else -> error("Wrong format of report. Expected object or array of objects.") + } +} fun parseNormalizeResults(results: String): Map> { val parsedNormalizeResults = mutableMapOf>() @@ -103,6 +108,26 @@ fun parseNormalizeResults(results: String): Map> { return parsedNormalizeResults } +fun mergeCompilerFlags(reports: List) = + reports.map { + val benchmarks = it.benchmarks.values.flatten().asSequence().filter { it.metric == BenchmarkResult.Metric.COMPILE_TIME } + .map { it.shortName }.distinct().sorted().joinToString() + "${it.compiler.backend.flags.joinToString()} for [$benchmarks]" + } + +fun mergeReportsWithDetailedFlags(reports: List) = + if (reports.size > 1) { + // Merge reports. + val detailedFlags = mergeCompilerFlags(reports) + reports.map { + BenchmarksReport(it.env, it.benchmarks.values.flatten(), + Compiler(Compiler.Backend(it.compiler.backend.type, it.compiler.backend.version, detailedFlags), + it.compiler.kotlinVersion)) + }.reduce { result, it -> result + it } + } else { + reports.first() + } + fun main(args: Array) { class Summary: Subcommand("summary") { val exec by option(ArgType.Choice(listOf("samples", "geomean")), @@ -129,7 +154,11 @@ fun main(args: Array) { val mainReport by argument(ArgType.String, description = "Main report for analysis") override fun execute() { - val benchsReport = SummaryBenchmarksReport(getBenchmarkReport(mainReport, user)) + val reportsList = getBenchmarkReport(mainReport, user) + val report = reportsList.reduce { result, it -> + result.merge(it) + } + val benchsReport = SummaryBenchmarksReport(report) val results = mutableListOf() val executionNormalize = execNormalize?.let { parseNormalizeResults(getFileContent(it)) @@ -183,9 +212,10 @@ fun main(args: Array) { if (argParser.parse(args).commandName == "benchmarksAnalyzer") { // Read contents of file. - val mainBenchsReport = getBenchmarkReport(mainReport, user) + val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user)) + var compareToBenchsReport = compareToReport?.let { - getBenchmarkReport(it, user) + mergeReportsWithDetailedFlags(getBenchmarkReport(it, user)) } // Generate comparasion report.