Benchmarks: Add compile benchmark plugin

This patch moves benchmark configuration logic described in performance/gradle/compileBenchmark.gradle to a separate plugin located in the buildSrc project.
2019-06-11 17:11:43 +07:00
parent 8600e4bd66
commit ba4fde666c
7 changed files with 215 additions and 69 deletions
@@ -63,8 +63,12 @@ compileGroovy {
 gradlePlugin {
    plugins {
        benchmarkPlugin {
-            id = 'benchmarking'
-            implementationClass = 'org.jetbrains.kotlin.benchmark.BenchmarkingPlugin'
+            id = "benchmarking"
+            implementationClass = "org.jetbrains.kotlin.benchmark.BenchmarkingPlugin"
+        }
+        compileBenchmarking {
+            id = "compile-benchmarking"
+            implementationClass = "org.jetbrains.kotlin.benchmark.CompileBenchmarkingPlugin"
        }
    }
 }
@@ -68,6 +68,8 @@ open class BenchmarkExtension @Inject constructor(val project: Project) {
    var linkerOpts: Collection<String> = emptyList()
 }

+// TODO: Override kotlin compiler with a correct version.
+
 /**
 * A plugin configuring a benchmark Kotlin/Native project.
 */
@@ -197,10 +199,9 @@ open class BenchmarkingPlugin: Plugin<Project> {
                val jarPath = (tasks.getByName("jvmJar") as Jar).archiveFile.get().asFile
                val jvmCompileTime = getJvmCompileTime(applicationName)
                val benchContents = buildDir.resolve(jvmBenchResults).readText()
-                
+
                val properties: Map<String, Any> = commonBenchmarkProperties + mapOf(
                    "type" to "jvm",
-                    // TODO: We had buildKotlinVersion here!!!!
                    "compilerVersion" to kotlinVersion,
                    "benchmarks" to benchContents,
                    "compileTime" to listOf(jvmCompileTime),
@@ -0,0 +1,111 @@
+package org.jetbrains.kotlin.benchmark
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.tasks.Delete
+import org.gradle.api.tasks.Exec
+import org.jetbrains.kotlin.*
+import javax.inject.Inject
+
+private typealias CommandList = List<String>
+
+open class CompileBenchmarkExtension @Inject constructor(val project: Project) {
+    var applicationName = project.name
+    var repeatNumber: Int = 1
+    var buildSteps: Map<String, CommandList> = emptyMap()
+}
+
+open class CompileBenchmarkingPlugin : Plugin<Project> {
+
+    private val exitCodes: MutableMap<String, Int> = mutableMapOf()
+    
+    private fun Project.configureUtilityTasks() {
+        tasks.create("configureBuild") {
+            it.doLast { mkdir(buildDir) }
+        }
+
+        tasks.create("clean", Delete::class.java) {
+            it.delete(buildDir)
+        }
+    }
+    
+    private fun Project.configureKonanRun(
+        benchmarkExtension: CompileBenchmarkExtension
+    ): Unit = with(benchmarkExtension) {
+        // Aggregate task.
+        val konanRun = tasks.create("konanRun") { task ->
+            task.dependsOn("configureBuild")
+        }
+
+        // Compile tasks.
+        afterEvaluate {
+            for (number in 1..repeatNumber) {
+                buildSteps.forEach { (taskName, command) ->
+                    tasks.create("$taskName$number", Exec::class.java).apply {
+                        commandLine(command)
+                        isIgnoreExitValue = true
+                        konanRun.dependsOn(this)
+                        doLast {
+                            exitCodes[name] = execResult.exitValue
+                        }
+                    }
+                }
+            }
+        }
+
+        // Report task.
+        tasks.create("konanJsonReport").apply {
+            doLast {
+                val nativeCompileTime = getCompileBenchmarkTime(
+                    applicationName,
+                    buildSteps.keys,
+                    repeatNumber,
+                    exitCodes
+                )
+                val nativeExecutable = buildDir.resolve("program${getNativeProgramExtension()}")
+                val properties = commonBenchmarkProperties + mapOf(
+                    "type" to "native",
+                    "compilerVersion" to konanVersion,
+                    "benchmarks" to "[]",
+                    "compileTime" to nativeCompileTime,
+                    "codeSize" to getCodeSizeBenchmark(applicationName, nativeExecutable.absolutePath)
+                )
+                val output = createJsonReport(properties)
+                buildDir.resolve(nativeJson).writeText(output)
+            }
+            konanRun.finalizedBy(this)
+        }
+    }
+
+    private fun Project.configureJvmRun(
+        benchmarkExtension: CompileBenchmarkExtension
+    ) {
+        val jvmRun = tasks.create("jvmRun") {
+            it.doLast { println("JVM run isn't supported") }
+        }
+
+        tasks.create("jvmJsonReport") {
+            it.doLast { println("JVM run isn't supported") }
+            jvmRun.finalizedBy(it)
+        }
+    }
+    
+    override fun apply(target: Project): Unit = with(target) {
+        addTimeListener(this)
+
+        val benchmarkExtension = extensions.create(
+            COMPILE_BENCHMARK_EXTENSION_NAME,
+            CompileBenchmarkExtension::class.java,
+            this
+        )
+
+        // Create tasks.
+        configureUtilityTasks()
+        configureKonanRun(benchmarkExtension)
+        configureJvmRun(benchmarkExtension)
+    }
+
+    companion object {
+        const val COMPILE_BENCHMARK_EXTENSION_NAME = "compileBenchmark"
+    }
+}
@@ -1,12 +0,0 @@
-import org.jetbrains.kotlin.MPPTools
-
-def dist = findProperty('org.jetbrains.kotlin.native.home') ?: 'dist'
-dist = (new File(dist)).isAbsolute() ? dist : "${project.getProjectDir()}/$dist"
-def toolExtension = System.getProperty("os.name").startsWith('Windows') ? ".bat" : ""
-project.ext {
-    applicationName = 'HelloWorld'
-    repeatNumber = 10
-    buildSteps = ["runKonanc": ["$dist/bin/konanc$toolExtension", "${project.getProjectDir()}/src/main/kotlin/main.kt", "-o",
-                                "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}"]]
-}
-apply from: rootProject.file("${rootProject.rootDir}/performance/gradle/compileBenchmark.gradle")
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+import org.jetbrains.kotlin.getNativeProgramExtension
+
+plugins {
+    id("compile-benchmarking")
+}
+
+val dist = file(findProperty("org.jetbrains.kotlin.native.home") ?: "dist")
+val toolSuffix = if (System.getProperty("os.name").startsWith("Windows")) ".bat" else ""
+val binarySuffix = getNativeProgramExtension()
+
+compileBenchmark {
+    applicationName = "HelloWorld"
+    repeatNumber = 10
+    buildSteps = mapOf(
+        "runKonanc" to listOf("$dist/bin/konanc$toolSuffix", "$projectDir/src/main/kotlin/main.kt", "-o", "$buildDir/program$binarySuffix")
+    )
+}
@@ -1,53 +0,0 @@
-import org.jetbrains.kotlin.PlatformInfo
-import org.jetbrains.kotlin.MPPTools
-
-def dist = findProperty('org.jetbrains.kotlin.native.home') ?: 'dist'
-dist = (new File(dist)).isAbsolute() ? dist : "${project.getProjectDir()}/$dist"
-def toolExtension = System.getProperty("os.name").startsWith('Windows') ? ".bat" : ""
-
-def linkerOpts = []
-if (PlatformInfo.isMac()) {
-    linkerOpts += ['-linker-options','-L/opt/local/lib', '-linker-options', '-L/usr/local/lib']
-} else if (PlatformInfo.isLinux()) {
-    linkerOpts += ['-linker-options', '-L/usr/lib/x86_64-linux-gnu', '-linker-options', '-L/usr/lib64']
-} else if (PlatformInfo.isWindows()) {
-    linkerOpts += ['-linker-options', "-L${MPPTools.mingwPath()}/lib"]
-}
-
-def includeDirsFfmpeg = []
-def filterDirsFfmpeg = []
-if (PlatformInfo.isMac()) {
-    filterDirsFfmpeg += ['-headerFilterAdditionalSearchPrefix', '/opt/local/include',
-                          '-headerFilterAdditionalSearchPrefix', '/usr/local/include']
-} else if (PlatformInfo.isLinux()) {
-    filterDirsFfmpeg += ['-headerFilterAdditionalSearchPrefix', '/usr/include',
-                          '-headerFilterAdditionalSearchPrefix', '/usr/include/x86_64-linux-gnu',
-                          '-headerFilterAdditionalSearchPrefix', '/usr/include/ffmpeg']
-} else if (PlatformInfo.isWindows()) {
-    includeDirsFfmpeg += ['-compiler-option', "-I${MPPTools.mingwPath()}/include"]
-}
-
-def includeDirsSdl = []
-if (PlatformInfo.isMac()) {
-    includeDirsSdl += ['-compiler-option', '-I/opt/local/include/SDL2',
-                       '-compiler-option', '-I/usr/local/include/SDL2']
-} else if (PlatformInfo.isLinux()) {
-    includeDirsSdl += ['-compiler-option', '-I/usr/include/SDL2']
-} else if (PlatformInfo.isWindows()) {
-    includeDirsSdl += ['-compiler-option', "-I${MPPTools.mingwPath()}/include/SDL2"]
-}
-
-project.ext {
-    applicationName = 'Videoplayer'
-    repeatNumber = 10
-    buildSteps = ["runCinteropFfmpeg": ["$dist/bin/cinterop$toolExtension", "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib",
-                                        "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/ffmpeg.def"] + filterDirsFfmpeg + includeDirsFfmpeg,
-                  "runCinteropSdl": ["$dist/bin/cinterop$toolExtension", "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib",
-                                     "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/sdl.def"] + includeDirsSdl,
-                  "runKonanProgram": ["$dist/bin/konanc$toolExtension", "-ea", "-p", "program", "-o", "${buildDir.absolutePath}/program${MPPTools.getNativeProgramExtension()}",
-                                      "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib",
-                                      "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib",
-                                      "-Xmulti-platform", "$dist/../samples/videoplayer/src/videoPlayerMain/kotlin",
-                                      "-entry", "sample.videoplayer.main"] + linkerOpts]
-}
-apply from: rootProject.file("${rootProject.rootDir}/performance/gradle/compileBenchmark.gradle")
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+import org.jetbrains.kotlin.PlatformInfo
+import org.jetbrains.kotlin.getNativeProgramExtension
+import org.jetbrains.kotlin.mingwPath
+
+plugins {
+    id("compile-benchmarking")
+}
+
+val dist = file(findProperty("org.jetbrains.kotlin.native.home") ?: "dist")
+val toolSuffix = if (System.getProperty("os.name").startsWith("Windows")) ".bat" else ""
+val binarySuffix = getNativeProgramExtension()
+
+val linkerOpts = when {
+    PlatformInfo.isMac() -> listOf("-linker-options","-L/opt/local/lib", "-linker-options", "-L/usr/local/lib")
+    PlatformInfo.isLinux() -> listOf("-linker-options", "-L/usr/lib/x86_64-linux-gnu", "-linker-options", "-L/usr/lib64")
+    PlatformInfo.isWindows() -> listOf("-linker-options", "-L$mingwPath/lib")
+    else -> error("Unsupported platform")
+}
+
+var includeDirsFfmpeg = emptyList<String>()
+var filterDirsFfmpeg = emptyList<String>()
+when {
+    PlatformInfo.isMac() -> filterDirsFfmpeg = listOf(
+        "-headerFilterAdditionalSearchPrefix", "/opt/local/include",
+        "-headerFilterAdditionalSearchPrefix", "/usr/local/include"
+    )
+    PlatformInfo.isLinux() -> filterDirsFfmpeg = listOf(
+        "-headerFilterAdditionalSearchPrefix", "/usr/include",
+        "-headerFilterAdditionalSearchPrefix", "/usr/include/x86_64-linux-gnu",
+        "-headerFilterAdditionalSearchPrefix", "/usr/include/ffmpeg"
+    )
+    PlatformInfo.isWindows() -> includeDirsFfmpeg = listOf("-compiler-option", "-I$mingwPath/include")
+}
+
+var includeDirsSdl = when {
+    PlatformInfo.isMac() -> listOf(
+        "-compiler-option", "-I/opt/local/include/SDL2",
+        "-compiler-option", "-I/usr/local/include/SDL2"
+    )
+    PlatformInfo.isLinux() -> listOf("-compiler-option", "-I/usr/include/SDL2")
+    PlatformInfo.isWindows() -> listOf("-compiler-option", "-I$mingwPath/include/SDL2")
+    else -> error("Unsupported platform")
+}
+
+compileBenchmark {
+    applicationName = "Videoplayer"
+    repeatNumber = 10
+    buildSteps = mapOf(
+        "runCinteropFfmpeg" to listOf("$dist/bin/cinterop$toolSuffix",
+            "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib",
+            "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/ffmpeg.def"
+        ) + filterDirsFfmpeg + includeDirsFfmpeg,
+
+        "runCinteropSdl" to listOf("$dist/bin/cinterop$toolSuffix",
+            "-o", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib",
+            "-def", "$dist/../samples/videoplayer/src/nativeInterop/cinterop/sdl.def"
+        ) + includeDirsSdl,
+
+        "runKonanProgram" to listOf("$dist/bin/konanc$toolSuffix",
+            "-ea", "-p", "program",
+            "-o", "${buildDir.absolutePath}/program$binarySuffix",
+            "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib",
+            "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib",
+            "-Xmulti-platform", "$dist/../samples/videoplayer/src/videoPlayerMain/kotlin",
+            "-entry", "sample.videoplayer.main"
+        ) + linkerOpts
+    )
+}