[K/N][perf] Move benchmarks related code to performance subdir

This commit is contained in:
Pavel Punegov
2023-01-26 12:46:01 +01:00
committed by Space Team
parent 0089517b25
commit 71f2c3cf9b
15 changed files with 60 additions and 32 deletions
@@ -0,0 +1,6 @@
package org.jetbrains.kotlin
enum class BenchmarkRepeatingType {
INTERNAL, // Let the benchmark perform warmups and repeats.
EXTERNAL, // Repeat by relaunching benchmark
}
@@ -0,0 +1,115 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.Project
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import java.io.BufferedReader
import java.io.FileInputStream
import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.net.HttpURLConnection
import java.net.URL
import java.util.*
private val Project.performanceServerUrl: String
get() = findProperty("kotlin.native.performance.server.url")?.toString() ?: "http://localhost:3000"
/**
* Task to save benchmarks results on server.
*
* @property bundleSize size of build
* @property onlyBranch register only builds for branch
* @property fileWithResult json file with benchmarks run results
* @property performanceServer URL of the performance server
*/
open class BuildRegister : DefaultTask() {
@get:Input
@get:Optional
var onlyBranch: String? = null
@get:Input
@get:Optional
var bundleSize: Int? = null
@get:Input
@get:Optional
var buildNumberSuffix: String? = null
@get:Input
@get:Optional
var fileWithResult: String = "nativeReport.json"
@get:Input
@get:Optional
var performanceServer: String? = project.performanceServerUrl
private fun sendPostRequest(url: String, body: String): String {
val connection = URL(url).openConnection() as HttpURLConnection
return connection.apply {
setRequestProperty("Content-Type", "application/json; charset=utf-8")
requestMethod = "POST"
doOutput = true
val outputWriter = OutputStreamWriter(outputStream)
outputWriter.write(body)
outputWriter.flush()
}.let {
if (it.responseCode == 200) it.inputStream else it.errorStream
}.let { streamToRead ->
BufferedReader(InputStreamReader(streamToRead)).use {
val response = StringBuffer()
var inputLine = it.readLine()
while (inputLine != null) {
response.append(inputLine)
inputLine = it.readLine()
}
it.close()
response.toString()
}
}
}
@TaskAction
fun run() {
// Get TeamCity properties.
val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?: error("Can't load teamcity config!")
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val teamCityUser = buildProperties.getProperty("teamcity.auth.userId")
val teamCityPassword = buildProperties.getProperty("teamcity.auth.password")
// Get branch.
val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword)
val branch = getBuildProperty(currentBuild, "branchName")
// Send post request to register build.
val requestBody = buildString {
append("{\"buildId\":\"$buildId\",")
append("\"teamCityUser\":\"$teamCityUser\",")
append("\"teamCityPassword\":\"$teamCityPassword\",")
append("\"fileWithResult\":\"$fileWithResult\",")
append("\"bundleSize\": ${bundleSize?.let { "\"$bundleSize\"" } ?: bundleSize},")
append("\"buildNumberSuffix\": ${buildNumberSuffix?.let { "\"$buildNumberSuffix\"" } ?: buildNumberSuffix}}")
}
if (onlyBranch == null || onlyBranch == branch) {
try {
println(sendPostRequest("$performanceServer/register", requestBody))
} catch (t: Throwable) {
println("Failed to send POST to '$performanceServer/register'")
throw t
}
} else {
println("Skipping registration. Current branch $branch, need registration for $onlyBranch!")
}
}
}
@@ -0,0 +1,54 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.NamedDomainObjectCollection
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.ExtraPropertiesExtension
import org.gradle.api.provider.Provider
import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeCompilation
import java.io.File
/*
* This file includes internal short-cuts visible only inside of the 'buildSrc' module.
*/
internal val hostOs by lazy { System.getProperty("os.name") }
internal val userHome by lazy { System.getProperty("user.home") }
internal val Project.ext: ExtraPropertiesExtension
get() = extensions.getByName("ext") as ExtraPropertiesExtension
internal val Project.kotlin: KotlinMultiplatformExtension
get() = extensions.getByName("kotlin") as KotlinMultiplatformExtension
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.macosX64: KotlinTargetPreset<*>
get() = getByName(::macosX64.name)
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.macosArm64: KotlinTargetPreset<*>
get() = getByName(::macosArm64.name)
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.linuxX64: KotlinTargetPreset<*>
get() = getByName(::linuxX64.name)
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.mingwX64: KotlinTargetPreset<*>
get() = getByName(::mingwX64.name)
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.linuxArm64: KotlinTargetPreset<*>
get() = getByName(::linuxArm64.name)
internal val NamedDomainObjectContainer<out KotlinCompilation<*>>.main: KotlinNativeCompilation
get() = getByName(::main.name) as KotlinNativeCompilation
internal val FileCollection.isNotEmpty: Boolean
get() = !isEmpty
internal fun Provider<File>.resolve(child: String): Provider<File> = map { it.resolve(child) }
@@ -0,0 +1,281 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
@file:JvmName("MPPTools")
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.TaskState
import org.gradle.api.execution.TaskExecutionListener
import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.report.*
import org.jetbrains.report.json.*
import java.nio.file.Paths
import java.io.File
import java.io.FileInputStream
import java.io.BufferedOutputStream
import java.io.BufferedInputStream
import java.net.HttpURLConnection
import java.net.URL
import java.util.Base64
/*
* This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future.
*/
// Short-cuts for mostly used paths.
@get:JvmName("mingwPath")
val mingwPath by lazy { System.getenv("MINGW64_DIR") ?: "c:/msys64/mingw64" }
@get:JvmName("kotlinNativeDataPath")
val kotlinNativeDataPath by lazy {
System.getenv("KONAN_DATA_DIR") ?: Paths.get(userHome, ".konan").toString()
}
// A short-cut for evaluation of the default host Kotlin/Native preset.
@JvmOverloads
fun defaultHostPreset(
subproject: Project,
whitelist: List<KotlinTargetPreset<*>> = listOf(subproject.kotlin.presets.macosX64, subproject.kotlin.presets.macosArm64,
subproject.kotlin.presets.linuxX64, subproject.kotlin.presets.mingwX64)
): KotlinTargetPreset<*> {
if (whitelist.isEmpty())
throw Exception("Preset whitelist must not be empty in Kotlin/Native ${subproject.displayName}.")
val presetCandidate = when {
PlatformInfo.isMac() -> if (PlatformInfo.hostName.endsWith("x64"))
subproject.kotlin.presets.macosX64
else subproject.kotlin.presets.macosArm64
PlatformInfo.isLinux() -> subproject.kotlin.presets.linuxX64
PlatformInfo.isWindows() -> subproject.kotlin.presets.mingwX64
else -> null
}
return if (presetCandidate != null && presetCandidate in whitelist)
presetCandidate
else
throw Exception("Host OS '$hostOs' is not supported in Kotlin/Native ${subproject.displayName}.")
}
fun targetHostPreset(
subproject: Project,
crossTarget: String
): KotlinTargetPreset<*> {
return when(crossTarget) {
"linuxArm64" -> subproject.kotlin.presets.linuxArm64
"linuxX64" -> subproject.kotlin.presets.linuxX64
else -> throw Exception("Running becnhmarks on target $crossTarget isn't supported yet.")
}
}
fun getNativeProgramExtension(): String = when {
PlatformInfo.isMac() -> ".kexe"
PlatformInfo.isLinux() -> ".kexe"
PlatformInfo.isWindows() -> ".exe"
else -> error("Unknown host")
}
fun getFileSize(filePath: String): Long? {
val file = File(filePath)
return if (file.exists()) file.length() else null
}
fun getCodeSizeBenchmark(programName: String, filePath: String): BenchmarkResult {
val codeSize = getFileSize(filePath)
return BenchmarkResult(programName,
codeSize?. let { BenchmarkResult.Status.PASSED } ?: run { BenchmarkResult.Status.FAILED },
codeSize?.toDouble() ?: 0.0, BenchmarkResult.Metric.CODE_SIZE, codeSize?.toDouble() ?: 0.0, 1, 0)
}
fun toCodeSizeBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult {
if (!metricDescription.startsWith("CODE_SIZE")) {
error("Wrong metric is used as code size.")
}
val codeSize = metricDescription.split(' ')[1].toDouble()
return BenchmarkResult(programName,
if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
codeSize, BenchmarkResult.Metric.CODE_SIZE, codeSize, 1, 0)
}
// Create benchmarks json report based on information get from gradle project
fun createJsonReport(projectProperties: Map<String, Any>): String {
fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown"
val machine = Environment.Machine(getValue("cpu"), getValue("os"))
val jdk = Environment.JDKInstance(getValue("jdkVersion"), getValue("jdkVendor"))
val env = Environment(machine, jdk)
val flags: List<String> = (projectProperties["flags"] as? List<*>)?.filterIsInstance<String>() ?: emptyList()
val backend = Compiler.Backend(Compiler.backendTypeFromString(getValue("type"))!! ,
getValue("compilerVersion"), flags)
val kotlin = Compiler(backend, getValue("kotlinVersion"))
val benchDesc = getValue("benchmarks")
val benchmarksArray = JsonTreeParser.parse(benchDesc)
val benchmarks = parseBenchmarksArray(benchmarksArray)
.union((projectProperties["compileTime"] as? List<*>)?.filterIsInstance<BenchmarkResult>() ?: emptyList()).union(
listOf(projectProperties["codeSize"] as? BenchmarkResult).filterNotNull()).toList()
val report = BenchmarksReport(env, benchmarks, kotlin)
return report.toJson()
}
fun mergeReports(reports: List<File>): String {
val reportsToMerge = reports.filter { it.exists() }.map {
val json = it.inputStream().bufferedReader().use { it.readText() }
val reportElement = JsonTreeParser.parse(json)
BenchmarksReport.create(reportElement)
}
val structuredReports = mutableMapOf<String, MutableList<BenchmarksReport>>()
reportsToMerge.map { it.compiler.backend.flags.joinToString() to it }.forEach {
structuredReports.getOrPut(it.first) { mutableListOf<BenchmarksReport>() }.add(it.second)
}
val jsons = structuredReports.map { (_, value) -> value.reduce { result, it -> result + it }.toJson() }
return when(jsons.size) {
0 -> ""
1 -> jsons[0]
else -> jsons.joinToString(prefix = "[", postfix = "]")
}
}
fun getCompileOnlyBenchmarksOpts(project: Project, defaultCompilerOpts: List<String>): List<String> {
val dist = project.file(project.findProperty("kotlin.native.home") ?: "dist")
val useCache = !project.hasProperty("disableCompilerCaches")
val cacheOption = "-Xcache-directory=$dist/klib/cache/${HostManager.host.name}-gSTATIC"
.takeIf { useCache && !PlatformInfo.isWindows() } // TODO: remove target condition when we have cache support for other targets.
return (project.findProperty("nativeBuildType") as String?)?.let {
if (it.equals("RELEASE", true))
listOf("-opt")
else if (it.equals("DEBUG", true))
listOfNotNull("-g", cacheOption)
else listOf()
} ?: defaultCompilerOpts + listOfNotNull(cacheOption?.takeIf { !defaultCompilerOpts.contains("-opt") })
}
// Find file with set name in directory.
fun findFile(fileName: String, directory: String): String? =
File(directory).walkTopDown().filter { !it.absolutePath.contains(".dSYM") }
.find { it.name == fileName }?.getAbsolutePath()
fun uploadFileToArtifactory(url: String, project: String, artifactoryFilePath: String,
filePath: String, password: String) {
val uploadUrl = "$url/$project/$artifactoryFilePath"
sendUploadRequest(uploadUrl, filePath, extraHeaders = listOf(Pair("X-JFrog-Art-Api", password)))
}
fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null,
extraHeaders: List<Pair<String, String>> = emptyList()) {
val uploadingFile = File(fileName)
val connection = URL(url).openConnection() as HttpURLConnection
connection.doOutput = true
connection.doInput = true
connection.requestMethod = "PUT"
connection.setRequestProperty("Content-type", "text/plain")
if (username != null && password != null) {
val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
extraHeaders.forEach {
connection.addRequestProperty(it.first, it.second)
}
try {
connection.connect()
BufferedOutputStream(connection.outputStream).use { output ->
BufferedInputStream(FileInputStream(uploadingFile)).use { input ->
input.copyTo(output)
}
}
val response = connection.responseMessage
println("Upload request ended with ${connection.responseCode} - $response")
} catch (t: Throwable) {
error("Couldn't upload file $fileName to $url")
}
}
// A short-cut to add a Kotlin/Native run task.
fun createRunTask(
subproject: Project,
name: String,
linkTask: Task,
executable: String,
outputFileName: String
): Task {
return subproject.tasks.create(name, RunKotlinNativeTask::class.java, linkTask, executable, outputFileName)
}
fun getJvmCompileTime(subproject: Project,programName: String): BenchmarkResult =
TaskTimerListener.getTimerListenerOfSubproject(subproject)
.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
@JvmOverloads
fun getNativeCompileTime(subproject: Project, programName: String,
tasks: List<String> = listOf("linkBenchmarkReleaseExecutableNative")): BenchmarkResult =
TaskTimerListener.getTimerListenerOfSubproject(subproject).getBenchmarkResult(programName, tasks)
fun getCompileBenchmarkTime(subproject: Project,
programName: String, tasksNames: Iterable<String>,
repeats: Int, exitCodes: Map<String, Int>) =
(1..repeats).map { number ->
var time = 0.0
var status = BenchmarkResult.Status.PASSED
tasksNames.forEach {
time += TaskTimerListener.getTimerListenerOfSubproject(subproject).getTime("$it$number")
status = if (exitCodes["$it$number"] != 0) BenchmarkResult.Status.FAILED else status
}
BenchmarkResult(programName, status, time, BenchmarkResult.Metric.COMPILE_TIME, time, number, 0)
}.toList()
fun toCompileBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult {
if (!metricDescription.startsWith("COMPILE_TIME")) {
error("Wrong metric is used as compile time.")
}
val time = metricDescription.split(' ')[1].toDouble()
return BenchmarkResult(programName,
if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0)
}
// Class time tracker for all tasks.
class TaskTimerListener: TaskExecutionListener {
companion object {
internal val timerListeners = mutableMapOf<String, TaskTimerListener>()
internal fun getTimerListenerOfSubproject(subproject: Project) =
timerListeners[subproject.name] ?: error("TimeListener for project ${subproject.name} wasn't set")
}
val tasksTimes = mutableMapOf<String, Double>()
fun getBenchmarkResult(programName: String, tasksNames: List<String>): BenchmarkResult {
val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum()
// TODO get this info from gradle plugin with exit code end stacktrace.
val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b }
return BenchmarkResult(programName,
if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0)
}
fun getTime(taskName: String) = tasksTimes[taskName] ?: 0.0
private var startTime = System.nanoTime()
override fun beforeExecute(task: Task) {
startTime = System.nanoTime()
}
override fun afterExecute(task: Task, taskState: TaskState) {
tasksTimes[task.name] = (System.nanoTime() - startTime) / 1000.0
}
}
fun addTimeListener(subproject: Project) {
val listener = TaskTimerListener()
TaskTimerListener.timerListeners.put(subproject.name, listener)
subproject.gradle.addListener(listener)
}
@@ -8,3 +8,33 @@ fun Project.kotlinInit(cacheRedirectorEnabled: Boolean) {
extensions.extraProperties["defaultSnapshotVersion"] = kotlinBuildProperties.defaultSnapshotVersion
extensions.extraProperties["kotlinVersion"] = findProperty("kotlinVersion")
}
fun String.splitCommaSeparatedOption(optionName: String) =
split("\\s*,\\s*".toRegex()).map {
if (it.isNotEmpty()) listOf(optionName, it) else listOf(null)
}.flatten().filterNotNull()
data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String)
val teamCityUrl = "https://buildserver.labs.intellij.net"
fun buildsUrl(buildLocator: String) =
"$teamCityUrl/app/rest/builds/?locator=$buildLocator"
fun getBuild(buildLocator: String, user: String, password: String) =
try {
sendGetRequest(buildsUrl(buildLocator), user, password)
} catch (t: Throwable) {
error("Try to get build! TeamCity is unreachable!")
}
fun sendGetRequest(url: String, username: String? = null, password: String? = null): String {
val connection = URL(url).openConnection() as HttpURLConnection
if (username != null && password != null) {
val auth = Base64.getEncoder().encode(("$username:$password").toByteArray()).toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
connection.setRequestProperty("Accept", "application/json");
connection.connect()
return connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
}
@@ -0,0 +1,18 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.jetbrains.kotlin.konan.target.*
object PlatformInfo {
@JvmStatic
fun isMac() = HostManager.hostIsMac
@JvmStatic
fun isWindows() = HostManager.hostIsMingw
@JvmStatic
fun isLinux() = HostManager.hostIsLinux
}
@@ -0,0 +1,125 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.api.tasks.Input
import org.jetbrains.kotlin.benchmark.LogLevel
import org.jetbrains.kotlin.benchmark.Logger
import org.jetbrains.report.json.*
import java.io.ByteArrayOutputStream
import java.io.File
data class ExecParameters(val warmupCount: Int, val repeatCount: Int,
val filterArgs: List<String>, val filterRegexArgs: List<String>,
val verbose: Boolean, val outputFileName: String?)
open class RunJvmTask: JavaExec() {
@Input
@Option(option = "filter", description = "Benchmarks to run (comma-separated)")
var filter: String = ""
@Input
@Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
var filterRegex: String = ""
@Input
@Option(option = "verbose", description = "Verbose mode of running benchmarks")
var verbose: Boolean = false
@Input
var warmupCount: Int = 0
@Input
var repeatCount: Int = 0
@Input
var repeatingType = BenchmarkRepeatingType.INTERNAL
@Input
var outputFileName: String? = null
private var predefinedArgs: List<String> = emptyList()
private fun executeTask(execParameters: ExecParameters): String {
// Firstly clean arguments.
setArgs(emptyList())
args(predefinedArgs)
args(execParameters.filterArgs)
args(execParameters.filterRegexArgs)
args("-w", execParameters.warmupCount)
args("-r", execParameters.repeatCount)
if (execParameters.verbose) {
args("-v")
}
execParameters.outputFileName?.let { args("-o", outputFileName) }
standardOutput = ByteArrayOutputStream()
super.exec()
return standardOutput.toString()
}
private fun getBenchmarksList(filterArgs: List<String>, filterRegexArgs: List<String>): List<String> {
// Firstly clean arguments.
setArgs(emptyList())
args("list")
standardOutput = ByteArrayOutputStream()
super.exec()
val benchmarks = standardOutput.toString().lines()
val regexes = filterRegexArgs.map { it.toRegex() }
return if (filterArgs.isNotEmpty() || regexes.isNotEmpty()) {
benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }
} else benchmarks.filter { !it.isEmpty() }
}
private fun execSeparateBenchmarkRepeatedly(benchmark: String): List<String> {
// Logging with application should be done only in case it controls running benchmarks itself.
// Although it's a responsibility of gradle task.
val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
logger.log("Warm up iterations for benchmark $benchmark\n")
for (i in 0.until(warmupCount)) {
executeTask(ExecParameters(0, 1, listOf("-f", benchmark),
emptyList(), false, null))
}
val result = mutableListOf<String>()
logger.log("Running benchmark $benchmark ")
for (i in 0.until(repeatCount)) {
logger.log(".", usePrefix = false)
val benchmarkReport = JsonTreeParser.parse(
executeTask(ExecParameters(0, 1, listOf("-f", benchmark),
emptyList(), false, null)
).removePrefix("[").removeSuffix("]")
).jsonObject
val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
put("repeat", JsonLiteral(i) as JsonElement)
put("warmup", JsonLiteral(warmupCount))
})
result.add(modifiedBenchmarkReport.toString())
}
logger.log("\n", usePrefix = false)
return result
}
private fun execBenchmarksRepeatedly(filterArgs: List<String>, filterRegexArgs: List<String>) {
val benchmarksToRun = getBenchmarksList(filterArgs, filterRegexArgs)
val results = benchmarksToRun.flatMap { benchmark ->
execSeparateBenchmarkRepeatedly(benchmark)
}
File(outputFileName).printWriter().use { out ->
out.println("[${results.joinToString(",")}]")
}
}
@TaskAction
override fun exec() {
assert(outputFileName != null) { "Output file name should be always set" }
predefinedArgs = args
val filterArgs = filter.splitCommaSeparatedOption("-f")
val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
when (repeatingType) {
BenchmarkRepeatingType.INTERNAL -> executeTask(
ExecParameters(warmupCount, repeatCount, filterArgs, filterRegexArgs, verbose, outputFileName)
)
BenchmarkRepeatingType.EXTERNAL -> execBenchmarksRepeatedly(filterArgs, filterRegexArgs)
}
}
}
@@ -0,0 +1,164 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.jetbrains.kotlin.benchmark.Logger
import org.jetbrains.kotlin.benchmark.LogLevel
import org.jetbrains.report.json.*
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Internal
import java.io.ByteArrayOutputStream
import java.io.File
import javax.inject.Inject
import kotlin.collections.HashMap
open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
private val executable: String,
private val outputFileName: String
) : DefaultTask() {
@Input
@Option(option = "filter", description = "Benchmarks to run (comma-separated)")
var filter: String = ""
@Input
@Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
var filterRegex: String = ""
@Input
@Option(option = "verbose", description = "Verbose mode of running benchmarks")
var verbose: Boolean = false
@Input
@Option(option = "baseOnly", description = "Run only set of base benchmarks")
var baseOnly: Boolean = false
@Input
var warmupCount: Int = 0
@Input
var repeatCount: Int = 0
@Input
var repeatingType = BenchmarkRepeatingType.INTERNAL
private val argumentsList = mutableListOf<String>()
init {
this.dependsOn += linkTask.name
this.finalizedBy("konanJsonReport")
}
fun depends(taskName: String) {
this.dependsOn += taskName
}
fun args(vararg arguments: String) {
argumentsList.addAll(arguments.toList())
}
@Internal
val remoteHost = project.findProperty("remoteHost")?.toString()
@Internal
val remoteHostFolder = project.findProperty("remoteHostFolder")?.toString()
private fun execBenchmarkOnce(benchmark: String, warmupCount: Int, repeatCount: Int) : String {
val output = ByteArrayOutputStream()
val useCset = project.findProperty("useCset")?.toString()?.toBoolean() ?: false
project.exec {
when {
useCset -> {
executable = "cset"
args("shield", "--exec", "--", this@RunKotlinNativeTask.executable)
}
remoteHost != null -> {
executable = "ssh"
val remoteExecutable = this@RunKotlinNativeTask.executable.split("/").last()
args (remoteHost, "$remoteHostFolder/$remoteExecutable")
}
else -> executable = this@RunKotlinNativeTask.executable
}
args(argumentsList)
args("-f", benchmark)
// Logging with application should be done only in case it controls running benchmarks itself.
// Although it's a responsibility of gradle task.
if (verbose && repeatingType == BenchmarkRepeatingType.INTERNAL) {
args("-v")
}
args("-w", warmupCount.toString())
args("-r", repeatCount.toString())
standardOutput = output
}
return output.toString().substringAfter("[").removeSuffix("]")
}
private fun execBenchmarkRepeatedly(benchmark: String, warmupCount: Int, repeatCount: Int) : List<String> {
val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
logger.log("Warm up iterations for benchmark $benchmark\n")
for (i in 0.until(warmupCount)) {
execBenchmarkOnce(benchmark, 0, 1)
}
val result = mutableListOf<String>()
logger.log("Running benchmark $benchmark ")
for (i in 0.until(repeatCount)) {
logger.log(".", usePrefix = false)
val benchmarkReport = JsonTreeParser.parse(execBenchmarkOnce(benchmark, 0, 1)).jsonObject
val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
put("repeat", JsonLiteral(i))
put("warmup", JsonLiteral(warmupCount))
})
result.add(modifiedBenchmarkReport.toString())
}
logger.log("\n", usePrefix = false)
return result
}
@TaskAction
fun run() {
val output = ByteArrayOutputStream()
remoteHost?.let {
requireNotNull(remoteHostFolder) {"Please provide folder on remote host with -PremoteHostFolder=<folder>"}
project.exec {
executable = "scp"
args(this@RunKotlinNativeTask.executable, "$it:$remoteHostFolder")
}
}
project.exec {
if (remoteHost != null) {
executable = "ssh"
val remoteExecutable = this@RunKotlinNativeTask.executable.split("/").last()
args (remoteHost, "$remoteHostFolder/$remoteExecutable")
} else {
executable = this@RunKotlinNativeTask.executable
}
if (baseOnly) {
args("baseOnlyList")
} else {
args("list")
}
standardOutput = output
}
val benchmarks = output.toString().lines()
val filterArgs = filter.splitCommaSeparatedOption("-f")
val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
val regexes = filterRegexArgs.map { it.toRegex() }
val benchmarksToRun = if (filterArgs.isNotEmpty() || regexes.isNotEmpty()) {
benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }.filter { it.isNotEmpty() }
} else benchmarks.filter { !it.isEmpty() }
val results = benchmarksToRun.flatMap { benchmark ->
when (repeatingType) {
BenchmarkRepeatingType.INTERNAL -> listOf(execBenchmarkOnce(benchmark, warmupCount, repeatCount))
BenchmarkRepeatingType.EXTERNAL -> execBenchmarkRepeatedly(benchmark, warmupCount, repeatCount)
}
}
File(outputFileName).printWriter().use { out ->
out.println("[${results.joinToString(",")}]")
}
}
}
@@ -0,0 +1,25 @@
package org.jetbrains.kotlin.benchmark
import java.text.SimpleDateFormat
import java.util.*
enum class LogLevel { DEBUG, OFF }
class Logger(val level: LogLevel = LogLevel.OFF) {
private fun printStderr(message: String) {
System.err.print(message)
}
private fun currentTime(): String =
SimpleDateFormat("HH:mm:ss").format(Date())
fun log(message: String, messageLevel: LogLevel = LogLevel.DEBUG, usePrefix: Boolean = true) {
if (messageLevel == level) {
if (usePrefix) {
printStderr("[$level][${currentTime()}] $message")
} else {
printStderr("$message")
}
}
}
}
@@ -0,0 +1,295 @@
package org.jetbrains.kotlin.benchmark
import groovy.lang.Closure
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.Dependency
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinNativeTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
import org.jetbrains.kotlin.konan.target.HostManager
import javax.inject.Inject
import kotlin.reflect.KClass
internal val NamedDomainObjectContainer<KotlinSourceSet>.commonMain
get() = maybeCreate("commonMain")
internal val NamedDomainObjectContainer<KotlinSourceSet>.nativeMain
get() = maybeCreate("nativeMain")
internal val Project.nativeWarmup: Int
get() = (property("nativeWarmup") as String).toInt()
internal val Project.attempts: Int
get() = (property("attempts") as String).toInt()
internal val Project.nativeBenchResults: String
get() = property("nativeBenchResults") as String
// Gradle property to add flags to benchmarks run from command line.
internal val Project.compilerArgs: List<String>
get() = (findProperty("compilerArgs") as String?)?.split("\\s".toRegex()).orEmpty()
internal val Project.kotlinVersion: String
get() = property("kotlinVersion") as String
internal val Project.konanVersion: String
get() = property("konanVersion") as String
internal val Project.nativeJson: String
get() = project.property("nativeJson") as String
internal val Project.jvmJson: String
get() = project.property("jvmJson") as String
internal val Project.buildType: NativeBuildType
get() = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: NativeBuildType.RELEASE
internal val Project.crossTarget: String?
get() = findProperty("crossTarget") as String?
internal val Project.commonBenchmarkProperties: Map<String, Any>
get() = mapOf(
"cpu" to System.getProperty("os.arch"),
"os" to System.getProperty("os.name"),
"jdkVersion" to System.getProperty("java.version"),
"jdkVendor" to System.getProperty("java.vendor"),
"kotlinVersion" to kotlinVersion
)
open class BenchmarkExtension @Inject constructor(val project: Project) {
var applicationName: String = project.name
var commonSrcDirs: Collection<Any> = emptyList()
var nativeSrcDirs: Collection<Any> = emptyList()
var compileTasks: List<String> = emptyList()
var linkerOpts: Collection<String> = emptyList()
var compilerOpts: List<String> = emptyList()
var buildType: NativeBuildType = project.buildType
var repeatingType: BenchmarkRepeatingType = BenchmarkRepeatingType.INTERNAL
var cleanBeforeRunTask: String? = "konanRun"
val dependencies: BenchmarkDependencies = BenchmarkDependencies()
fun dependencies(action: BenchmarkDependencies.() -> Unit) =
dependencies.action()
fun dependencies(action: Closure<*>) {
project.configure(dependencies, action)
}
inner class BenchmarkDependencies {
public val sourceSets: NamedDomainObjectContainer<KotlinSourceSet>
get() = project.kotlin.sourceSets
fun project(path: String): Dependency = project.dependencies.project(mapOf("path" to path))
fun project(path: String, configuration: String): Dependency =
project.dependencies.project(mapOf("path" to path, "configuration" to configuration))
fun common(notation: Any) = sourceSets.commonMain.dependencies {
implementation(notation)
}
fun native(notation: Any) = sourceSets.nativeMain.dependencies {
implementation(notation)
}
}
}
/**
* A plugin configuring a benchmark Kotlin/Native project.
*/
abstract class BenchmarkingPlugin: Plugin<Project> {
protected abstract val Project.nativeExecutable: String
protected abstract val Project.nativeLinkTask: Task
protected abstract val Project.benchmark: BenchmarkExtension
protected abstract val benchmarkExtensionName: String
protected abstract val benchmarkExtensionClass: KClass<*>
protected val mingwPath: String = System.getenv("MINGW64_DIR") ?: "c:/msys64/mingw64"
protected open fun Project.determinePreset(): AbstractKotlinNativeTargetPreset<*> =
(crossTarget?.let { targetHostPreset(this, it) } ?:
defaultHostPreset(this).also { preset ->
logger.quiet("$project has been configured for ${preset.name} platform.")
}) as AbstractKotlinNativeTargetPreset<*>
protected abstract fun NamedDomainObjectContainer<KotlinSourceSet>.configureSources(project: Project)
protected open fun NamedDomainObjectContainer<KotlinSourceSet>.additionalConfigurations(project: Project) {}
protected open fun Project.configureSourceSets(kotlinVersion: String) {
with(kotlin.sourceSets) {
commonMain.dependencies {
implementation(files("${project.findProperty("kotlin_dist")}/kotlinc/lib/kotlin-stdlib.jar"))
}
repositories.flatDir {
dir("${project.findProperty("kotlin_dist")}/kotlinc/lib")
}
additionalConfigurations(this@configureSourceSets)
// Add sources specified by a user in the benchmark DSL.
afterEvaluate {
configureSources(project)
}
}
}
protected open fun KotlinNativeTarget.configureNativeOutput(project: Project) {
binaries.executable(NATIVE_EXECUTABLE_NAME, listOf(project.benchmark.buildType)) {
if (HostManager.hostIsMingw) {
linkerOpts.add("-L${mingwPath}/lib")
}
runTask?.apply {
group = ""
enabled = false
}
// Specify settings configured by a user in the benchmark extension.
project.afterEvaluate {
linkerOpts.addAll(project.benchmark.linkerOpts)
freeCompilerArgs = project.benchmark.compilerOpts + project.compilerArgs
}
}
}
protected fun Project.configureNativeTarget(hostPreset: AbstractKotlinNativeTargetPreset<*>) {
kotlin.targetFromPreset(hostPreset, NATIVE_TARGET_NAME) {
compilations.named("main").configure {
@Suppress("DEPRECATION")
kotlinOptions.freeCompilerArgs = benchmark.compilerOpts + project.compilerArgs
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.5")
}
}
configureNativeOutput(this@configureNativeTarget)
}
}
protected open fun configureMPPExtension(project: Project) {
project.configureSourceSets(project.kotlinVersion)
project.configureNativeTarget(project.determinePreset())
}
protected open fun Project.configureNativeTask(nativeTarget: KotlinNativeTarget): Task {
val konanRun = createRunTask(this, "konanRun", nativeLinkTask,
nativeExecutable, buildDir.resolve(nativeBenchResults).absolutePath).apply {
group = BENCHMARKING_GROUP
description = "Runs the benchmark for Kotlin/Native."
}
afterEvaluate {
val task = konanRun as RunKotlinNativeTask
task.args("-p", "${benchmark.applicationName}::")
task.warmupCount = nativeWarmup
task.repeatCount = attempts
task.repeatingType = benchmark.repeatingType
}
return konanRun
}
protected abstract fun Project.configureJvmTask(): Task
protected fun compilerFlagsFromBinary(project: Project): List<String> {
val result = mutableListOf<String>()
if (project.benchmark.buildType.optimized) {
result.add("-opt")
}
if (project.benchmark.buildType.debuggable) {
result.add("-g")
}
return result
}
@Suppress("DEPRECATION")
protected open fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) =
compilerFlagsFromBinary(project) + nativeTarget.compilations.main.kotlinOptions.freeCompilerArgs.map { "\"$it\"" }
protected open fun Project.collectCodeSize(applicationName: String) =
getCodeSizeBenchmark(applicationName, nativeExecutable)
@OptIn(ExperimentalStdlibApi::class)
protected open fun Project.configureKonanJsonTask(nativeTarget: KotlinNativeTarget): Task {
return tasks.create("konanJsonReport") {
group = BENCHMARKING_GROUP
description = "Builds the benchmarking report for Kotlin/Native."
doLast {
val applicationName = benchmark.applicationName
val benchContents = buildDir.resolve(nativeBenchResults).readText()
val nativeCompileTasks = if (benchmark.compileTasks.isEmpty()) {
listOf("linkBenchmark${benchmark.buildType.name.lowercase().replaceFirstChar { it.uppercase() }}ExecutableNative")
} else benchmark.compileTasks
val nativeCompileTime = getNativeCompileTime(project, applicationName, nativeCompileTasks)
val properties = commonBenchmarkProperties + mapOf(
"type" to "native",
"compilerVersion" to konanVersion,
"flags" to getCompilerFlags(project, nativeTarget).sorted(),
"benchmarks" to benchContents,
"compileTime" to listOf(nativeCompileTime),
"codeSize" to collectCodeSize(applicationName)
)
val output = createJsonReport(properties)
buildDir.resolve(nativeJson).writeText(output)
}
}
}
protected abstract fun Project.configureJvmJsonTask(jvmRun: Task): Task
protected open fun Project.configureExtraTasks() {}
private fun Project.configureTasks() {
val nativeTarget = kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget
configureExtraTasks()
// Native run task.
configureNativeTask(nativeTarget)
// JVM run task.
val jvmRun = configureJvmTask()
// Native report task.
configureKonanJsonTask(nativeTarget)
// JVM report task.
configureJvmJsonTask(jvmRun)
project.afterEvaluate {
// Need to rebuild benchmark to collect compile time.
project.benchmark.cleanBeforeRunTask?.let { tasks.getByName(it).dependsOn("clean") }
}
}
override fun apply(target: Project) = with(target) {
pluginManager.apply("kotlin-multiplatform")
// Use Kotlin compiler version specified by the project property.
target.logger.info("BenchmarkingPlugin.kt:apply($kotlinVersion)")
dependencies.add(
"kotlinCompilerClasspath", files(
"${project.findProperty("kotlin_dist")}/kotlinc/lib/kotlin-compiler.jar",
"${project.findProperty("kotlin_dist")}/kotlinc/lib/kotlin-daemon.jar"
)
)
addTimeListener(this)
extensions.create(benchmarkExtensionName, benchmarkExtensionClass.java, this)
configureMPPExtension(this)
configureTasks()
}
companion object {
const val NATIVE_TARGET_NAME = "native"
const val NATIVE_EXECUTABLE_NAME = "benchmark"
const val BENCHMARKING_GROUP = "benchmarking"
}
}
@@ -0,0 +1,145 @@
package org.jetbrains.kotlin.benchmark
import groovy.lang.Closure
import org.gradle.api.*
import org.gradle.api.tasks.Delete
import org.gradle.api.tasks.Exec
import org.jetbrains.kotlin.*
import javax.inject.Inject
class BuildStep (private val _name: String): Named {
override fun getName(): String = _name
lateinit var command: List<String>
fun command(vararg command: String) {
this.command = command.toList()
}
}
class BuildStepContainer(val project: Project): NamedDomainObjectContainer<BuildStep> by project.container(BuildStep::class.java) {
fun step(name: String, configure: Action<BuildStep>) =
maybeCreate(name).apply { configure.execute(this) }
fun step(name: String, configure: Closure<Unit>) =
step(name, { project.configure(this, configure) })
}
open class CompileBenchmarkExtension @Inject constructor(val project: Project) {
var applicationName = project.name
var repeatNumber: Int = 1
var buildSteps: BuildStepContainer = BuildStepContainer(project)
var compilerOpts: List<String> = emptyList()
fun buildSteps(configure: Action<BuildStepContainer>): Unit = buildSteps.let { configure.execute(it) }
fun buildSteps(configure: Closure<Unit>): Unit = buildSteps { project.configure(this, configure) }
}
open class CompileBenchmarkingPlugin : Plugin<Project> {
private val exitCodes: MutableMap<String, Int> = mutableMapOf()
private fun Project.configureUtilityTasks() {
tasks.create("configureBuild") {
doLast { mkdir(buildDir) }
}
tasks.create("clean", Delete::class.java) {
delete(buildDir)
}
}
private fun Project.configureKonanRun(
benchmarkExtension: CompileBenchmarkExtension
): Unit = with(benchmarkExtension) {
// Aggregate task.
val konanRun = tasks.create("konanRun") {
dependsOn("configureBuild")
group = BenchmarkingPlugin.BENCHMARKING_GROUP
description = "Runs the compile only benchmark for Kotlin/Native."
}
// Compile tasks.
afterEvaluate {
for (number in 1..repeatNumber) {
buildSteps.forEach { step ->
val taskName = step.name
tasks.create("$taskName$number", Exec::class.java).apply {
commandLine(step.command)
isIgnoreExitValue = true
konanRun.dependsOn(this)
doLast {
exitCodes[name] = executionResult.get().exitValue
}
}
}
}
}
// Report task.
tasks.create("konanJsonReport").apply {
group = BenchmarkingPlugin.BENCHMARKING_GROUP
description = "Builds the benchmarking report for Kotlin/Native."
doLast {
val nativeCompileTime = getCompileBenchmarkTime(
project,
applicationName,
buildSteps.names,
repeatNumber,
exitCodes
)
val nativeExecutable = buildDir.resolve("program${getNativeProgramExtension()}")
val properties = commonBenchmarkProperties + mapOf(
"type" to "native",
"compilerVersion" to konanVersion,
"benchmarks" to "[]",
"flags" to getCompilerFlags(benchmarkExtension).sorted(),
"compileTime" to nativeCompileTime,
"codeSize" to getCodeSizeBenchmark(applicationName, nativeExecutable.absolutePath)
)
val output = createJsonReport(properties)
buildDir.resolve(nativeJson).writeText(output)
}
konanRun.finalizedBy(this)
}
}
private fun getCompilerFlags(benchmarkExtension: CompileBenchmarkExtension) =
benchmarkExtension.compilerOpts
private fun Project.configureJvmRun() {
val jvmRun = tasks.create("jvmRun") {
group = BenchmarkingPlugin.BENCHMARKING_GROUP
description = "Runs the compile only benchmark for Kotlin/JVM."
doLast { println("JVM run isn't supported") }
}
tasks.create("jvmJsonReport") {
group = BenchmarkingPlugin.BENCHMARKING_GROUP
description = "Builds the benchmarking report for Kotlin/Native."
doLast { println("JVM run isn't supported") }
jvmRun.finalizedBy(this)
}
}
override fun apply(target: Project): Unit = with(target) {
addTimeListener(this)
val benchmarkExtension = extensions.create(
COMPILE_BENCHMARK_EXTENSION_NAME,
CompileBenchmarkExtension::class.java,
this
)
// Create tasks.
configureUtilityTasks()
configureKonanRun(benchmarkExtension)
configureJvmRun()
}
companion object {
const val COMPILE_BENCHMARK_EXTENSION_NAME = "compileBenchmark"
}
}
@@ -0,0 +1,149 @@
package org.jetbrains.kotlin.benchmark
import org.gradle.jvm.tasks.Jar
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Project
import org.gradle.api.Task
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.plugin.mpp.Executable
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
import org.jetbrains.kotlin.konan.target.HostManager
import javax.inject.Inject
import kotlin.reflect.KClass
private val NamedDomainObjectContainer<KotlinSourceSet>.jvmMain
get() = maybeCreate("jvmMain")
private val Project.jvmWarmup: Int
get() = (property("jvmWarmup") as String).toInt()
private val Project.jvmBenchResults: String
get() = property("jvmBenchResults") as String
open class KotlinNativeBenchmarkExtension @Inject constructor(project: Project) : BenchmarkExtension(project) {
var jvmSrcDirs: Collection<Any> = emptyList()
var mingwSrcDirs: Collection<Any> = emptyList()
var posixSrcDirs: Collection<Any> = emptyList()
fun BenchmarkExtension.BenchmarkDependencies.jvm(notation: Any) = sourceSets.jvmMain.dependencies {
implementation(notation)
}
}
/**
* A plugin configuring a benchmark Kotlin/Native project.
*/
open class KotlinNativeBenchmarkingPlugin: BenchmarkingPlugin() {
override fun Project.configureJvmJsonTask(jvmRun: Task): Task {
return tasks.create("jvmJsonReport") {
group = BENCHMARKING_GROUP
description = "Builds the benchmarking report for Kotlin/JVM."
doLast {
val applicationName = benchmark.applicationName
val jarPath = (tasks.getByName("jvmJar") as Jar).archiveFile.get().asFile
val jvmCompileTime = getJvmCompileTime(project, applicationName)
val benchContents = buildDir.resolve(jvmBenchResults).readText()
val properties: Map<String, Any> = commonBenchmarkProperties + mapOf(
"type" to "jvm",
"compilerVersion" to kotlinVersion,
"benchmarks" to benchContents,
"compileTime" to listOf(jvmCompileTime),
"codeSize" to getCodeSizeBenchmark(applicationName, jarPath.absolutePath)
)
val output = createJsonReport(properties)
buildDir.resolve(jvmJson).writeText(output)
}
jvmRun.finalizedBy(this)
}
}
override fun Project.configureJvmTask(): Task {
return tasks.create("jvmRun", RunJvmTask::class.java) {
dependsOn("jvmJar")
val mainCompilation = kotlin.jvm().compilations.getByName("main")
val runtimeDependencies = configurations.getByName(mainCompilation.runtimeDependencyConfigurationName)
classpath(files(mainCompilation.output.allOutputs, runtimeDependencies))
mainClass.set("MainKt")
group = BENCHMARKING_GROUP
description = "Runs the benchmark for Kotlin/JVM."
// Specify settings configured by a user in the benchmark extension.
afterEvaluate {
args("-p", "${benchmark.applicationName}::")
warmupCount = jvmWarmup
repeatCount = attempts
outputFileName = buildDir.resolve(jvmBenchResults).absolutePath
repeatingType = benchmark.repeatingType
}
}
}
override val benchmarkExtensionClass: KClass<*>
get() = KotlinNativeBenchmarkExtension::class
override val Project.benchmark: KotlinNativeBenchmarkExtension
get() = extensions.getByName(benchmarkExtensionName) as KotlinNativeBenchmarkExtension
override val benchmarkExtensionName: String = "benchmark"
private val Project.nativeBinary: Executable
get() = (kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget)
.binaries.getExecutable(NATIVE_EXECUTABLE_NAME, benchmark.buildType)
override val Project.nativeExecutable: String
get() = nativeBinary.outputFile.absolutePath
override val Project.nativeLinkTask: Task
get() = nativeBinary.linkTask
override fun configureMPPExtension(project: Project) {
super.configureMPPExtension(project)
project.configureJVMTarget()
}
override fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) =
super.getCompilerFlags(project, nativeTarget) + project.nativeBinary.freeCompilerArgs.map { "\"$it\"" }
override fun NamedDomainObjectContainer<KotlinSourceSet>.configureSources(project: Project) {
project.benchmark.let {
commonMain.kotlin.srcDirs(*it.commonSrcDirs.toTypedArray())
if (HostManager.hostIsMingw) {
nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.mingwSrcDirs).toTypedArray())
} else {
nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.posixSrcDirs).toTypedArray())
}
jvmMain.kotlin.srcDirs(*it.jvmSrcDirs.toTypedArray())
}
}
override fun NamedDomainObjectContainer<KotlinSourceSet>.additionalConfigurations(project: Project) {
jvmMain.dependencies {
implementation(project.files("${project.findProperty("kotlin_dist")}/kotlinc/lib/kotlin-stdlib-jdk8.jar"))
if (project.hasProperty("kotlin_dist"))
implementation(project(":endorsedLibraries:kotlinx.cli"))
}
}
private fun Project.configureJVMTarget() {
kotlin.jvm {
compilations.all {
@Suppress("DEPRECATION")
compileKotlinTask.kotlinOptions {
jvmTarget = "1.8"
suppressWarnings = true
freeCompilerArgs = project.benchmark.compilerOpts + project.compilerArgs
}
}
}
}
companion object {
const val BENCHMARK_EXTENSION_NAME = "benchmark"
}
}
@@ -0,0 +1,110 @@
package org.jetbrains.kotlin.benchmark
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Project
import org.gradle.api.Task
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.plugin.mpp.Framework
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinNativeTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
import java.io.File
import javax.inject.Inject
import java.nio.file.Paths
import kotlin.reflect.KClass
enum class CodeSizeEntity { FRAMEWORK, EXECUTABLE }
open class SwiftBenchmarkExtension @Inject constructor(project: Project) : BenchmarkExtension(project) {
var swiftSources: List<String> = emptyList()
var useCodeSize: CodeSizeEntity = CodeSizeEntity.FRAMEWORK // use as code size metric framework size or executable
}
/**
* A plugin configuring a benchmark Kotlin/Native project.
*/
open class SwiftBenchmarkingPlugin : BenchmarkingPlugin() {
override fun Project.configureJvmJsonTask(jvmRun: Task): Task {
return tasks.create("jvmJsonReport") {
logger.info("JVM run is unsupported")
jvmRun.finalizedBy(this)
}
}
override fun Project.configureJvmTask(): Task {
return tasks.create("jvmRun") {
doLast {
logger.info("JVM run is unsupported")
}
}
}
override val benchmarkExtensionClass: KClass<*>
get() = SwiftBenchmarkExtension::class
override val Project.benchmark: SwiftBenchmarkExtension
get() = extensions.getByName(benchmarkExtensionName) as SwiftBenchmarkExtension
override val benchmarkExtensionName: String = "swiftBenchmark"
override val Project.nativeExecutable: String
get() = Paths.get(buildDir.absolutePath, benchmark.applicationName).toString()
override val Project.nativeLinkTask: Task
get() = tasks.getByName("buildSwift")
private lateinit var framework: Framework
val nativeFrameworkName = "benchmark"
override fun NamedDomainObjectContainer<KotlinSourceSet>.configureSources(project: Project) {
project.benchmark.let {
commonMain.kotlin.srcDirs(*it.commonSrcDirs.toTypedArray())
nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs).toTypedArray())
}
}
override fun Project.determinePreset(): AbstractKotlinNativeTargetPreset<*> =
defaultHostPreset(this).also { preset ->
logger.quiet("$project has been configured for ${preset.name} platform.")
} as AbstractKotlinNativeTargetPreset<*>
override fun KotlinNativeTarget.configureNativeOutput(project: Project) {
binaries.framework(nativeFrameworkName, listOf(project.benchmark.buildType)) {
// Specify settings configured by a user in the benchmark extension.
project.afterEvaluate {
linkerOpts.addAll(project.benchmark.linkerOpts)
}
}
}
override fun Project.configureExtraTasks() {
val nativeTarget = kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget
// Build executable from swift code.
framework = nativeTarget.binaries.getFramework(nativeFrameworkName, benchmark.buildType)
tasks.create("buildSwift") {
dependsOn(framework.linkTaskName)
doLast {
val frameworkParentDirPath = framework.outputDirectory.absolutePath
val options = listOf("-O", "-wmo", "-Xlinker", "-rpath", "-Xlinker", frameworkParentDirPath, "-F", frameworkParentDirPath)
compileSwift(project, nativeTarget.konanTarget, benchmark.swiftSources, options,
Paths.get(buildDir.absolutePath, benchmark.applicationName), false)
}
}
}
override fun Project.collectCodeSize(applicationName: String) =
getCodeSizeBenchmark(applicationName,
if (benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK)
File("${framework.outputFile.absolutePath}/$nativeFrameworkName").canonicalPath
else
nativeExecutable
)
override fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) =
if (project.benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK) {
super.getCompilerFlags(project, nativeTarget) + framework.freeCompilerArgs.map { "\"$it\"" }
} else {
listOf("-O", "-wmo")
}
}
@@ -0,0 +1,42 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.jetbrains.report.json.*
// List of commits.
class CommitsList(data: JsonElement): ConvertedFromJson {
val commits: List<Commit>
init {
if (data !is JsonObject) {
error("Commits description is expected to be a json object!")
}
val changesElement = data.getOptionalField("change")
commits = changesElement?.let {
if (changesElement !is JsonArray) {
error("Change field is expected to be an array. Please, check source.")
}
changesElement.jsonArray.map {
with(it as JsonObject) {
Commit(elementToString(getRequiredField("version"), "version"),
elementToString(getRequiredField("username"), "username"),
elementToString(getRequiredField("webUrl"), "webUrl")
)
}
}
} ?: listOf<Commit>()
}
}
fun getBuildProperty(buildJsonDescription: String, property: String) =
with(JsonTreeParser.parse(buildJsonDescription) as JsonObject) {
if (getPrimitive("count").int == 0) {
error("No build information on TeamCity for $buildJsonDescription!")
}
(getArray("build").getObject(0).getPrimitive(property) as JsonLiteral).unquoted()
}