azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added usage of benchmarksAnalyzer for analysis of performance, so removed extra code from benchmark application and gradle. (#2501)

Browse Source
This commit is contained in:
LepilkinaElena
2019-01-09 13:30:17 +03:00
committed by GitHub
parent c7c566ce0f
commit 74b42b4370
7 changed files with 123 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
performance/build.gradle
+18 -61
View File
@@ -32,6 +32,8 @@ repositories {
}
defaultTasks 'bench'
private def determinePreset() {
def preset = MPPTools.defaultHostPreset(project)
println("$project has been configured for ${preset.name} platform.")
@@ -63,7 +65,6 @@ kotlin {
targets {
fromPreset(presets.jvm, 'jvm') {
def mainOutput = compilations.main.output
compilations.all {
tasks[compileKotlinTaskName].kotlinOptions {
jvmTarget = '1.8'
@@ -80,12 +81,13 @@ kotlin {
}
}
MPPTools.addTimeListener(project)
MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) {
workingDir = project.provider {
kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile
}
args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}")
outputFileName("${buildDir.absolutePath}/${nativeTextReport}")
}
task jvmRun(type: JavaExec) {
@@ -98,19 +100,17 @@ task jvmRun(type: JavaExec) {
classpath runtimeClasspath
main = "MainKt"
args "$jvmWarmup", "$attempts", "${buildDir.absolutePath}/${jvmBenchResults}"
standardOutput = output
doLast {
dumpReport("${buildDir.absolutePath}/${jvmTextReport}", output)
}
}
task konanJsonReport {
doLast {
def nativeCompileTime = MPPTools.getNativeCompileTime('Ring')
String benchContents = new File("${buildDir.absolutePath}/${nativeBenchResults}").text
def properties = getCommonProperties() + ['type' : 'native',
'compilerVersion': "${konanVersion}".toString(),
'flags' : kotlin.targets.native.compilations.main.extraOpts.collect{ '"' + it + '"'},
'benchmarks' : benchContents]
'benchmarks' : benchContents,
'compileTime' : nativeCompileTime]
def output = MPPTools.createJsonReport(properties)
new File("${buildDir.absolutePath}/${nativeJson}").write(output)
}
@@ -118,11 +118,12 @@ task konanJsonReport {
task jvmJsonReport {
doLast {
def jvmCompileTime = MPPTools.getJvmCompileTime('Ring')
String benchContents = new File("${buildDir.absolutePath}/${jvmBenchResults}").text
def properties = getCommonProperties() + ['type' : 'jvm',
'compilerVersion': "${buildKotlinVersion}".toString(),
'benchmarks' : benchContents]
println(properties['compilerVersion'])
'benchmarks' : benchContents,
'compileTime' : jvmCompileTime]
def output = MPPTools.createJsonReport(properties)
new File("${buildDir.absolutePath}/${jvmJson}").write(output)
}
@@ -145,58 +146,14 @@ private def getCommonProperties() {
'kotlinVersion': "${kotlinVersion}".toString()]
}
task bench(type:DefaultTask) {
task bench(type:Exec) {
dependsOn jvmRun
dependsOn konanRun
doLast {
def jvmReport = new Report(project.file("${buildDir.absolutePath}/${jvmTextReport}"))
def konanReport = new Report(project.file("${buildDir.absolutePath}/${nativeTextReport}"))
def average = "none"
def absoluteAverage = "none"
jvmReport.report
.sort { konanReport.report[it.key].mean / it.value.mean }
.each { k, v ->
def konanValue = konanReport.report[k]
def ratio = konanValue.mean / v.mean
// This is a hack since neither mean nor variance of ratio of two distributions is known.
def minRatio = (konanValue.mean - konanValue.stdDev) / (v.mean + v.stdDev)
def maxRatio = (konanValue.mean + konanValue.stdDev) / (v.mean - v.stdDev)
def ratioConfInt = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio))
def formattedKonanValue = String.format('%.4f us +- %.4f us', konanValue.mean / 1000, konanValue.stdDev / 1000)
def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioConfInt)
if (k == 'RingAverage') {
average = formattedRatio
absoluteAverage = formattedKonanValue
} else {
println("$k : absolute = $formattedKonanValue, ratio = $formattedRatio")
}
if (System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") != null)
println("##teamcity[buildStatisticValue key='$k' value='$ratio']")
}
println()
println("Average Ring score: absolute = $absoluteAverage, ratio = $average")
def extension = MPPTools.getNativeProgramExtension()
def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}")
if (analyzer != null) {
commandLine "${analyzer}", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}"
} else {
println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}")
}
}
class Results {
def Double mean
def Double stdDev
Results(Double mean, Double stdDev) {
this.mean = mean
this.stdDev = stdDev
}
}
class Report {
Map<String, Results> report = new TreeMap()
Report(File path) {
path.readLines().drop(3).findAll { it.split(':').length == 3 }.each {
def p = it.split(':')
report.put(p[0].trim(), new Results(Double.parseDouble(p[1].trim()), Double.parseDouble(p[2].trim())))
}
}
}
}
performance/buildSrc/src/main/kotlin/MPPTools.kt
+53 -1
View File
@@ -8,11 +8,14 @@
import groovy.lang.Closure
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.TaskState
import org.gradle.api.execution.TaskExecutionListener
import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
import org.jetbrains.report.*
import org.jetbrains.report.json.*
import java.nio.file.Paths
import java.io.File
/*
* This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future.
@@ -60,9 +63,16 @@ fun defaultHostPreset(
throw Exception("Host OS '$hostOs' is not supported in Kotlin/Native ${subproject.displayName}.")
}
fun getNativeProgramExtension(): String = when {
isMacos -> ".kexe"
isLinux -> ".kexe"
isWindows -> ".exe"
else -> error("Unknown host")
}
// Create benchmarks json report based on information get from gradle project
fun createJsonReport(projectProperties: Map<String, Any>): String {
fun getValue(key: String): String = projectProperties[key] as? String ?: "uknown"
fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown"
val machine = Environment.Machine(getValue("cpu"), getValue("os"))
val jdk = Environment.JDKInstance(getValue("jdkVersion"), getValue("jdkVendor"))
val env = Environment(machine, jdk)
@@ -73,10 +83,15 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
val benchDesc = getValue("benchmarks")
val benchmarksArray = JsonTreeParser.parse(benchDesc)
val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray)
.union(listOf<BenchmarkResult>(projectProperties["compileTime"] as BenchmarkResult)).toList()
val report = BenchmarksReport(env, benchmarks, kotlin)
return report.toJson()
}
// Find file with set name in directory.
fun findFile(fileName: String, directory: String): String? =
File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()
// A short-cut to add a Kotlin/Native run task.
@JvmOverloads
fun createRunTask(
@@ -89,3 +104,40 @@ fun createRunTask(
task.configure(configureClosure ?: task.emptyConfigureClosure())
return task
}
fun getJvmCompileTime(programName: String): BenchmarkResult =
TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
fun getNativeCompileTime(programName: String): BenchmarkResult =
TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkReleaseExecutableNative"))
// Class time tracker for all tasks.
class TaskTimerListener: TaskExecutionListener {
companion object {
val tasksTimes = mutableMapOf<String, Double>()
fun getBenchmarkResult(programName: String, tasksNames: List<String>): BenchmarkResult {
val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum()
// TODO get this info from gradle plugin with exit code end stacktrace.
val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b }
return BenchmarkResult("$programName.compileTime",
if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
time, time, 1, 0)
}
}
private var startTime = System.currentTimeMillis()
override fun beforeExecute(task: Task) {
startTime = System.nanoTime()
}
override fun afterExecute(task: Task, taskState: TaskState) {
tasksTimes[task.name] = (System.nanoTime() - startTime) / 1000.0
}
}
fun addTimeListener(subproject: Project) {
subproject.gradle.addListener(TaskTimerListener())
}
performance/gradle.properties
+2 -1
View File
@@ -8,4 +8,5 @@ nativeTextReport = nativeReport.txt
jvmTextReport = jvmReport.txt
nativeJson = nativeReport.json
jvmJson = jvmReport.json
analyzerTool = benchmarksAnalyzer
analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin
performance/src/main/kotlin/org/jetbrains/ring/launcher.kt
-51
View File
@@ -27,7 +27,6 @@ val BENCHMARK_SIZE = 100
class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
class Results(val mean: Double, val variance: Double)
val results = mutableMapOf<String, Results>()
val benchmarkResults = mutableListOf<BenchmarkResult>()
fun launch(benchmark: () -> Any?, name: String) { // If benchmark runs too long - use coeff to speed it up.
@@ -66,10 +65,6 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
scaledTime / 1000, scaledTime / 1000,
k + 1, numWarmIterations))
}
val mean = samples.sum() / numberOfAttempts
val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / numberOfAttempts
results[name] = Results(mean, variance)
}
//-------------------------------------------------------------------------//
@@ -101,44 +96,11 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
runWithIndiciesBenchmark()
runOctoTest()
printResultsNormalized()
return benchmarkResults
}
//-------------------------------------------------------------------------//
fun printResultsAsTime() {
results.forEach {
val niceName = "\"${it.key}\"".padEnd(51)
val niceTime = "${it.value}".padStart(10)
println(" $niceName to ${niceTime}L,")
}
}
//-------------------------------------------------------------------------//
private val zStar = 1.96 // For 95% confidence interval.
fun printResultsNormalized() {
var totalMean = 0.0
var totalVariance = 0.0
results.asSequence().sortedBy { it.key }.forEach {
val niceName = it.key.padEnd(50, ' ')
val mean = it.value.mean
val variance = it.value.variance
val confidenceInterval = sqrt(variance / numberOfAttempts) * zStar
println("$niceName : ${mean.toString(9)} : ${confidenceInterval.toString(9)}")
totalMean += mean
totalVariance += variance
}
val averageMean = totalMean / results.size
val averageConfidenceInterval = sqrt(totalVariance / numberOfAttempts) * zStar / results.size
println("\nRingAverage: ${averageMean.toString(9)} : ${averageConfidenceInterval.toString(9)}")
}
//-------------------------------------------------------------------------//
fun runAbstractMethodBenchmark() {
val benchmark = AbstractMethodBenchmark()
@@ -493,17 +455,4 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
fun runOctoTest() {
launch(::octoTest, "OctoTest")
}
//-------------------------------------------------------------------------//
fun Double.toString(n: Int): String {
val str = this.toString()
if (str.contains('e', ignoreCase = true)) return str
val len = str.length
val pointIdx = str.indexOf('.')
val dropCnt = len - pointIdx - n - 1
if (dropCnt < 1) return str
return str.dropLast(dropCnt)
}
}
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt
+6 -1
View File
@@ -122,4 +122,9 @@ fun collectMeanResults(benchmarks: Map<String, List<BenchmarkResult>>): Benchmar
currentWarmup)
name to MeanVarianceBenchmark(meanBenchmark, varianceBenchmark)
}.toMap()
}
}
fun collectBenchmarksDurations(benchmarks: Map<String, List<BenchmarkResult>>): Map<String, Double> =
benchmarks.map { (name, resultsSet) ->
name to resultsSet.sumByDouble { it.runtimeInUs }
}.toMap()
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt
+16
View File
@@ -34,6 +34,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
val meaningfulChangesValue: Double = 0.5) {
// Report created by joining comparing reports.
val mergedReport: Map<String, SummaryBenchmark>
val benchmarksDurations: Map<String, Pair<Double?, Double?>>
// Lists of benchmarks in different status.
private val benchmarksWithChangedStatus = mutableListOf<FieldChange<BenchmarkResult.Status>>()
@@ -70,6 +71,9 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! }
val currentBenchmarksDuration: Map<String, Double>
get() = benchmarksDurations.filter{ it.value.first != null }.map { it.key to it.value.first!! }.toMap()
init {
// Count avarage values for each benchmark.
val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks)
@@ -77,6 +81,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
collectMeanResults(previousReport.benchmarks)
}
mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable)
benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable)
if (previousReport != null) {
// Check changes in environment and tools.
@@ -101,6 +106,17 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
return MeanVarianceBenchmark(meanBenchmark, varianceBenchmark)
}
// Generate map with summary durations of each benchmark.
private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?):
Map<String, Pair<Double?, Double?>> {
val currentDurations = collectBenchmarksDurations(currentReport.benchmarks)
val previousDurations = previousReport?.let {
collectBenchmarksDurations(previousReport.benchmarks)
} ?: mapOf<String, Double>()
return currentDurations.keys.union(previousDurations.keys)
.map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap()
}
// Merge current and compare to report.
private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?):
Map<String, SummaryBenchmark> {
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt
+28 -8
View File
@@ -25,20 +25,40 @@ class TeamCityStatisticsRender: Render {
private var content = StringBuilder()
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val currentDurations = report.currentBenchmarksDuration
content.append("##teamcity[testSuiteStarted name='Benchmarks']\n")
// For current benchmarks print score as TeamCity Test Metadata
report.currentMeanVarianceBenchmarks.forEach { benchmark ->
content.append(renderBenchmark(benchmark.meanBenchmark, "Mean"))
content.append(renderBenchmark(benchmark.varianceBenchmark, "Variance"))
renderBenchmark(benchmark.meanBenchmark, currentDurations[benchmark.meanBenchmark.name]!!)
renderSummaryBecnhmarkValue(benchmark.meanBenchmark, "Mean")
renderSummaryBecnhmarkValue(benchmark.varianceBenchmark, "Variance")
}
content.append("##teamcity[testSuiteFinished name='Benchmarks']\n")
// Report geometric mean as build statistic value
content.append(renderGeometricMean(report.geoMeanBenchmark.first!!))
renderGeometricMean(report.geoMeanBenchmark.first!!)
return content.toString()
}
private fun renderBenchmark(benchmark: BenchmarkResult, metric: String): String =
"##teamcity[testMetadata testName='${benchmark.name}' name='$metric' type='number' value='${benchmark.score}']\n"
private fun renderSummaryBecnhmarkValue(benchmark: BenchmarkResult, metric: String) =
content.append("##teamcity[testMetadata testName='${benchmark.name}' name='$metric'" +
" type='number' value='${benchmark.score}']\n")
private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark): String =
"##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n" +
"##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n"
// Produce benchmark as test in TeamCity
private fun renderBenchmark(benchmark: BenchmarkResult , duration: Double) {
content.append("##teamcity[testStarted name='${benchmark.name}']\n")
if (benchmark.status == BenchmarkResult.Status.FAILED) {
content.append("##teamcity[testFailed name='${benchmark.name}']\n")
}
// test_duration_in_milliseconds is set for TeamCity
content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n")
}
private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) {
content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n")
content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n")
}
}