diff --git a/performance/build.gradle b/performance/build.gradle index b5a4e084b64..45bc74c10d3 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -32,6 +32,8 @@ repositories { } +defaultTasks 'bench' + private def determinePreset() { def preset = MPPTools.defaultHostPreset(project) println("$project has been configured for ${preset.name} platform.") @@ -63,7 +65,6 @@ kotlin { targets { fromPreset(presets.jvm, 'jvm') { - def mainOutput = compilations.main.output compilations.all { tasks[compileKotlinTaskName].kotlinOptions { jvmTarget = '1.8' @@ -80,12 +81,13 @@ kotlin { } } +MPPTools.addTimeListener(project) + MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) { workingDir = project.provider { kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile } args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}") - outputFileName("${buildDir.absolutePath}/${nativeTextReport}") } task jvmRun(type: JavaExec) { @@ -98,19 +100,17 @@ task jvmRun(type: JavaExec) { classpath runtimeClasspath main = "MainKt" args "$jvmWarmup", "$attempts", "${buildDir.absolutePath}/${jvmBenchResults}" - standardOutput = output - doLast { - dumpReport("${buildDir.absolutePath}/${jvmTextReport}", output) - } } task konanJsonReport { doLast { + def nativeCompileTime = MPPTools.getNativeCompileTime('Ring') String benchContents = new File("${buildDir.absolutePath}/${nativeBenchResults}").text def properties = getCommonProperties() + ['type' : 'native', 'compilerVersion': "${konanVersion}".toString(), 'flags' : kotlin.targets.native.compilations.main.extraOpts.collect{ '"' + it + '"'}, - 'benchmarks' : benchContents] + 'benchmarks' : benchContents, + 'compileTime' : nativeCompileTime] def output = MPPTools.createJsonReport(properties) new File("${buildDir.absolutePath}/${nativeJson}").write(output) } @@ -118,11 +118,12 @@ task konanJsonReport { task jvmJsonReport { doLast { + def jvmCompileTime = MPPTools.getJvmCompileTime('Ring') String benchContents = new File("${buildDir.absolutePath}/${jvmBenchResults}").text def properties = getCommonProperties() + ['type' : 'jvm', 'compilerVersion': "${buildKotlinVersion}".toString(), - 'benchmarks' : benchContents] - println(properties['compilerVersion']) + 'benchmarks' : benchContents, + 'compileTime' : jvmCompileTime] def output = MPPTools.createJsonReport(properties) new File("${buildDir.absolutePath}/${jvmJson}").write(output) } @@ -145,58 +146,14 @@ private def getCommonProperties() { 'kotlinVersion': "${kotlinVersion}".toString()] } -task bench(type:DefaultTask) { +task bench(type:Exec) { dependsOn jvmRun dependsOn konanRun - - doLast { - def jvmReport = new Report(project.file("${buildDir.absolutePath}/${jvmTextReport}")) - def konanReport = new Report(project.file("${buildDir.absolutePath}/${nativeTextReport}")) - def average = "none" - def absoluteAverage = "none" - jvmReport.report - .sort { konanReport.report[it.key].mean / it.value.mean } - .each { k, v -> - def konanValue = konanReport.report[k] - def ratio = konanValue.mean / v.mean - // This is a hack since neither mean nor variance of ratio of two distributions is known. - def minRatio = (konanValue.mean - konanValue.stdDev) / (v.mean + v.stdDev) - def maxRatio = (konanValue.mean + konanValue.stdDev) / (v.mean - v.stdDev) - def ratioConfInt = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio)) - def formattedKonanValue = String.format('%.4f us +- %.4f us', konanValue.mean / 1000, konanValue.stdDev / 1000) - def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioConfInt) - if (k == 'RingAverage') { - average = formattedRatio - absoluteAverage = formattedKonanValue - } else { - println("$k : absolute = $formattedKonanValue, ratio = $formattedRatio") - } - if (System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") != null) - println("##teamcity[buildStatisticValue key='$k' value='$ratio']") - } - - println() - println("Average Ring score: absolute = $absoluteAverage, ratio = $average") + def extension = MPPTools.getNativeProgramExtension() + def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}") + if (analyzer != null) { + commandLine "${analyzer}", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}" + } else { + println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}") } -} - -class Results { - def Double mean - def Double stdDev - - Results(Double mean, Double stdDev) { - this.mean = mean - this.stdDev = stdDev - } -} - -class Report { - Map report = new TreeMap() - - Report(File path) { - path.readLines().drop(3).findAll { it.split(':').length == 3 }.each { - def p = it.split(':') - report.put(p[0].trim(), new Results(Double.parseDouble(p[1].trim()), Double.parseDouble(p[2].trim()))) - } - } -} +} \ No newline at end of file diff --git a/performance/buildSrc/src/main/kotlin/MPPTools.kt b/performance/buildSrc/src/main/kotlin/MPPTools.kt index 15095964b94..d0086b73d43 100644 --- a/performance/buildSrc/src/main/kotlin/MPPTools.kt +++ b/performance/buildSrc/src/main/kotlin/MPPTools.kt @@ -8,11 +8,14 @@ import groovy.lang.Closure import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.tasks.TaskState +import org.gradle.api.execution.TaskExecutionListener import org.jetbrains.kotlin.gradle.plugin.KotlinTarget import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset import org.jetbrains.report.* import org.jetbrains.report.json.* import java.nio.file.Paths +import java.io.File /* * This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future. @@ -60,9 +63,16 @@ fun defaultHostPreset( throw Exception("Host OS '$hostOs' is not supported in Kotlin/Native ${subproject.displayName}.") } +fun getNativeProgramExtension(): String = when { + isMacos -> ".kexe" + isLinux -> ".kexe" + isWindows -> ".exe" + else -> error("Unknown host") +} + // Create benchmarks json report based on information get from gradle project fun createJsonReport(projectProperties: Map): String { - fun getValue(key: String): String = projectProperties[key] as? String ?: "uknown" + fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown" val machine = Environment.Machine(getValue("cpu"), getValue("os")) val jdk = Environment.JDKInstance(getValue("jdkVersion"), getValue("jdkVendor")) val env = Environment(machine, jdk) @@ -73,10 +83,15 @@ fun createJsonReport(projectProperties: Map): String { val benchDesc = getValue("benchmarks") val benchmarksArray = JsonTreeParser.parse(benchDesc) val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray) + .union(listOf(projectProperties["compileTime"] as BenchmarkResult)).toList() val report = BenchmarksReport(env, benchmarks, kotlin) return report.toJson() } +// Find file with set name in directory. +fun findFile(fileName: String, directory: String): String? = + File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath() + // A short-cut to add a Kotlin/Native run task. @JvmOverloads fun createRunTask( @@ -89,3 +104,40 @@ fun createRunTask( task.configure(configureClosure ?: task.emptyConfigureClosure()) return task } + +fun getJvmCompileTime(programName: String): BenchmarkResult = + TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar")) + +fun getNativeCompileTime(programName: String): BenchmarkResult = + TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkReleaseExecutableNative")) + +// Class time tracker for all tasks. +class TaskTimerListener: TaskExecutionListener { + companion object { + val tasksTimes = mutableMapOf() + + fun getBenchmarkResult(programName: String, tasksNames: List): BenchmarkResult { + val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum() + // TODO get this info from gradle plugin with exit code end stacktrace. + val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b } + return BenchmarkResult("$programName.compileTime", + if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED, + time, time, 1, 0) + } + + } + + private var startTime = System.currentTimeMillis() + + override fun beforeExecute(task: Task) { + startTime = System.nanoTime() + } + + override fun afterExecute(task: Task, taskState: TaskState) { + tasksTimes[task.name] = (System.nanoTime() - startTime) / 1000.0 + } +} + +fun addTimeListener(subproject: Project) { + subproject.gradle.addListener(TaskTimerListener()) +} diff --git a/performance/gradle.properties b/performance/gradle.properties index 7d10881186f..3a760731811 100644 --- a/performance/gradle.properties +++ b/performance/gradle.properties @@ -8,4 +8,5 @@ nativeTextReport = nativeReport.txt jvmTextReport = jvmReport.txt nativeJson = nativeReport.json jvmJson = jvmReport.json - +analyzerTool = benchmarksAnalyzer +analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin \ No newline at end of file diff --git a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt index ae448972622..971291d0fd9 100644 --- a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt +++ b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt @@ -27,7 +27,6 @@ val BENCHMARK_SIZE = 100 class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) { class Results(val mean: Double, val variance: Double) - val results = mutableMapOf() val benchmarkResults = mutableListOf() fun launch(benchmark: () -> Any?, name: String) { // If benchmark runs too long - use coeff to speed it up. @@ -66,10 +65,6 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) { scaledTime / 1000, scaledTime / 1000, k + 1, numWarmIterations)) } - val mean = samples.sum() / numberOfAttempts - val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / numberOfAttempts - - results[name] = Results(mean, variance) } //-------------------------------------------------------------------------// @@ -101,44 +96,11 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) { runWithIndiciesBenchmark() runOctoTest() - printResultsNormalized() return benchmarkResults } //-------------------------------------------------------------------------// - fun printResultsAsTime() { - results.forEach { - val niceName = "\"${it.key}\"".padEnd(51) - val niceTime = "${it.value}".padStart(10) - println(" $niceName to ${niceTime}L,") - } - } - - //-------------------------------------------------------------------------// - - private val zStar = 1.96 // For 95% confidence interval. - - fun printResultsNormalized() { - var totalMean = 0.0 - var totalVariance = 0.0 - results.asSequence().sortedBy { it.key }.forEach { - val niceName = it.key.padEnd(50, ' ') - val mean = it.value.mean - val variance = it.value.variance - val confidenceInterval = sqrt(variance / numberOfAttempts) * zStar - println("$niceName : ${mean.toString(9)} : ${confidenceInterval.toString(9)}") - - totalMean += mean - totalVariance += variance - } - val averageMean = totalMean / results.size - val averageConfidenceInterval = sqrt(totalVariance / numberOfAttempts) * zStar / results.size - println("\nRingAverage: ${averageMean.toString(9)} : ${averageConfidenceInterval.toString(9)}") - } - - //-------------------------------------------------------------------------// - fun runAbstractMethodBenchmark() { val benchmark = AbstractMethodBenchmark() @@ -493,17 +455,4 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) { fun runOctoTest() { launch(::octoTest, "OctoTest") } - - //-------------------------------------------------------------------------// - - fun Double.toString(n: Int): String { - val str = this.toString() - if (str.contains('e', ignoreCase = true)) return str - - val len = str.length - val pointIdx = str.indexOf('.') - val dropCnt = len - pointIdx - n - 1 - if (dropCnt < 1) return str - return str.dropLast(dropCnt) - } } \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt index 5e42e95a0a3..17de976d291 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt @@ -122,4 +122,9 @@ fun collectMeanResults(benchmarks: Map>): Benchmar currentWarmup) name to MeanVarianceBenchmark(meanBenchmark, varianceBenchmark) }.toMap() -} \ No newline at end of file +} + +fun collectBenchmarksDurations(benchmarks: Map>): Map = + benchmarks.map { (name, resultsSet) -> + name to resultsSet.sumByDouble { it.runtimeInUs } + }.toMap() \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt index 810fd98a0ce..83b7553cd28 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt @@ -34,6 +34,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, val meaningfulChangesValue: Double = 0.5) { // Report created by joining comparing reports. val mergedReport: Map + val benchmarksDurations: Map> // Lists of benchmarks in different status. private val benchmarksWithChangedStatus = mutableListOf>() @@ -70,6 +71,9 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, val currentMeanVarianceBenchmarks: List get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! } + val currentBenchmarksDuration: Map + get() = benchmarksDurations.filter{ it.value.first != null }.map { it.key to it.value.first!! }.toMap() + init { // Count avarage values for each benchmark. val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks) @@ -77,6 +81,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, collectMeanResults(previousReport.benchmarks) } mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable) + benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport) geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable) if (previousReport != null) { // Check changes in environment and tools. @@ -101,6 +106,17 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, return MeanVarianceBenchmark(meanBenchmark, varianceBenchmark) } + // Generate map with summary durations of each benchmark. + private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?): + Map> { + val currentDurations = collectBenchmarksDurations(currentReport.benchmarks) + val previousDurations = previousReport?.let { + collectBenchmarksDurations(previousReport.benchmarks) + } ?: mapOf() + return currentDurations.keys.union(previousDurations.keys) + .map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap() + } + // Merge current and compare to report. private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?): Map { diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt index b6f886ebd37..8e6bcabbf3d 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt @@ -25,20 +25,40 @@ class TeamCityStatisticsRender: Render { private var content = StringBuilder() override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { + val currentDurations = report.currentBenchmarksDuration + + content.append("##teamcity[testSuiteStarted name='Benchmarks']\n") + // For current benchmarks print score as TeamCity Test Metadata report.currentMeanVarianceBenchmarks.forEach { benchmark -> - content.append(renderBenchmark(benchmark.meanBenchmark, "Mean")) - content.append(renderBenchmark(benchmark.varianceBenchmark, "Variance")) + renderBenchmark(benchmark.meanBenchmark, currentDurations[benchmark.meanBenchmark.name]!!) + renderSummaryBecnhmarkValue(benchmark.meanBenchmark, "Mean") + renderSummaryBecnhmarkValue(benchmark.varianceBenchmark, "Variance") } + content.append("##teamcity[testSuiteFinished name='Benchmarks']\n") + // Report geometric mean as build statistic value - content.append(renderGeometricMean(report.geoMeanBenchmark.first!!)) + renderGeometricMean(report.geoMeanBenchmark.first!!) + return content.toString() } - private fun renderBenchmark(benchmark: BenchmarkResult, metric: String): String = - "##teamcity[testMetadata testName='${benchmark.name}' name='$metric' type='number' value='${benchmark.score}']\n" + private fun renderSummaryBecnhmarkValue(benchmark: BenchmarkResult, metric: String) = + content.append("##teamcity[testMetadata testName='${benchmark.name}' name='$metric'" + + " type='number' value='${benchmark.score}']\n") - private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark): String = - "##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n" + - "##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n" + // Produce benchmark as test in TeamCity + private fun renderBenchmark(benchmark: BenchmarkResult , duration: Double) { + content.append("##teamcity[testStarted name='${benchmark.name}']\n") + if (benchmark.status == BenchmarkResult.Status.FAILED) { + content.append("##teamcity[testFailed name='${benchmark.name}']\n") + } + // test_duration_in_milliseconds is set for TeamCity + content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n") + } + + private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) { + content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n") + content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n") + } } \ No newline at end of file