diff --git a/build.gradle b/build.gradle index f7c03f33f30..d07c589338c 100644 --- a/build.gradle +++ b/build.gradle @@ -462,10 +462,11 @@ task uploadBundle { } } -task performance { +// TODO return when problems with HostManagers will be solved +/*task performance { dependsOn 'dist' dependsOn ':performance:bench' -} +}*/ task teamcityKonanVersion { doLast { diff --git a/settings.gradle b/settings.gradle index 9033cf52713..67086c3c6b9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -29,6 +29,7 @@ include ':backend.native:tests' include ':backend.native:debugger-tests' include ':utilities' //include ':performance' TODO: return when there is one HostManager class version +//include ':tools:benchmarksAnalyzer' include ':platformLibs' includeBuild 'shared' @@ -36,6 +37,7 @@ includeBuild 'extracted/konan.metadata' includeBuild 'extracted/konan.serializer' includeBuild 'tools/kotlin-native-gradle-plugin' + if (hasProperty("kotlinProjectPath")) { includeBuild(kotlinProjectPath) { dependencySubstitution { diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt index 8448d6b4321..161eaf30d23 100644 --- a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt +++ b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt @@ -79,7 +79,7 @@ interface EntityFromJsonFactory: ConvertedFromJson { } // Class for benchcmarks report with all information of run. -data class BenchmarksReport(val env: Environment, val benchmarks: List, val compiler: Compiler): +class BenchmarksReport(val env: Environment, benchmarksList: List, val compiler: Compiler): JsonSerializable { companion object: EntityFromJsonFactory { @@ -88,8 +88,8 @@ data class BenchmarksReport(val env: Environment, val benchmarks: List) = + benchmarksList.groupBy{ it.name } } + val benchmarks: Map> = structBenchmarks(benchmarksList) + override fun toJson(): String { return """ { "env": ${env.toJson()}, "kotlin": ${compiler.toJson()}, - "benchmarks": ${arrayToJson(benchmarks)} + "benchmarks": ${arrayToJson(benchmarks.flatMap{it.value})} } """ } @@ -279,6 +285,8 @@ class BenchmarkResult(val name: String, val status: Status, val score: Double, val runtimeInUs: Double, val repeat: Int, val warmup: Int): JsonSerializable { + constructor(name: String, score: Double) : this(name, Status.PASSED, score, 0.0, 0, 0) + companion object: EntityFromJsonFactory { override fun create(data: JsonElement): BenchmarkResult { diff --git a/tools/benchmarksAnalyzer/build.gradle b/tools/benchmarksAnalyzer/build.gradle new file mode 100644 index 00000000000..01a5df53bcc --- /dev/null +++ b/tools/benchmarksAnalyzer/build.gradle @@ -0,0 +1,94 @@ +buildscript { + ext.rootBuildDirectory = file('../..') + + apply from: "$rootBuildDirectory/gradle/loadRootProperties.gradle" + apply from: "$rootBuildDirectory/gradle/kotlinGradlePlugin.gradle" + + repositories { + maven { + url 'https://cache-redirector.jetbrains.com/jcenter' + } + maven { + url kotlinCompilerRepo + } + } + + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" + } +} + +apply plugin: 'kotlin-multiplatform' + +repositories { + maven { + url 'https://cache-redirector.jetbrains.com/jcenter' + } + maven { + url kotlinCompilerRepo + } + maven { + url buildKotlinCompilerRepo + } +} + +kotlin { + sourceSets { + commonMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion" + } + kotlin.srcDir '../benchmarks/shared/src' + kotlin.srcDir 'src/main/kotlin' + + } + commonTest { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-test-common:$kotlinVersion" + implementation "org.jetbrains.kotlin:kotlin-test-annotations-common:$kotlinVersion" + } + kotlin.srcDir 'src/tests' + } + jvmTest { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" + implementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" + } + } + nativeMain { + dependsOn commonMain + kotlin.srcDir 'src/main/kotlin-native' + } + jvmMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" + } + kotlin.srcDir 'src/main/kotlin-jvm' + } + linuxMain { dependsOn nativeMain } + windowsMain { dependsOn nativeMain } + macosMain {dependsOn nativeMain } + } + + targets { + fromPreset(presets.jvm, 'jvm') { + def mainOutput = compilations.main.output + compilations.all { + tasks[compileKotlinTaskName].kotlinOptions { + jvmTarget = '1.8' + } + tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true + } + } + + fromPreset(presets.mingwX64, 'windows') + fromPreset(presets.linuxX64, 'linux') + fromPreset(presets.macosX64, 'macos') + + configure([windows, linux, macos]) { + compilations.main.outputKinds('EXECUTABLE') + compilations.main.extraOpts '-opt' + compilations.main.buildTypes = [RELEASE] + } + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/gradle.properties b/tools/benchmarksAnalyzer/gradle.properties new file mode 100644 index 00000000000..87803bed88a --- /dev/null +++ b/tools/benchmarksAnalyzer/gradle.properties @@ -0,0 +1 @@ +org.jetbrains.kotlin.native.home=../../dist \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/settings.gradle b/tools/benchmarksAnalyzer/settings.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt new file mode 100644 index 00000000000..1c9e620a980 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.analyzer + +import java.io.File +import java.io.InputStream + +actual fun readFile(fileName: String): String { + val inputStream = File(fileName).inputStream() + val inputString = inputStream.bufferedReader().use { it.readText() } + return inputString +} + +actual fun format(number: Double, decimalNumber: Int): String = + "%.${decimalNumber}f".format(number) + +actual fun writeToFile(fileName: String, text: String) { + File(fileName).printWriter().use { out -> + out.println(text) + } +} + +actual fun assert(value: Boolean, lazyMessage: () -> Any) { + kotlin.assert(value, lazyMessage) +} + +actual fun getEnv(variableName:String): String? = + System.getenv(variableName) diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt new file mode 100644 index 00000000000..fedc84c274c --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.analyzer + +import platform.posix.* +import kotlinx.cinterop.CValuesRef +import kotlinx.cinterop.* + +actual fun readFile(fileName: String): String { + val file = fopen(fileName, "r") ?: error("Cannot write file '$fileName'") + var buffer = ByteArray(1024) + var text = StringBuilder() + try { + while (true) { + val nextLine = fgets(buffer.refTo(0), buffer.size, file)?.toKString() + if (nextLine == null) break + text.append(nextLine) + } + } finally { + fclose(file) + } + return text.toString() +} + +actual fun format(number: Double, decimalNumber: Int): String { + var buffer = ByteArray(1024) + snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", number) + return buffer.stringFromUtf8() +} + +actual fun writeToFile(fileName: String, text: String) { + val file = fopen(fileName, "wt") ?: error("Cannot write file '$fileName'") + try { + if (fputs(text, file) == EOF) throw Error("File write error") + } finally { + fclose(file) + } +} + +actual fun assert(value: Boolean, lazyMessage: () -> Any) { + kotlin.assert(value, lazyMessage) +} + +actual fun getEnv(variableName:String): String? = + getenv(variableName)?.toKString() diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt new file mode 100644 index 00000000000..722acd86fcf --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.jetbrains.analyzer.getEnv +import org.jetbrains.analyzer.readFile +import org.jetbrains.analyzer.SummaryBenchmarksReport +import org.jetbrains.kliopt.* +import org.jetbrains.renders.TextRender +import org.jetbrains.renders.TeamCityStatisticsRender +import org.jetbrains.report.BenchmarksReport +import org.jetbrains.report.json.JsonTreeParser + +fun main(args: Array) { + + val options = listOf( + OptionDescriptor(ArgType.STRING, "output", "o", "Output file"), + OptionDescriptor(ArgType.DOUBLE, "eps", "e", "Meaningful performance changes", "0.5"), + OptionDescriptor(ArgType.BOOLEAN, "short", "s", "Show short version of report", "false") + ) + + val arguments = listOf( + ArgDescriptor(ArgType.STRING, "mainReport", "Main report for analysis"), + ArgDescriptor(ArgType.STRING, "compareToReport", "Report to compare to", isRequired = false) + ) + + // Parse args. + val argParser = ArgParser(options, arguments) + if (argParser.parse(args)) { + // Read contents of file. + val mainBenchsResults = readFile(argParser.get("mainReport")!!.stringValue) + val mainReportElement = JsonTreeParser.parse(mainBenchsResults) + val mainBenchsReport = BenchmarksReport.create(mainReportElement) + var compareToBenchsReport = argParser.get("compareToReport")?.stringValue?.let { + val compareToResults = readFile(it) + val compareToReportElement = JsonTreeParser.parse(compareToResults) + BenchmarksReport.create(compareToReportElement) + } + + // Generate comparasion report + val summaryReport = SummaryBenchmarksReport(mainBenchsReport, + compareToBenchsReport, + argParser.get("eps")!!.doubleValue) + TextRender().print(summaryReport, argParser.get("short")!!.booleanValue, + argParser.get("output")?.stringValue) + // Produce information for TeamCity if needed. + getEnv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { + TeamCityStatisticsRender().print(summaryReport, argParser.get("short")!!.booleanValue) + } + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt new file mode 100644 index 00000000000..aaa0b618663 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.analyzer + +// Report with changes of different fields. +class ChangeReport(val entityName: String, val changes: List>) { + fun renderAsTextReport(): String { + var content = "" + if (!changes.isEmpty()) { + content = "$content$entityName changes\n" + content = "$content====================\n" + changes.forEach { + content = "$content${it.renderAsText()}" + } + } + return content + } +} + +// Change of report field. +class FieldChange(val field: String, val previous: T, val current: T) { + companion object { + fun getFieldChangeOrNull(field: String, previous: T, current: T): FieldChange? { + if (previous != current) { + return FieldChange(field, previous, current) + } + return null + } + } + + fun renderAsText(): String { + return "$field: $previous -> $current\n" + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt new file mode 100644 index 00000000000..5e42e95a0a3 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.analyzer +import org.jetbrains.report.BenchmarkResult +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.math.sqrt + +// Entity to describe avarage values which conssists of mean and variance values. +data class MeanVariance(val mean: Double, val variance: Double) { + override fun toString(): String { + val format = { number: Double -> format(number, 2)} + return "${format(mean)} ± ${format(variance)}" + } +} + +// Composite benchmark which descibes avarage result for several runs and contains mean and variance value. +data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianceBenchmark: BenchmarkResult) { + + // Calculate difference in percentage compare to another. + fun calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance { + assert(other.meanBenchmark.score > 0 && + other.varianceBenchmark.score > 0 && + other.meanBenchmark.score - other.varianceBenchmark.score != 0.0, + { "Mean and variance should be positive and not equal!" }) + val mean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score + val maxValueChange = abs(meanBenchmark.score + varianceBenchmark.score - + other.meanBenchmark.score + other.varianceBenchmark.score) / + abs(other.meanBenchmark.score + other.varianceBenchmark.score) + + val minValueChange = abs(meanBenchmark.score - varianceBenchmark.score - + other.meanBenchmark.score - other.varianceBenchmark.score) / + abs(other.meanBenchmark.score - other.varianceBenchmark.score) + + val variance = abs(abs(mean) - max(minValueChange, maxValueChange)) + return MeanVariance(mean * 100, variance * 100) + } + + // Calculate ratio value compare to another. + fun calcRatio(other: MeanVarianceBenchmark): MeanVariance { + assert(other.meanBenchmark.score > 0 && + other.varianceBenchmark.score > 0 && + other.meanBenchmark.score - other.varianceBenchmark.score != 0.0, + { "Mean and variance should be positive and not equal!" }) + val mean = meanBenchmark.score / other.meanBenchmark.score + val minRatio = (meanBenchmark.score - varianceBenchmark.score) / (other.meanBenchmark.score + other.varianceBenchmark.score) + val maxRatio = (meanBenchmark.score + varianceBenchmark.score) / (other.meanBenchmark.score - other.varianceBenchmark.score) + val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean)) + return MeanVariance(mean, ratioConfInt) + } + + override fun toString(): String { + val format = { number: Double -> format(number)} + return "${format(meanBenchmark.score)} ± ${format(varianceBenchmark.score)}" + } +} + +fun geometricMean(values: List) = values.map { it.pow(1.0 / values.size) }.reduce { a, b -> a * b } + +fun computeMeanVariance(samples: List): MeanVariance { + val zStar = 1.96 // Critical point for 90% confidence of normal distribution. + val mean = samples.sum() / samples.size + val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size + val confidenceInterval = sqrt(variance / samples.size) * zStar + return MeanVariance(mean, confidenceInterval) +} + +// Calculate avarage results for bencmarks (each becnhmark can be run several times). +fun collectMeanResults(benchmarks: Map>): BenchmarksTable { + return benchmarks.map {(name, resultsSet) -> + val repeatedSequence = IntArray(resultsSet.size) + var currentStatus = BenchmarkResult.Status.PASSED + var currentWarmup = -1 + + // Collect common becnhmark values and check them. + resultsSet.forEachIndexed { index, result -> + // If there was at least one failure, summary is marked as failure. + if (result.status == BenchmarkResult.Status.FAILED) { + currentStatus = result.status + } + repeatedSequence[index] = result.repeat + if (currentWarmup != -1) + if (result.warmup != currentWarmup) + println("Check data consistency. Warmup value for benchmark '${result.name}' differs.") + currentWarmup = result.warmup + } + + repeatedSequence.sort() + // Check if there are missed loop during running benchmarks. + repeatedSequence.forEachIndexed { index, element -> + if (index != 0) + if ((element - repeatedSequence[index - 1]) != 1) + println("Check data consistency. For benchmark '$name' there is no run" + + " between ${repeatedSequence[index - 1]} and $element.") + } + + // Create mean and variance benchmarks result. + val scoreMeanVariance = computeMeanVariance(resultsSet.map { it.score }) + val runtimeInUsMeanVariance = computeMeanVariance(resultsSet.map { it.runtimeInUs }) + val meanBenchmark = BenchmarkResult(name, currentStatus, scoreMeanVariance.mean, + runtimeInUsMeanVariance.mean, repeatedSequence[resultsSet.size - 1], + currentWarmup) + val varianceBenchmark = BenchmarkResult(name, currentStatus, scoreMeanVariance.variance, + runtimeInUsMeanVariance.variance, repeatedSequence[resultsSet.size - 1], + currentWarmup) + name to MeanVarianceBenchmark(meanBenchmark, varianceBenchmark) + }.toMap() +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt new file mode 100644 index 00000000000..810fd98a0ce --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt @@ -0,0 +1,209 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.analyzer + +import kotlin.math.abs +import org.jetbrains.report.BenchmarkResult +import org.jetbrains.report.Environment +import org.jetbrains.report.Compiler +import org.jetbrains.report.BenchmarksReport + +typealias SummaryBenchmark = Pair +typealias BenchmarksTable = Map +typealias SummaryBenchmarksTable = Map +typealias ScoreChange = Pair + +// Summary report with comparasion of separate benchmarks results. +class SummaryBenchmarksReport (val currentReport: BenchmarksReport, + val previousReport: BenchmarksReport? = null, + val meaningfulChangesValue: Double = 0.5) { + // Report created by joining comparing reports. + val mergedReport: Map + + // Lists of benchmarks in different status. + private val benchmarksWithChangedStatus = mutableListOf>() + + // Maps with changes of performance. + var regressions = mapOf() + private set + var improvements = mapOf() + private set + + // Summary value of report - geometric mean. + val geoMeanBenchmark: SummaryBenchmark + var geoMeanScoreChange: ScoreChange? = null + private set + + // Changes in environment and tools. + val envChanges: List> + val kotlinChanges: List> + + // Countable properties. + val failedBenchmarks: List + get() = mergedReport.filter { it.value.first?.meanBenchmark?.status == BenchmarkResult.Status.FAILED } + .map { it.key } + + val addedBenchmarks: List + get() = mergedReport.filter { it.value.second == null }.map { it.key } + + val removedBenchmarks: List + get() = mergedReport.filter { it.value.first == null }.map { it.key } + + val benchmarksNumber: Int + get() = mergedReport.keys.size + + val currentMeanVarianceBenchmarks: List + get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! } + + init { + // Count avarage values for each benchmark. + val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks) + val previousBenchmarksTable = previousReport?.let { + collectMeanResults(previousReport.benchmarks) + } + mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable) + geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable) + if (previousReport != null) { + // Check changes in environment and tools. + envChanges = analyzeEnvChanges(currentReport.env, previousReport.env) + kotlinChanges = analyzeKotlinChanges(currentReport.compiler, previousReport.compiler) + analyzePerformanceChanges() + } else { + envChanges = listOf>() + kotlinChanges = listOf>() + } + } + + fun getBenchmarksWithChangedStatus(): List> = benchmarksWithChangedStatus + + // Create geometric mean. + private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark { + val geoMeanBenchmarkName = "Geometric mean" + val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.meanBenchmark.score }) + val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.varianceBenchmark.score }) + val meanBenchmark = BenchmarkResult(geoMeanBenchmarkName, geoMean) + val varianceBenchmark = BenchmarkResult(geoMeanBenchmarkName, varianceGeoMean) + return MeanVarianceBenchmark(meanBenchmark, varianceBenchmark) + } + + // Merge current and compare to report. + private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?): + Map { + val mergedTable = mutableMapOf() + mergedTable.apply { + currentBenchmarks.forEach { (name, current) -> + val currentBenchmark = current.meanBenchmark + // Check existance of benchmark in previous results. + if (previousBenchmarks == null || name !in previousBenchmarks) { + getOrPut(name) { SummaryBenchmark(current, null) } + } else { + val previousBenchmark = previousBenchmarks[name]!!.meanBenchmark + getOrPut(name) { SummaryBenchmark(current, previousBenchmarks[name]) } + // Explore change of status. + if (previousBenchmark.status != currentBenchmark.status) { + val statusChange = FieldChange("$name", previousBenchmark.status, currentBenchmark.status) + benchmarksWithChangedStatus.add(statusChange) + } + } + } + } + + // Add removed benchmarks to merged report. + mergedTable.apply { + previousBenchmarks?.filter { (key, _) -> key !in currentBenchmarks }?.forEach { (key, value) -> + getOrPut(key) { SummaryBenchmark(null, value) } + } + } + + return mergedTable + } + + // Calculate geometric mean. + private fun calculateGeoMeanBenchmark(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?): + SummaryBenchmark { + // Calculate geometric mean. + val currentGeoMean = createGeoMeanBenchmark(currentBenchmarks) + val previousGeoMean = previousBenchmarks?. let { createGeoMeanBenchmark(previousBenchmarks) } + return SummaryBenchmark(currentGeoMean, previousGeoMean) + } + + private fun getBenchmarkPerfomanceChange(name: String, benchmark: SummaryBenchmark): Pair? { + val (current, previous) = benchmark + current?.let { + previous?.let { + // Calculate metrics for showing difference. + val percent = current.calcPercentageDiff(previous) + val ratio = current.calcRatio(previous) + if (abs(percent.mean) >= meaningfulChangesValue) { + return Pair(name, Pair(percent, ratio)) + } + } + } + return null + } + + // Analyze and collect changes in performance between same becnhmarks. + private fun analyzePerformanceChanges() { + val performanceChanges = mergedReport.asSequence().map {(name, element) -> + getBenchmarkPerfomanceChange(name, element) + }.filterNotNull().groupBy { + if (it.second.first.mean > 0) "regressions" else "improvements" + } + + // Sort regressions and improvements. + regressions = performanceChanges["regressions"] + ?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second } + ?.toMap() ?: mapOf() + improvements = performanceChanges["improvements"] + ?.sortedBy { it.second.first.mean }?.map { it.first to it.second } + ?.toMap() ?: mapOf() + + // Calculate change for geometric mean. + val (current, previous) = geoMeanBenchmark + geoMeanScoreChange = current?. let { + previous?. let { + Pair(current.calcPercentageDiff(previous), current.calcRatio(previous)) + } + } + } + + private fun MutableList>.addFieldChange(field: String, previous: T, current: T) { + FieldChange.getFieldChangeOrNull(field, previous, current)?.let { + add(it) + } + } + + private fun analyzeEnvChanges(currentEnv: Environment, previousEnv: Environment): List> { + return mutableListOf>().apply { + addFieldChange("Machine CPU", previousEnv.machine.cpu, currentEnv.machine.cpu) + addFieldChange("Machine OS", previousEnv.machine.os, currentEnv.machine.os) + addFieldChange("JDK version", previousEnv.jdk.version, currentEnv.jdk.version) + addFieldChange("JDK vendor", previousEnv.jdk.vendor, currentEnv.jdk.vendor) + } + } + + private fun analyzeKotlinChanges(currentCompiler: Compiler, previousCompiler: Compiler): List> { + return mutableListOf>().apply { + addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type) + addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version) + addFieldChange("Backend flags", previousCompiler.backend.flags.toString(), + currentCompiler.backend.flags.toString()) + addFieldChange( "Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion) + } + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt new file mode 100644 index 00000000000..e689be5dfc9 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.analyzer + +expect fun readFile(fileName: String): String +expect fun format(number: Double, decimalNumber: Int = 4): String +expect fun writeToFile(fileName: String, text: String) +expect fun assert(value: Boolean, lazyMessage: () -> Any) +expect fun getEnv(variableName:String): String? \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt new file mode 100644 index 00000000000..f1ca367c479 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt @@ -0,0 +1,224 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.kliopt + +// Possible types of arguments. +enum class ArgType(val hasParameter: Boolean) { + BOOLEAN(false), + STRING(true), + INT(true), + DOUBLE(true) +} + +// Common descriptor both for options and positional arguments. +abstract class Descriptor(val type: ArgType, + val longName: String, + val description: String? = null, + val defaultValue: String? = null, + val isRequired: Boolean = false) { + abstract val textDescription: String + abstract val helpMessage: String +} + +class OptionDescriptor( + type: ArgType, + longName: String, + val shortName: String ? = null, + description: String? = null, + defaultValue: String? = null, + isRequired: Boolean = false) : Descriptor (type, longName, description, defaultValue, isRequired) { + override val textDescription: String + get() = "option -$longName" + + override val helpMessage: String + get() { + val result = StringBuilder() + result.append(" -${longName}") + shortName?.let { result.append(", -$it") } + defaultValue?.let { result.append(" [$it]") } + description?.let {result.append(" -> ${it}")} + if (!isRequired) result.append(" (optional)") + result.append("\n") + return result.toString() + } +} + +class ArgDescriptor( + type: ArgType, + longName: String, + description: String? = null, + defaultValue: String? = null, + isRequired: Boolean = true) : Descriptor (type, longName, description, defaultValue, isRequired) { + override val textDescription: String + get() = "argument $longName" + + override val helpMessage: String + get() { + val result = StringBuilder() + result.append(" ${longName}") + defaultValue?.let { result.append(" [$it]") } + description?.let {result.append(" -> ${it}")} + if (!isRequired) result.append(" (optional)") + result.append("\n") + return result.toString() + } +} + +// Arguments parser. +class ArgParser(optionsList: List, argsList: List = listOf()) { + private val options = optionsList.union(listOf(OptionDescriptor(ArgType.BOOLEAN, "help", + "h", "Usage info"))) + .toList() + private val arguments = argsList + private lateinit var parsedValues: MutableMap + + inner class ParsedArg(val descriptor: Descriptor, val value: String) { + val intValue: Int + get() { + if (descriptor.type != ArgType.INT) + error("Incorrect value for ${descriptor.textDescription}, must be an int") + return value.toInt() + } + + val stringValue: String + get() { + if (descriptor.type != ArgType.STRING) + printError("Incorrect value for ${descriptor.textDescription}, must be a string") + return value + } + + val booleanValue: Boolean + get() { + if (descriptor.type != ArgType.BOOLEAN) + printError("Incorrect value for ${descriptor.textDescription}, must be a boolean") + return value == "true" + } + + val doubleValue: Double + get() { + if (descriptor.type != ArgType.DOUBLE) + printError("Incorrect option value for ${descriptor.textDescription}, must be a double") + return value.toDouble() + } + } + + // Output error. Also adds help usage information for easy understanding of problem. + private fun printError(message: String) { + error("$message\n${makeUsage()}") + } + + private fun saveAsArg(argDescriptors: Map, arg: String): Boolean { + // Find uninitialized arguments. + val nullArgs = argDescriptors.keys.filter { parsedValues[it] == null } + val name = nullArgs.firstOrNull() + name?. let { + parsedValues[name] = ParsedArg(argDescriptors[name]!!, arg) + return true + } + return false + } + + // Parse arguments. + // Returns true if all arguments were parsed, otherwise return false and print help message. + fun parse(args: Array): Boolean { + var index = 0 + val optDescriptors = options.map { it.longName to it }.toMap() + val shortNames = options.filter { it.shortName != null }.map { it.shortName!! to it.longName }.toMap() + val argDescriptors = arguments.map { it.longName to it }.toMap() + parsedValues = optDescriptors.keys.union(argDescriptors.keys).toList().map { it to null }.toMap().toMutableMap() + while (index < args.size) { + val arg = args[index] + if (arg.startsWith('-')) { + // Option is found. + val name = shortNames[arg.substring(1)] ?: arg.substring(1) + val descriptor = optDescriptors[name] + descriptor?. let { + if (descriptor.type.hasParameter) { + if (index < args.size - 1) { + parsedValues[name] = ParsedArg(descriptor, args[index + 1]) + index++ + } else { + // An error, option with value without value. + printError("No value for ${descriptor.textDescription}") + } + } else { + if (name == "help") { + println(makeUsage()) + return false + } + parsedValues[name] = ParsedArg(descriptor, "true") + } + } ?: run { + // Try save as argument. + if (!saveAsArg(argDescriptors, arg)) { + printError("Unknown option $arg") + } + } + } else { + // Argument is found. + if (!saveAsArg(argDescriptors, arg)) { + printError("Too many arguments!") + } + } + index++ + } + + parsedValues.forEach { (key, value) -> + val descriptor = optDescriptors[key] ?: argDescriptors[key]!! + // Not inited, append default value if needed. + parsedValues[key] = value ?: run { + descriptor.defaultValue?. let { + ParsedArg(descriptor, descriptor.defaultValue) + } + } + // Check if arg is always required. + parsedValues[key] ?: run { + if (descriptor.isRequired) { + printError("Please, provide value for ${descriptor.textDescription}. It should be always set") + } + } + } + + return true + } + + fun get(name: String): ParsedArg? { + if (::parsedValues.isInitialized) { + return parsedValues[name] + } else { + println("Method parse() of ArgParser class should be called before getting arguments and options.") + return null + } + } + + private fun makeUsage(): String { + val result = StringBuilder() + result.append("Usage: \n") + if (!arguments.isEmpty()) { + result.append("Arguments: \n") + arguments.forEach { + result.append(it.helpMessage) + } + } + result.append("Options: \n") + options.forEach { + result.append(it.helpMessage) + } + return result.toString() + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt new file mode 100644 index 00000000000..e5542cef17f --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt @@ -0,0 +1,213 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.renders + +import org.jetbrains.analyzer.* +import org.jetbrains.report.BenchmarkResult + +import kotlin.math.abs + +// Common interface for printing report in different formats. +interface Render { + fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean = false): String + // Print report using render. + fun print(report: SummaryBenchmarksReport, onlyChanges: Boolean = false, outputFile: String? = null) { + val content = render(report, onlyChanges) + outputFile?.let { + writeToFile(outputFile, content) + } ?: println(content) + } +} + +// Report render to text format. +class TextRender: Render { + private val content = StringBuilder() + private val headerSeparator = "=================" + private val wideColumnWidth = 50 + private val standardColumnWidth = 25 + + private fun append(text: String = "") { + content.append("$text\n") + } + + override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { + renderEnvChanges(report.envChanges, "Environment") + renderEnvChanges(report.kotlinChanges, "Compiler") + renderStatusSummary(report) + renderStatusChangesDetails(report.getBenchmarksWithChangedStatus()) + renderPerformanceSummary(report.regressions, report.improvements) + renderPerformanceDetails(report, onlyChanges) + return content.toString() + } + + private fun printBucketInfo(bucket: Collection, name: String) { + if (!bucket.isEmpty()) { + append("$name: ${bucket.size}") + } + } + + private fun printStatusChangeInfo(bucket: List>, name: String) { + if (!bucket.isEmpty()) { + append("$name:") + for (change in bucket) { + append(change.renderAsText()) + } + } + } + + fun renderEnvChanges(envChanges: List>, bucketName: String) { + if (!envChanges.isEmpty()) { + append(ChangeReport(bucketName, envChanges).renderAsTextReport()) + } + } + + private fun printPerformanceBucket(bucket: Map, bucketName: String) { + if (!bucket.isEmpty()) { + var percentsList = bucket.values.map{ it.first.mean } + // Maps of regressions and improvements are sorted. + val maxValue = percentsList.first() + var geomeanValue: Double + if (percentsList.first() > 0.0) { + geomeanValue = geometricMean(percentsList) + } else { + // Geometric mean can be counted on positive numbers. + val precision = abs(maxValue) + 1 + percentsList = percentsList.map{it + precision} + geomeanValue = geometricMean(percentsList) - precision + } + + append("$bucketName: Maximum = ${formatValue(maxValue, true)}," + + " Geometric mean = ${formatValue(geomeanValue, true)}") + } + } + + fun renderStatusChangesDetails(benchmarksWithChangedStatus: List>) { + if (!benchmarksWithChangedStatus.isEmpty()) { + append("Changes in status") + append(headerSeparator) + printStatusChangeInfo(benchmarksWithChangedStatus + .filter { it.current == BenchmarkResult.Status.FAILED }, "New failures") + printStatusChangeInfo(benchmarksWithChangedStatus + .filter { it.current == BenchmarkResult.Status.PASSED }, "New passes") + append() + } + } + + fun renderStatusSummary(report: SummaryBenchmarksReport) { + append("Status summary") + append(headerSeparator) + + val failedBenchmarks = report.failedBenchmarks + val addedBenchmarks = report.addedBenchmarks + val removedBenchmarks = report.removedBenchmarks + if (failedBenchmarks.isEmpty()) { + append("All benchmarks passed!") + } + if (!failedBenchmarks.isEmpty() || !addedBenchmarks.isEmpty() || !removedBenchmarks.isEmpty()) { + printBucketInfo(failedBenchmarks, "Failed benchmarks") + printBucketInfo(addedBenchmarks, "Added benchmarks") + printBucketInfo(removedBenchmarks, "Removed benchmarks") + } + append("Total becnhmarks number: ${report.benchmarksNumber}") + append() + } + + fun renderPerformanceSummary(regressions: Map, + improvements: Map) { + if (!regressions.isEmpty() || !improvements.isEmpty()) { + append("Performance summary") + append(headerSeparator) + printPerformanceBucket(regressions, "Regressions") + printPerformanceBucket(improvements, "Improvements") + append() + } + } + + private fun formatValue(number: Double, isPercent: Boolean = false): String = + if (isPercent) format(number, 2) + "%" else format(number) + + private fun formatColumn(content:String, isWide: Boolean = false): String = + content.padEnd(if (isWide) wideColumnWidth else standardColumnWidth, ' ') + + private fun printBenchmarksDetails(fullSet: Map, + bucket: Map? = null) { + if (bucket != null) { + // There are changes in performance. + // Output changed benchmarks. + for ((name, change) in bucket) { + append(formatColumn(name, true) + + formatColumn(fullSet[name]!!.first!!.toString()) + + formatColumn(fullSet[name]!!.second!!.toString()) + + formatColumn(change.first.toString() + " %") + + formatColumn(change.second.toString())) + } + } else { + // Output all values without performance changes. + val placeholder = "-" + for ((name, value) in fullSet) { + append(formatColumn(name, true) + + formatColumn(value.first?.toString() ?: placeholder) + + formatColumn(value.second?.toString() ?: placeholder) + + formatColumn(placeholder) + + formatColumn(placeholder)) + } + } + } + + private fun printTableLineSeparator(tableWidth: Int) = + append("${"-".padEnd(tableWidth, '-')}") + + private fun printPerformanceTableHeader(): Int { + val wideColumns = listOf(formatColumn("Benchmark", true)) + val standardColumns = listOf(formatColumn("Current score"), + formatColumn("Previous score"), + formatColumn("Percent"), + formatColumn("Ratio")) + val tableWidth = wideColumnWidth * wideColumns.size + standardColumnWidth * standardColumns.size + append("${wideColumns.joinToString(separator = "")}${standardColumns.joinToString(separator = "")}") + printTableLineSeparator(tableWidth) + return tableWidth + } + + fun renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean = false) { + append("Performance details") + append(headerSeparator) + + if (onlyChanges) { + if (report.regressions.isEmpty() && report.improvements.isEmpty()) { + append("All becnhmarks are stable.") + } + } + + val tableWidth = printPerformanceTableHeader() + // Print geometric mean. + val geoMeanChangeMap = report.geoMeanScoreChange?. + let { mapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanScoreChange!!) } + printBenchmarksDetails( + mutableMapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanBenchmark), + geoMeanChangeMap) + printTableLineSeparator(tableWidth) + printBenchmarksDetails(report.mergedReport, report.regressions) + printBenchmarksDetails(report.mergedReport, report.improvements) + if (!onlyChanges) { + // Print all remaining results. + printBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys && + it.key !in report.improvements.keys }) + } + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt new file mode 100644 index 00000000000..b6f886ebd37 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.renders + +import org.jetbrains.analyzer.* +import org.jetbrains.report.BenchmarkResult + +// Report render to text format. +class TeamCityStatisticsRender: Render { + private var content = StringBuilder() + + override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { + // For current benchmarks print score as TeamCity Test Metadata + report.currentMeanVarianceBenchmarks.forEach { benchmark -> + content.append(renderBenchmark(benchmark.meanBenchmark, "Mean")) + content.append(renderBenchmark(benchmark.varianceBenchmark, "Variance")) + } + // Report geometric mean as build statistic value + content.append(renderGeometricMean(report.geoMeanBenchmark.first!!)) + return content.toString() + } + + private fun renderBenchmark(benchmark: BenchmarkResult, metric: String): String = + "##teamcity[testMetadata testName='${benchmark.name}' name='$metric' type='number' value='${benchmark.score}']\n" + + private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark): String = + "##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n" + + "##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n" +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt new file mode 100644 index 00000000000..ff04085c9a1 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.analyzer + +import kotlin.test.* +import kotlin.math.abs +import org.jetbrains.report.BenchmarkResult + +class AnalyzerTests { + private val eps = 0.000001 + + private fun createMeanVarianceBenchmarks(): Pair { + val firstMean = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 9.0, 9.0, 10, 10) + val firstVariance = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 0.0001, 0.0001, 10, 10) + val first = MeanVarianceBenchmark(firstMean, firstVariance) + + val secondMean = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 10.0, 10.0, 10, 10) + val secondVariance = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 0.0001, 0.0001, 10, 10) + val second = MeanVarianceBenchmark(secondMean, secondVariance) + + return Pair(first, second) + } + + @Test + fun testGeoMean() { + val numbers = listOf(4.0, 6.0, 9.0) + val value = geometricMean(numbers) + val expected = 6.0 + assertTrue(abs(value - expected) < eps) + } + + @Test + fun testComputeMeanVariance() { + val numbers = listOf(10.1, 10.2, 10.3) + val value = computeMeanVariance(numbers) + val expectedMean = 10.2 + val expectedVariance = 0.092395 + assertTrue(abs(value.mean - expectedMean) < eps) + assertTrue(abs(value.variance - expectedVariance) < eps) + } + + @Test + fun calcPercentageDiff() { + val inputs = createMeanVarianceBenchmarks() + + val percent = inputs.first.calcPercentageDiff(inputs.second) + val expectedMean = -10.0 + val expectedVariance = 0.0021 + assertTrue(abs(percent.mean - expectedMean) < eps) + assertTrue(abs(percent.variance - expectedVariance) < eps) + } + + @Test + fun calcRatio() { + val inputs = createMeanVarianceBenchmarks() + + val ratio = inputs.first.calcRatio(inputs.second) + val expectedMean = 0.9 + val expectedVariance = 0.00001899 + + assertTrue(abs(ratio.mean - expectedMean) < eps) + assertTrue(abs(ratio.variance - expectedVariance) < eps) + } +} \ No newline at end of file