diff --git a/performance/gradle.properties b/performance/gradle.properties index b73254cda25..b2a436c56eb 100644 --- a/performance/gradle.properties +++ b/performance/gradle.properties @@ -1,8 +1,8 @@ org.jetbrains.kotlin.native.home=../dist org.jetbrains.kotlin.native.jvmArgs=-Xmx6G jvmWarmup = 10000 -nativeWarmup = 10 -attempts = 10 +nativeWarmup = 20 +attempts = 60 jvmBenchResults = jvmBenchResults.json nativeBenchResults = nativeBenchResults.json nativeTextReport = nativeReport.txt @@ -14,4 +14,4 @@ analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin outputReport = ../report/report.html bintrayUrl = https://api.bintray.com/content/lepilkinaelena bintrayRepo = KotlinNativePerformance -bintrayPackage = jsonReports \ No newline at end of file +bintrayPackage = jsonReports diff --git a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt index 971291d0fd9..3145387afda 100644 --- a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt +++ b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt @@ -20,7 +20,7 @@ import octoTest import kotlin.math.sqrt import org.jetbrains.report.BenchmarkResult -val BENCHMARK_SIZE = 100 +const val BENCHMARK_SIZE = 10000 //-----------------------------------------------------------------------------// @@ -455,4 +455,4 @@ class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) { fun runOctoTest() { launch(::octoTest, "OctoTest") } -} \ No newline at end of file +} diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index eaf6ee58512..62775608e4a 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -81,7 +81,7 @@ fun main(args: Array) { val options = listOf( OptionDescriptor(ArgType.String(), "output", "o", "Output file"), - OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "0.5"), + OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "1.0"), OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"), OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")), "renders", "r", "Renders for showing information", "text", isMultiple = true), diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt index b42005e7d98..4d9a18cf198 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt @@ -71,10 +71,11 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc } -fun geometricMean(values: List) = values.map { it.pow(1.0 / values.size) }.reduce { a, b -> a * b } +fun geometricMean(values: List, totalNumber: Int? = null) = + values.map { it.pow(1.0 / (totalNumber ?: values.size)) }.reduce { a, b -> a * b } fun computeMeanVariance(samples: List): MeanVariance { - val zStar = 1.96 // Critical point for 90% confidence of normal distribution. + val zStar = 1.67 // Critical point for 90% confidence of normal distribution. val mean = samples.sum() / samples.size val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size val confidenceInterval = sqrt(variance / samples.size) * zStar diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt index 2f78893e757..7a42e465962 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt @@ -142,12 +142,12 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, return 0.0 var percentsList = bucket.values.map { it.first.mean } return if (percentsList.first() > 0.0) { - geometricMean(percentsList) + geometricMean(percentsList, benchmarksNumber) } else { // Geometric mean can be counted on positive numbers. val precision = abs(getMaximumChange(bucket)) + 1 percentsList = percentsList.map { it + precision } - geometricMean(percentsList) - precision + geometricMean(percentsList, benchmarksNumber) - precision } } diff --git a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt index ff04085c9a1..5bad651c192 100644 --- a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt +++ b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt @@ -49,7 +49,7 @@ class AnalyzerTests { val numbers = listOf(10.1, 10.2, 10.3) val value = computeMeanVariance(numbers) val expectedMean = 10.2 - val expectedVariance = 0.092395 + val expectedVariance = 0.12539360253 assertTrue(abs(value.mean - expectedMean) < eps) assertTrue(abs(value.variance - expectedVariance) < eps) } @@ -76,4 +76,4 @@ class AnalyzerTests { assertTrue(abs(ratio.mean - expectedMean) < eps) assertTrue(abs(ratio.variance - expectedVariance) < eps) } -} \ No newline at end of file +}