diff --git a/performance/build.gradle b/performance/build.gradle index 5ce0da61013..588d69ce678 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -26,6 +26,7 @@ repositories { //TODO: property def jvmWarmup = 10000 def nativeWarmup = 10 +def attempts = 10 ext."konan.home" = distDir @@ -64,7 +65,7 @@ task jvmRun(type: JavaExec) { def output = new ByteArrayOutputStream() classpath sourceSets.main.runtimeClasspath main = "MainKt" - args "$jvmWarmup" + args "$jvmWarmup", "$attempts" standardOutput = output doLast { dumpReport('jvmReport', output) @@ -80,7 +81,7 @@ private void dumpReport(String name, ByteArrayOutputStream output) { task konanRun(type: Exec) { dependsOn 'build' def output = new ByteArrayOutputStream() - commandLine konanArtifacts.Ring.getByTarget('host').artifact.absolutePath, "$nativeWarmup" + commandLine konanArtifacts.Ring.getByTarget('host').artifact.absolutePath, "$nativeWarmup", "$attempts" standardOutput = output doLast { dumpReport('konanReport', output) @@ -105,11 +106,12 @@ task bench(type:DefaultTask) { .each { k, v -> def konanValue = konanReport.report[k] def ratio = konanValue.mean / v.mean + // This is a hack since neither mean nor variance of ratio of two distributions is known. def minRatio = (konanValue.mean - konanValue.stdDev) / (v.mean + v.stdDev) def maxRatio = (konanValue.mean + konanValue.stdDev) / (v.mean - v.stdDev) - def ratioStdDev = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio)) + def ratioConfInt = Math.min(Math.abs(minRatio - ratio), Math.abs(maxRatio - ratio)) def formattedKonanValue = String.format('%.4f us +- %.4f us', konanValue.mean / 1000, konanValue.stdDev / 1000) - def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioStdDev) + def formattedRatio = String.format('%.2f +- %.2f', ratio, ratioConfInt) if (k == 'RingAverage') { average = formattedRatio absoluteAverage = formattedKonanValue diff --git a/performance/src/main/kotlin/main.kt b/performance/src/main/kotlin/main.kt index a7bd667da57..a98cffe2b36 100644 --- a/performance/src/main/kotlin/main.kt +++ b/performance/src/main/kotlin/main.kt @@ -17,12 +17,23 @@ import org.jetbrains.ring.Launcher fun main(args: Array) { - var numWarmIterations = 0 // Should be 100000 for jdk based run + var numWarmIterations = 0 // Should be 100000 for jdk based run + var numberOfAttempts = 10 - if (args.size == 1) - numWarmIterations = args[0].toInt() + when (args.size) { + 0 -> { } + 1 -> numWarmIterations = args[0].toInt() + 2 -> { + numWarmIterations = args[0].toInt() + numberOfAttempts = args[1].toInt() + } + else -> { + println("Usage: perf [# warmup iterations] [# attempts]") + return + } + } println("Ring starting") println(" warmup iterations count: $numWarmIterations") - Launcher(numWarmIterations).runBenchmarks() + Launcher(numWarmIterations, numberOfAttempts).runBenchmarks() } \ No newline at end of file diff --git a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt index b7fb4586821..78e2dbac0ac 100644 --- a/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt +++ b/performance/src/main/kotlin/org/jetbrains/ring/launcher.kt @@ -17,13 +17,14 @@ package org.jetbrains.ring import octoTest +import kotlin.math.sqrt import kotlin.system.measureNanoTime val BENCHMARK_SIZE = 100 //-----------------------------------------------------------------------------// -class Launcher(val numWarmIterations: Int) { +class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) { class Results(val mean: Double, val variance: Double) val results = mutableMapOf() @@ -48,8 +49,7 @@ class Launcher(val numWarmIterations: Int) { autoEvaluatedNumberOfMeasureIteration *= 2 } - val attempts = 10 - val samples = DoubleArray(attempts) + val samples = DoubleArray(numberOfAttempts) for (k in samples.indices) { i = autoEvaluatedNumberOfMeasureIteration val time = measureNanoTime { @@ -60,8 +60,8 @@ class Launcher(val numWarmIterations: Int) { } samples[k] = time * 1.0 / autoEvaluatedNumberOfMeasureIteration } - val mean = samples.sum() / attempts - val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / attempts + val mean = samples.sum() / numberOfAttempts + val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / numberOfAttempts return Results(mean, variance) } @@ -110,19 +110,24 @@ class Launcher(val numWarmIterations: Int) { //-------------------------------------------------------------------------// + private val zStar = 1.96 // For 95% confidence interval. + fun printResultsNormalized() { var totalMean = 0.0 var totalVariance = 0.0 results.asSequence().sortedBy { it.key }.forEach { val niceName = it.key.padEnd(50, ' ') - println("$niceName : ${it.value.mean.toString(9)} : ${kotlin.math.sqrt(it.value.variance).toString(9)}") + val mean = it.value.mean + val variance = it.value.variance + val confidenceInterval = sqrt(variance / numberOfAttempts) * zStar + println("$niceName : ${mean.toString(9)} : ${confidenceInterval.toString(9)}") - totalMean += it.value.mean - totalVariance += it.value.variance + totalMean += mean + totalVariance += variance } val averageMean = totalMean / results.size - val averageStdDev = kotlin.math.sqrt(totalVariance) / results.size - println("\nRingAverage: ${averageMean.toString(9)} : ${averageStdDev.toString(9)}") + val averageConfidenceInterval = sqrt(totalVariance / numberOfAttempts) * zStar / results.size + println("\nRingAverage: ${averageMean.toString(9)} : ${averageConfidenceInterval.toString(9)}") } //-------------------------------------------------------------------------//