From e36374671738fa130b63950a3ebb529ce47d9392 Mon Sep 17 00:00:00 2001 From: LepilkinaElena Date: Thu, 25 Jul 2019 14:28:18 +0300 Subject: [PATCH] Fix in analyzer + debug mode for compile only benchmarks (#3210) --- performance/framework/build.gradle | 6 ++--- performance/helloworld/build.gradle.kts | 2 +- performance/videoplayer/build.gradle.kts | 2 +- .../src/main/kotlin/main.kt | 26 ++++++++++++------- .../org/jetbrains/analyzer/Statistics.kt | 8 +++++- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/performance/framework/build.gradle b/performance/framework/build.gradle index 56f71211646..05b6c7b9cfa 100644 --- a/performance/framework/build.gradle +++ b/performance/framework/build.gradle @@ -52,7 +52,7 @@ kotlin { } macosX64("macos").binaries { - framework(frameworkName, [RELEASE]) + framework(frameworkName, [DEBUG]) } } @@ -75,10 +75,10 @@ task konanJsonReport { if (PlatformInfo.isMac()) { def applicationName = "FrameworkBenchmarksAnalyzer" def frameworkPath = kotlin.macosX64("macos").binaries. - getFramework(frameworkName, kotlin.macosX64("macos").binaries.RELEASE).outputFile.absolutePath + getFramework(frameworkName, kotlin.macosX64("macos").binaries.DEBUG).outputFile.absolutePath def nativeExecutable = new File("$frameworkPath/$frameworkName").canonicalPath def nativeCompileTime = MPPTools.getNativeCompileTime(applicationName, ['compileKotlinMacos', - 'linkBenchmarksAnalyzerReleaseFrameworkMacos', + 'linkBenchmarksAnalyzerDebugFrameworkMacos', 'cinteropLibcurlMacos']) def properties = getCommonProperties() + ['type' : 'native', 'compilerVersion': "${konanVersion}".toString(), diff --git a/performance/helloworld/build.gradle.kts b/performance/helloworld/build.gradle.kts index 7eb395f9b70..76fca5c3bda 100644 --- a/performance/helloworld/build.gradle.kts +++ b/performance/helloworld/build.gradle.kts @@ -18,7 +18,7 @@ compileBenchmark { repeatNumber = 10 buildSteps { step("runKonanc") { - command("$dist/bin/konanc$toolSuffix", "$projectDir/src/main/kotlin/main.kt", "-o", "$buildDir/program$binarySuffix") + command("$dist/bin/konanc$toolSuffix", "$projectDir/src/main/kotlin/main.kt", "-g", "-o", "$buildDir/program$binarySuffix") } } } diff --git a/performance/videoplayer/build.gradle.kts b/performance/videoplayer/build.gradle.kts index 8c505f97de4..cb3c59a3326 100644 --- a/performance/videoplayer/build.gradle.kts +++ b/performance/videoplayer/build.gradle.kts @@ -68,7 +68,7 @@ compileBenchmark { step("runKonanProgram") { command = listOf( "$dist/bin/konanc$toolSuffix", - "-ea", "-p", "program", + "-ea", "-p", "program", "-g", "-o", "${buildDir.absolutePath}/program$binarySuffix", "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-ffmpeg.klib", "-l", "$dist/../samples/videoplayer/build/classes/kotlin/videoPlayer/main/videoplayer-cinterop-sdl.klib", diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index db783276ce0..b5fa956b8db 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -141,20 +141,26 @@ fun main(args: Array) { val codesizeNormalize = codesizeNormalize?.let { parseNormalizeResults(getFileContent(it)) } + results.apply { add(benchsReport.failedBenchmarks.size.toString()) + if (!execSamples.isEmpty()) { + val filterExec = if (execSamples.first() == "all") null else execSamples + add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.EXECUTION_TIME, + exec == "geomean", filterExec, executionNormalize).joinToString(";")) + } - val filterExec = if (execSamples.first() == "all") null else execSamples - add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.EXECUTION_TIME, - exec == "geomean", filterExec, executionNormalize).joinToString(";")) + if (!compileSamples.isEmpty()) { + val filterCompile = if (compileSamples.first() == "all") null else compileSamples + add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.COMPILE_TIME, + compile == "geomean", filterCompile, compileNormalize).joinToString(";")) + } - val filterCompile = if (compileSamples.first() == "all") null else compileSamples - add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.COMPILE_TIME, - compile == "geomean", filterCompile, compileNormalize).joinToString(";")) - - val filterCodesize = if (codesizeSamples.first() == "all") null else codesizeSamples - add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.CODE_SIZE, - codesize == "geomean", filterCodesize, codesizeNormalize).joinToString(";")) + if (!codesizeSamples.isEmpty()) { + val filterCodesize = if (codesizeSamples.first() == "all") null else codesizeSamples + add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.CODE_SIZE, + codesize == "geomean", filterCodesize, codesizeNormalize).joinToString(";")) + } } println(results.joinToString()) diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt index 760c3fcc972..d08f018925b 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt @@ -83,7 +83,13 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc } fun geometricMean(values: Collection, totalNumber: Int = values.size) = - values.asSequence().filter{ it != 0.0 }.map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b } + with(values.asSequence().filter { it != 0.0 }) { + if (count() == 0) { + 0.0 + } else { + map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b } + } + } fun computeMeanVariance(samples: List): MeanVariance { val zStar = 1.67 // Critical point for 90% confidence of normal distribution.