diff --git a/performance/buildSrc/src/main/kotlin/BuildRegister.kt b/performance/buildSrc/src/main/kotlin/BuildRegister.kt index 0a4ba665bf9..4e27af10d79 100644 --- a/performance/buildSrc/src/main/kotlin/BuildRegister.kt +++ b/performance/buildSrc/src/main/kotlin/BuildRegister.kt @@ -46,7 +46,8 @@ open class BuildRegister : DefaultTask() { var bundleSize: Int? = null - val buildInfoToken: Int = 4 + val buildInfoTokens: Int = 4 + val frameworkInfoTokens: Int = 3 val compileTimeSamplesNumber: Int = 2 val buildNumberTokens: Int = 3 val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg" @@ -96,30 +97,50 @@ open class BuildRegister : DefaultTask() { val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword) val branch = getBuildProperty(currentBuild,"branchName") + val target = System.getProperty("os.name").replace("\\s".toRegex(), "") + // Get summary information. val output = arrayOf("$analyzer", "summary", "-exec-samples", "all", "-compile", "samples", "-compile-samples", "HelloWorld,Videoplayer", "-codesize-samples", "all", "-exec-normalize", "bintray:goldenResults.csv", "-codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile") .runCommand() + // Postprocess information. val buildInfoParts = output.split(',') - if (buildInfoParts.size != buildInfoToken) { - error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile.") + if (buildInfoParts.size != buildInfoTokens) { + error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $output") } val (failures, executionTime, compileTime, codeSize) = buildInfoParts.map { it.trim() } // Add legends. val geometricMean = "Geometric Mean-" val executionTimeInfo = "$geometricMean$executionTime" - val codeSizeInfo = "$geometricMean$codeSize" + var codeSizeInfo = "$geometricMean$codeSize" val compileTimeSamples = compileTime.split(';') if (compileTimeSamples.size != compileTimeSamplesNumber) { error("Problems with getting compile time samples value. Expected at least $compileTimeSamplesNumber samples, got ${compileTimeSamples.size}") } val (helloWorldCompile, videoplayerCompile) = compileTimeSamples - val compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile" - val target = System.getProperty("os.name").replace("\\s".toRegex(), "") + var compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile" + + // Collect framework run details. + if (target == "MacOSX") { + + val frameworkOutput = arrayOf("$analyzer", "summary", "-compile", "samples", + "-compile-samples", "FrameworkBenchmarksAnalyzer", "-codesize-samples", "FrameworkBenchmarksAnalyzer", + "-codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile") + .runCommand() + + val buildInfoPartsFramework = frameworkOutput.split(',') + if (buildInfoPartsFramework.size != frameworkInfoTokens) { + error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $frameworkOutput") + } + val (_, frameworkCompileTime, frameworkCodeSize) = buildInfoPartsFramework.map { it.trim() } + codeSizeInfo += ";FrameworkBenchmarksAnalyzer-$frameworkCodeSize" + compileTimeInfo += ";FrameworkBenchmarksAnalyzer-$frameworkCompileTime" + } + val buildNumberParts = buildNumber.split("-") if (buildNumberParts.size != buildNumberTokens) { error("Wrong format of build number $buildNumber.") diff --git a/performance/buildSrc/src/main/kotlin/MPPTools.kt b/performance/buildSrc/src/main/kotlin/MPPTools.kt index b958a4f6544..9eb8ab8037e 100644 --- a/performance/buildSrc/src/main/kotlin/MPPTools.kt +++ b/performance/buildSrc/src/main/kotlin/MPPTools.kt @@ -117,7 +117,7 @@ fun mergeReports(reports: List): String { BenchmarksReport.create(reportElement) } - return reportsToMerge.reduce { result, it -> result + it }.toJson() + return if (reportsToMerge.isEmpty()) "" else reportsToMerge.reduce { result, it -> result + it }.toJson() } // Find file with set name in directory. @@ -172,8 +172,10 @@ fun createRunTask( fun getJvmCompileTime(programName: String): BenchmarkResult = TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar")) -fun getNativeCompileTime(programName: String): BenchmarkResult = - TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative")) +@JvmOverloads +fun getNativeCompileTime(programName: String, + tasks: List = listOf("compileKotlinNative", "linkMainReleaseExecutableNative")): BenchmarkResult = + TaskTimerListener.getBenchmarkResult(programName, tasks) fun getCompileBenchmarkTime(programName: String, tasksNames: Iterable, repeats: Int, exitCodes: Map) = (1..repeats).map { number -> diff --git a/performance/framework/build.gradle b/performance/framework/build.gradle new file mode 100644 index 00000000000..4f5b787129a --- /dev/null +++ b/performance/framework/build.gradle @@ -0,0 +1,115 @@ +buildscript { + ext.rootBuildDirectory = file('../..') + + apply from: "$rootBuildDirectory/gradle/loadRootProperties.gradle" + apply from: "$rootBuildDirectory/gradle/kotlinGradlePlugin.gradle" + + repositories { + maven { + url 'https://cache-redirector.jetbrains.com/jcenter' + } + maven { + url kotlinCompilerRepo + } + } + + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" + } +} + +apply plugin: 'kotlin-multiplatform' + +repositories { + maven { + url 'https://cache-redirector.jetbrains.com/jcenter' + } + maven { + url kotlinCompilerRepo + } + maven { + url buildKotlinCompilerRepo + } +} + +def toolsPath = '../../tools' +def frameworkName = 'benchmarksAnalyzer' + +kotlin { + sourceSets { + macosMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion" + } + kotlin.srcDir "$toolsPath/benchmarks/shared/src" + kotlin.srcDir "$toolsPath/benchmarksAnalyzer/src/main/kotlin" + kotlin.srcDir "$toolsPath/kliopt" + kotlin.srcDir "$toolsPath/benchmarksAnalyzer/src/main/kotlin-native" + } + } + + configure([macosX64("macos")]) { + compilations.main.cinterops { + libcurl { + defFile "$toolsPath/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def" + } + } + } + + macosX64("macos").binaries { + framework(frameworkName, [RELEASE]) + } +} + +MPPTools.addTimeListener(project) + +task konanRun { + if (MPPTools.isMacos()) { + dependsOn 'build' + } +} + +task jvmRun(type: RunJvmTask) { + doLast { + println("JVM run is unsupported") + } +} + +task konanJsonReport { + doLast { + if (MPPTools.isMacos()) { + def applicationName = "FrameworkBenchmarksAnalyzer" + def frameworkPath = kotlin.macosX64("macos").binaries. + getFramework(frameworkName, kotlin.macosX64("macos").binaries.RELEASE).outputFile.absolutePath + def nativeExecutable = new File("$frameworkPath/$frameworkName").canonicalPath + def nativeCompileTime = MPPTools.getNativeCompileTime(applicationName, ['compileKotlinMacos', + 'linkBenchmarksAnalyzerReleaseFrameworkMacos', + 'cinteropLibcurlMacos']) + def properties = getCommonProperties() + ['type' : 'native', + 'compilerVersion': "${konanVersion}".toString(), + 'flags' : [], + 'benchmarks' : '[]', + 'compileTime' : [nativeCompileTime], + 'codeSize' : MPPTools.getCodeSizeBenchmark(applicationName, nativeExecutable)] + def output = MPPTools.createJsonReport(properties) + new File("${buildDir.absolutePath}/${nativeJson}").write(output) + } + } +} + +task jvmJsonReport { + doLast { + println("JVM run is unsupported") + } +} + +jvmRun.finalizedBy jvmJsonReport +konanRun.finalizedBy konanJsonReport + +private def getCommonProperties() { + return ['cpu': System.getProperty("os.arch"), + 'os': System.getProperty("os.name"), // OperatingSystem.current().getName() + 'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion + 'jdkVendor': System.getProperty("java.vendor"), + 'kotlinVersion': "${kotlinVersion}".toString()] +} \ No newline at end of file diff --git a/performance/framework/gradle.properties b/performance/framework/gradle.properties new file mode 100644 index 00000000000..87803bed88a --- /dev/null +++ b/performance/framework/gradle.properties @@ -0,0 +1 @@ +org.jetbrains.kotlin.native.home=../../dist \ No newline at end of file diff --git a/performance/settings.gradle b/performance/settings.gradle index 89a03c33563..ce753c6c8f2 100644 --- a/performance/settings.gradle +++ b/performance/settings.gradle @@ -1,4 +1,5 @@ include ':ring' include ':cinterop' include ':helloworld' -include ':videoplayer' \ No newline at end of file +include ':videoplayer' +include ':framework' \ No newline at end of file diff --git a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt index 5ed9c034bc6..e486bd9ed06 100644 --- a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt +++ b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt @@ -72,8 +72,8 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v object BenchmarksRunner { fun parse(args: Array): ArgParser { val options = listOf( - OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "0"), - OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "10"), + OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "20"), + OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "60"), OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""), OptionDescriptor(ArgType.String(), "output", "o", "Output file"), OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true) diff --git a/tools/performance-server/ui/css/style.css b/tools/performance-server/ui/css/style.css index ad8e1c06073..20e250c4ee9 100644 --- a/tools/performance-server/ui/css/style.css +++ b/tools/performance-server/ui/css/style.css @@ -45,14 +45,18 @@ border-color: gold; } .ct-legend .ct-series-2:before { + background-color: limegreen; + border-color: limegreen; +} +.ct-legend .ct-series-3:before { background-color: mediumorchid; border-color: mediumorchid; } -.ct-legend .ct-series-3:before { - background-color: #d17905; - border-color: #d17905; -} .ct-legend .ct-series-4:before { + background-color: navy; + border-color: navy; +} +.ct-legend .ct-series-5:before { background-color: #453d3f; border-color: #453d3f; } @@ -66,16 +70,18 @@ .ct-series-c .ct-line, .ct-series-c .ct-point { /* Set the colour of this series line */ - stroke: mediumorchid; -} - -.ct-series-d .ct-line { - /* Set the colour of this series line */ - stroke: mediumorchid; + stroke: limegreen; } +.ct-series-d .ct-line, .ct-series-d .ct-point { /* Set the colour of this series line */ stroke: mediumorchid; - stroke-width: 20; -} \ No newline at end of file +} + +.ct-series-e .ct-line, +.ct-series-e .ct-point { + /* Set the colour of this series line */ + stroke: navy; +} +.tooltip { pointer-events: none; } \ No newline at end of file diff --git a/tools/performance-server/ui/index.ejs b/tools/performance-server/ui/index.ejs index 03dac2662a6..6891279ac62 100644 --- a/tools/performance-server/ui/index.ejs +++ b/tools/performance-server/ui/index.ejs @@ -11,7 +11,7 @@ - diff --git a/tools/performance-server/ui/src/main/kotlin/main.kt b/tools/performance-server/ui/src/main/kotlin/main.kt index 35c4a3311ec..26a924bf984 100644 --- a/tools/performance-server/ui/src/main/kotlin/main.kt +++ b/tools/performance-server/ui/src/main/kotlin/main.kt @@ -69,13 +69,13 @@ fun separateValues(values: String, valuesContainer: MutableMap, valuesList: Collection>, className: String? = null): dynamic { +fun getChartData(labels: List, valuesList: Collection>, classNames: Array? = null): dynamic { val chartData: dynamic = object{} chartData["labels"] = labels.toTypedArray() - chartData["series"] = valuesList.map { + chartData["series"] = valuesList.mapIndexed { index, it -> val series: dynamic = object{} series["data"] = it.toTypedArray() - className?.let { series["className"] = className } + classNames?.let { series["className"] = classNames[index] } series }.toTypedArray() return chartData @@ -142,7 +142,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam "L", data.x + pointSize, data.y + pointSize/2, "z").joinToString(" ") svgParameters["style"] = "fill:rgb(255,0,0);stroke-width:0" val triangle = Chartist.Svg("path", svgParameters, chartContainer) - element = data.element._node.replace(triangle) + element = data.element.replace(triangle) } else if (currentBuild.buildNumber == parameters["build"]) { // Higlight choosen build. val svgParameters: dynamic = object{} @@ -228,7 +228,7 @@ fun main(args: Array) { val branchesUrl = "$serverUrl/branches/${parameters["target"]}" val branches: Array = JSON.parse(sendGetRequest(branchesUrl)) - val releaseBranches = branches.filter { "v\\d+\\.\\d+\\.\\d+-fixes".toRegex().find(it) != null } + val releaseBranches = branches.filter { "^v\\d+\\.\\d+\\.\\d+-fixes$".toRegex().find(it) != null } // Fill autocomplete list. val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}" @@ -312,17 +312,17 @@ fun main(args: Array) { bundleSize.add(it.bundleSize?.toInt()?. let { it / 1024 / 1024 }) } - val sizeClassName = "ct-series-c" + val sizeClassNames = arrayOf("ct-series-d", "ct-series-e") // Draw charts. val execChart = Chartist.Line("#exec_chart", getChartData(labels, executionTime.values), getChartOptions(executionTime.keys.toTypedArray(), "Normalized time")) val compileChart = Chartist.Line("#compile_chart", getChartData(labels, compileTime.values), getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds")) - val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassName), - getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-2"))) - val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassName), - getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-2"))) + val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassNames), + getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-3", "ct-series-4"))) + val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassNames), + getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-3"))) // Tooltips and higlights. customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)