diff --git a/build-tools/build.gradle.kts b/build-tools/build.gradle.kts index 2f884107fb4..78c437329b1 100644 --- a/build-tools/build.gradle.kts +++ b/build-tools/build.gradle.kts @@ -65,7 +65,7 @@ dependencies { } sourceSets["main"].withConvention(KotlinSourceSet::class) { - kotlin.srcDir("$projectDir/../tools/benchmarks/shared/src") + kotlin.srcDir("$projectDir/../tools/benchmarks/shared/src/main/kotlin/report") } gradlePlugin { diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt index 5cde686e331..a67b2f1c113 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt @@ -6,46 +6,31 @@ package org.jetbrains.kotlin import org.gradle.api.DefaultTask -import org.gradle.api.tasks.Input import org.gradle.api.tasks.TaskAction - -import org.jetbrains.report.json.* - -import java.io.FileInputStream -import java.io.OutputStreamWriter import java.io.BufferedReader +import java.io.FileInputStream import java.io.InputStreamReader +import java.io.OutputStreamWriter import java.net.HttpURLConnection import java.net.URL -import java.util.Properties +import java.util.* -typealias performanceAdditionalResult = Triple /** - * Task to produce regressions report and send it to slack. Requires a report with current benchmarks result - * and path to analyzer tool + * Task to save benchmarks results on server. * - * @property currentBenchmarksReportFile path to file with becnhmarks result - * @property analyzer path to analyzer tool * @property bundleSize size of build * @property onlyBranch register only builds for branch + * @property fileWithResult json file with benchmarks run results */ open class BuildRegister : DefaultTask() { - @Input - lateinit var currentBenchmarksReportFile: String - @Input - lateinit var analyzer: String - var onlyBranch: String? = null var bundleSize: Int? = null + var fileWithResult: String = "nativeReport.json" - val buildInfoTokens: Int = 4 - val additionalInfoTokens: Int = 3 - val compileTimeSamplesNumber: Int = 2 - val buildNumberRegex: Regex = "\\d+(\\.\\d+)+(-M\\d)?-(\\w+)-\\d+".toRegex() val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg" - private fun sendPostRequest(url: String, body: String) : String { + private fun sendPostRequest(url: String, body: String): String { val connection = URL(url).openConnection() as HttpURLConnection return connection.apply { setRequestProperty("Content-Type", "application/json; charset=utf-8") @@ -71,131 +56,33 @@ open class BuildRegister : DefaultTask() { } } - fun getAdditionalInfo(analyzer: String, reportFile: String, alwaysExists: Boolean, benchmarkName: String, - showedName: String = benchmarkName) : - performanceAdditionalResult? { - val output = arrayOf(analyzer, "summary", "--compile", "samples", - "--compile-samples", benchmarkName, "--codesize-samples", benchmarkName, - "--codesize-normalize", "artifactory:builds/goldenResults.csv", reportFile) - .runCommand() - - - val buildInfoParts = output.split(',') - if (buildInfoParts.size != additionalInfoTokens) { - val message = "Problems with getting summary information using $analyzer and $reportFile. $output" - if (!alwaysExists) { - println(message) - return null - } - error(message) - } - val (failures, compileTime, codeSize) = buildInfoParts.map { it.trim() } - val codeSizeInfo = "$showedName-$codeSize" - val compileTimeInfo = "$showedName-$compileTime" - return performanceAdditionalResult(failures, compileTimeInfo, codeSizeInfo) - } - @TaskAction fun run() { // Get TeamCity properties. - val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?: - error("Can't load teamcity config!") + val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?: error("Can't load teamcity config!") val buildProperties = Properties() buildProperties.load(FileInputStream(teamcityConfig)) val buildId = buildProperties.getProperty("teamcity.build.id") val teamCityUser = buildProperties.getProperty("teamcity.auth.userId") val teamCityPassword = buildProperties.getProperty("teamcity.auth.password") - val buildNumber = buildProperties.getProperty("build.number") - val apiKey = buildProperties.getProperty("artifactory.apikey") // Get branch. val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword) - val branch = getBuildProperty(currentBuild,"branchName") - - val target = System.getProperty("os.name").replace("\\s".toRegex(), "") - - // Get summary information. - val output = arrayOf("$analyzer", "summary", "--exec-samples", "all", "--compile", "samples", - "--compile-samples", "HelloWorld,Videoplayer", "--codesize-samples", "all", - "--exec-normalize", "artifactory:builds/goldenResults.csv", - "--codesize-normalize", "artifactory:builds/goldenResults.csv", "$currentBenchmarksReportFile") - .runCommand() - - // Postprocess information. - val buildInfoParts = output.split(',') - if (buildInfoParts.size != buildInfoTokens) { - error("Problems with getting summary information using $analyzer and $currentBenchmarksReportFile. $output") - } - - val (failures, executionTime, compileTime, codeSize) = buildInfoParts.map { it.trim() } - var failuresNumber = failures.toInt() - // Add legends. - val geometricMean = "Geometric Mean-" - val executionTimeInfo = "$geometricMean$executionTime" - var codeSizeInfo = "$geometricMean$codeSize" - val compileTimeSamples = compileTime.split(';') - if (compileTimeSamples.size != compileTimeSamplesNumber) { - error("Problems with getting compile time samples value. Expected at least $compileTimeSamplesNumber samples, got ${compileTimeSamples.size}") - } - val (helloWorldCompile, videoplayerCompile) = compileTimeSamples - var compileTimeInfo = "HelloWorld-$helloWorldCompile;Videoplayer-$videoplayerCompile" - - // Collect framework run details. - if (target == "MacOSX") { - val frameworkResults = getAdditionalInfo(analyzer, currentBenchmarksReportFile, true, - "FrameworkBenchmarksAnalyzer") - frameworkResults?.let { - val (_, frameworkCompileTime, frameworkCodeSize) = it - codeSizeInfo += ";$frameworkCodeSize" - compileTimeInfo += ";$frameworkCompileTime" - } - - val spaceResults = getAdditionalInfo(analyzer, - "artifactory:$buildNumber:$target:spaceFrameworkReport.json", false, - "circlet_iosX64", "SpaceFramework_iosX64") - spaceResults?.let { - val (failures, frameworkCompileTime, frameworkCodeSize) = it - failuresNumber += failures.toInt() - codeSizeInfo += ";$frameworkCodeSize" - compileTimeInfo += ";$frameworkCompileTime" - } - } - - if (target == "Linux") { - val coroutinesResults = getAdditionalInfo(analyzer, - "artifactory:$buildNumber:$target:externalReport.json", false, - "kotlinx.coroutines") - coroutinesResults?.let { - val (failures, libraryCompileTime, libraryCodeSize) = it - failuresNumber += failures.toInt() - codeSizeInfo += ";$libraryCodeSize" - compileTimeInfo += ";$libraryCompileTime" - } - } - - val matchResult = buildNumberRegex.find(buildNumber) ?: error("Wrong format of build number $buildNumber.") - val buildType = matchResult.groups[3]!!.value + val branch = getBuildProperty(currentBuild, "branchName") // Send post request to register build. val requestBody = buildString { append("{\"buildId\":\"$buildId\",") append("\"teamCityUser\":\"$teamCityUser\",") append("\"teamCityPassword\":\"$teamCityPassword\",") - append("\"artifactoryApiKey\":\"$apiKey\",") - append("\"target\": \"$target\",") - append("\"buildType\": \"$buildType\",") - append("\"failuresNumber\": $failuresNumber,") - append("\"executionTime\": \"$executionTimeInfo\",") - append("\"compileTime\": \"$compileTimeInfo\",") - append("\"codeSize\": \"$codeSizeInfo\",") - append("\"bundleSize\": ${bundleSize?.let {"\"$bundleSize\""} ?: bundleSize}}") + append("\"fileWithResult\":\"$fileWithResult\",") + append("\"bundleSize\": ${bundleSize?.let { "\"$bundleSize\"" } ?: bundleSize}}") } if (onlyBranch == null || onlyBranch == branch) { println(sendPostRequest("$performanceServer/register", requestBody)) } else { println("Skipping registration. Current branch $branch, need registration for $onlyBranch!") } - } } \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt index a0c8e191eb1..03a076be46e 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt @@ -104,7 +104,7 @@ fun createJsonReport(projectProperties: Map): String { val kotlin = Compiler(backend, getValue("kotlinVersion")) val benchDesc = getValue("benchmarks") val benchmarksArray = JsonTreeParser.parse(benchDesc) - val benchmarks = BenchmarksReport.parseBenchmarksArray(benchmarksArray) + val benchmarks = parseBenchmarksArray(benchmarksArray) .union(projectProperties["compileTime"] as List).union( listOf(projectProperties["codeSize"] as? BenchmarkResult).filterNotNull()).toList() val report = BenchmarksReport(env, benchmarks, kotlin) diff --git a/performance/build.gradle b/performance/build.gradle index 86cb5b0ebaf..de781fcb4cb 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -56,7 +56,8 @@ private String findAnalyzerBinary() { // Produce and send slack report. task slackReport(type: RegressionsReporter) { - def analyzerBinary = findAnalyzerBinary() + def analyzerBinary = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}", + "${rootBuildDirectory}/${analyzerToolDirectory}") def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") if (teamcityConfig && analyzerBinary != null) { // Create folder for report (root Kotlin project settings make create report in separate folder). @@ -159,21 +160,23 @@ task registerExternalBenchmarks { } task registerBuild(type: BuildRegister) { - def analyzerBinary = findAnalyzerBinary() - if (analyzerBinary != null) { - onlyBranch = project.findProperty('kotlin.register.branch') - // Get bundle size. - bundleSize = null - if (project.findProperty('kotlin.bundleBuild') != null) { - def dist = findProperty('kotlin.native.home') ?: 'dist' - dist = (new File(dist)).isAbsolute() ? dist : "${project.getProjectDir()}/$dist" - bundleSize = (new File(dist)).directorySize() - } - currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}" - analyzer = analyzerBinary + onlyBranch = project.findProperty('kotlin.register.branch') + // Get bundle size. + bundleSize = null + if (project.findProperty('kotlin.bundleBuild') != null) { + def dist = findProperty('kotlin.native.home') ?: 'dist' + dist = (new File(dist)).isAbsolute() ? dist : "${project.getProjectDir()}/$dist" + bundleSize = (new File(dist)).directorySize() } } +task registerExternalBuild(type: BuildRegister) { + onlyBranch = project.findProperty('kotlin.register.branch') + fileWithResult = externalBenchmarksReport +} + +registerExternalBenchmarks.finalizedBy registerExternalBuild + def mergeReports(String fileName) { def reports = [] subprojects.each { @@ -265,4 +268,4 @@ task videoplayer { task KotlinVsSwift { dependsOn 'clean' dependsOn 'KotlinVsSwift:konanRun' -} +} \ No newline at end of file diff --git a/performance/cinterop/build.gradle.kts b/performance/cinterop/build.gradle.kts index f325edf12fc..f885f4325ba 100644 --- a/performance/cinterop/build.gradle.kts +++ b/performance/cinterop/build.gradle.kts @@ -14,7 +14,7 @@ val defaultBuildType = NativeBuildType.RELEASE benchmark { applicationName = "Cinterop" - commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") + commonSrcDirs = listOf("../../tools/benchmarks/shared/src/main/kotlin/report", "src/main/kotlin", "../shared/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm") nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") diff --git a/performance/numerical/build.gradle.kts b/performance/numerical/build.gradle.kts index 9fd84a31c68..ede81d6f69c 100644 --- a/performance/numerical/build.gradle.kts +++ b/performance/numerical/build.gradle.kts @@ -17,7 +17,7 @@ val defaultBuildType = NativeBuildType.RELEASE benchmark { applicationName = "Numerical" - commonSrcDirs = listOf("src/main/kotlin", "../../tools/benchmarks/shared/src", "../shared/src/main/kotlin") + commonSrcDirs = listOf("src/main/kotlin", "../../tools/benchmarks/shared/src/main/kotlin/report", "../shared/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm") nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("../shared/src/main/kotlin-native/mingw") diff --git a/performance/objcinterop/build.gradle.kts b/performance/objcinterop/build.gradle.kts index 62bea4337c8..71c4c7322f1 100644 --- a/performance/objcinterop/build.gradle.kts +++ b/performance/objcinterop/build.gradle.kts @@ -17,7 +17,7 @@ val defaultBuildType = NativeBuildType.RELEASE benchmark { applicationName = "ObjCInterop" - commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") + commonSrcDirs = listOf("../../tools/benchmarks/shared/src/main/kotlin/report", "src/main/kotlin", "../shared/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm") nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") diff --git a/performance/ring/build.gradle.kts b/performance/ring/build.gradle.kts index f73da76453e..2eda534e016 100644 --- a/performance/ring/build.gradle.kts +++ b/performance/ring/build.gradle.kts @@ -13,7 +13,7 @@ val defaultBuildType = NativeBuildType.RELEASE benchmark { applicationName = "Ring" - commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") + commonSrcDirs = listOf("../../tools/benchmarks/shared/src/main/kotlin/report", "src/main/kotlin", "../shared/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm") nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") diff --git a/performance/startup/build.gradle.kts b/performance/startup/build.gradle.kts index 5effe87088b..df66b9598f2 100644 --- a/performance/startup/build.gradle.kts +++ b/performance/startup/build.gradle.kts @@ -15,7 +15,7 @@ val defaultBuildType = NativeBuildType.RELEASE benchmark { applicationName = "Startup" - commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") + commonSrcDirs = listOf("../../tools/benchmarks/shared/src/main/kotlin/report", "src/main/kotlin", "../shared/src/main/kotlin") jvmSrcDirs = listOf("../shared/src/main/kotlin-jvm") nativeSrcDirs = listOf("../shared/src/main/kotlin-native/common") mingwSrcDirs = listOf("../shared/src/main/kotlin-native/mingw") diff --git a/performance/swiftinterop/build.gradle.kts b/performance/swiftinterop/build.gradle.kts index 70bc0ac11e6..e5149a22829 100644 --- a/performance/swiftinterop/build.gradle.kts +++ b/performance/swiftinterop/build.gradle.kts @@ -15,7 +15,7 @@ val defaultBuildType = NativeBuildType.RELEASE swiftBenchmark { applicationName = "swiftInterop" - commonSrcDirs = listOf("$toolsPath/benchmarks/shared/src/main/kotlin", "src", "../shared/src/main/kotlin") + commonSrcDirs = listOf("$toolsPath/benchmarks/shared/src/main/kotlin/report", "src", "../shared/src/main/kotlin") nativeSrcDirs = listOf("../shared/src/main/kotlin-native/common", "../shared/src/main/kotlin-native/posix") swiftSources = listOf("$projectDir/swiftSrc/benchmarks.swift", "$projectDir/swiftSrc/main.swift") compileTasks = listOf("compileKotlinNative", "linkBenchmarkReleaseFrameworkNative") diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt b/tools/benchmarks/shared/src/main/kotlin/analyzer/FieldChange.kt similarity index 62% rename from tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt rename to tools/benchmarks/shared/src/main/kotlin/analyzer/FieldChange.kt index aaa0b618663..8bef7cb23e8 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt +++ b/tools/benchmarks/shared/src/main/kotlin/analyzer/FieldChange.kt @@ -1,20 +1,8 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.analyzer // Report with changes of different fields. diff --git a/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt b/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt new file mode 100644 index 00000000000..adb1d226f6e --- /dev/null +++ b/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt @@ -0,0 +1,131 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.analyzer + +import org.jetbrains.report.BenchmarkResult +import org.jetbrains.report.MeanVariance +import org.jetbrains.report.MeanVarianceBenchmark +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.math.sqrt + +val MeanVariance.description: String + get() { + val format = { number: Double -> number.format(2) } + return "${format(mean)} ± ${format(variance)}" + } + +val MeanVarianceBenchmark.description: String + get() = "${score.format()} ± ${variance.format()}" + +// Calculate difference in percentage compare to another. +fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance { + assert(other.score >= 0 && + other.variance >= 0 && + other.score - other.variance != 0.0, + { "Mean and variance should be positive and not equal!" }) + // Analyze intervals. Calculate difference between border points. + val (bigValue, smallValue) = if (score > other.score) Pair(this, other) else Pair(other, this) + val bigValueIntervalStart = bigValue.score - bigValue.variance + val bigValueIntervalEnd = bigValue.score + bigValue.variance + val smallValueIntervalStart = smallValue.score - smallValue.variance + val smallValueIntervalEnd = smallValue.score + smallValue.variance + if (smallValueIntervalEnd > bigValueIntervalStart) { + // Interval intersect. + return MeanVariance(0.0, 0.0) + } + val mean = ((smallValueIntervalEnd - bigValueIntervalStart) / bigValueIntervalStart) * + (if (score > other.score) -1 else 1) + + val maxValueChange = ((bigValueIntervalEnd - smallValueIntervalEnd) / bigValueIntervalEnd) + val minValueChange = ((bigValueIntervalStart - smallValueIntervalStart) / bigValueIntervalStart) + val variance = abs(abs(mean) - max(minValueChange, maxValueChange)) + return MeanVariance(mean * 100, variance * 100) +} + +// Calculate ratio value compare to another. +fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance { + assert(other.score >= 0 && + other.variance >= 0 && + other.score - other.variance != 0.0, + { "Mean and variance should be positive and not equal!" }) + val mean = score / other.score + val minRatio = (score - variance) / (other.score + other.variance) + val maxRatio = (score + variance) / (other.score - other.variance) + val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean)) + return MeanVariance(mean, ratioConfInt) +} + +fun geometricMean(values: Collection, totalNumber: Int = values.size) = + with(values.asSequence().filter { it != 0.0 }) { + if (count() == 0) { + 0.0 + } else { + map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b } + } + } + +fun computeMeanVariance(samples: List): MeanVariance { + val zStar = 1.67 // Critical point for 90% confidence of normal distribution. + val mean = samples.sum() / samples.size + val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size + val confidenceInterval = sqrt(variance / samples.size) * zStar + return MeanVariance(mean, confidenceInterval) +} + +// Calculate average results for benchmarks (each benchmark can be run several times). +fun collectMeanResults(benchmarks: Map>): BenchmarksTable { + return benchmarks.map { (name, resultsSet) -> + val repeatedSequence = IntArray(resultsSet.size) + var metric = BenchmarkResult.Metric.EXECUTION_TIME + var currentStatus = BenchmarkResult.Status.PASSED + var currentWarmup = -1 + + // Results can be already processed. + if (resultsSet[0] is MeanVarianceBenchmark) { + assert(resultsSet.size == 1) { "Several MeanVarianceBenchmark instances." } + name to resultsSet[0] as MeanVarianceBenchmark + } else { + // Collect common benchmark values and check them. + resultsSet.forEachIndexed { index, result -> + // If there was at least one failure, summary is marked as failure. + if (result.status == BenchmarkResult.Status.FAILED) { + currentStatus = result.status + } + repeatedSequence[index] = result.repeat + if (currentWarmup != -1) + if (result.warmup != currentWarmup) + println("Check data consistency. Warmup value for benchmark '${result.name}' differs.") + currentWarmup = result.warmup + metric = result.metric + } + + repeatedSequence.sort() + // Check if there are missed loop during running benchmarks. + repeatedSequence.forEachIndexed { index, element -> + if (index != 0) + if ((element - repeatedSequence[index - 1]) != 1) + println("Check data consistency. For benchmark '$name' there is no run" + + " between ${repeatedSequence[index - 1]} and $element.") + } + + // Create mean and variance benchmarks result. + val scoreMeanVariance = computeMeanVariance(resultsSet.map { it.score }) + val runtimeInUsMeanVariance = computeMeanVariance(resultsSet.map { it.runtimeInUs }) + val meanBenchmark = MeanVarianceBenchmark(name, currentStatus, scoreMeanVariance.mean, metric, + runtimeInUsMeanVariance.mean, repeatedSequence[resultsSet.size - 1], + currentWarmup, scoreMeanVariance.variance) + name to meanBenchmark + } + }.toMap() +} + +fun collectBenchmarksDurations(benchmarks: Map>): Map = + benchmarks.map { (name, resultsSet) -> + name to resultsSet.sumByDouble { it.runtimeInUs } + }.toMap() \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt similarity index 77% rename from tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt rename to tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt index 2725a07c006..6048d975197 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt +++ b/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt @@ -1,27 +1,17 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.analyzer -import kotlin.math.abs import org.jetbrains.report.BenchmarkResult -import org.jetbrains.report.Environment -import org.jetbrains.report.Compiler import org.jetbrains.report.BenchmarksReport +import org.jetbrains.report.Compiler +import org.jetbrains.report.Environment +import org.jetbrains.report.MeanVariance +import org.jetbrains.report.MeanVarianceBenchmark +import kotlin.math.abs typealias SummaryBenchmark = Pair typealias BenchmarksTable = Map @@ -29,12 +19,12 @@ typealias SummaryBenchmarksTable = Map typealias ScoreChange = Pair // Summary report with comparasion of separate benchmarks results. -class SummaryBenchmarksReport (val currentReport: BenchmarksReport, - val previousReport: BenchmarksReport? = null, - val meaningfulChangesValue: Double = 0.5) { +class SummaryBenchmarksReport(val currentReport: BenchmarksReport, + val previousReport: BenchmarksReport? = null, + val meaningfulChangesValue: Double = 0.5) { // Report created by joining comparing reports. val mergedReport: Map - val benchmarksDurations: Map> + private val benchmarksDurations: Map> // Lists of benchmarks in different status. private val benchmarksWithChangedStatus = mutableListOf>() @@ -56,8 +46,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, // Countable properties. val failedBenchmarks: List - get() = mergedReport.filter { it.value.first?.meanBenchmark?.status == BenchmarkResult.Status.FAILED } - .map { it.key } + get() = mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED } + .map { it.key } val addedBenchmarks: List get() = mergedReport.filter { it.value.second == null }.map { it.key } @@ -72,7 +62,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! } val currentBenchmarksDuration: Map - get() = benchmarksDurations.filter{ it.value.first != null }.map { it.key to it.value.first!! }.toMap() + get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap() val maximumRegression: Double get() = getMaximumChange(regressions) @@ -133,8 +123,15 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, } } + // Get benchmark report. + fun getBenchmarksReport(takeMainReport: Boolean = true) = + if (takeMainReport) + BenchmarksReport(environments.first, mergedReport.map { (_, value) -> value.first!! }, compilers.first) + else + BenchmarksReport(environments.second!!, mergedReport.map { (_, value) -> value.second!! }, compilers.second!!) + fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List? = null, - normalizeData: Map>? = null): List { + normalizeData: Map>? = null): List { val benchmarks = filter?.let { mergedReport.filter { entry -> filter.find { @@ -142,30 +139,26 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, } != null } } ?: mergedReport - if (benchmarks.isEmpty()) { - error("There is no benchmarks from provided list") - } - val filteredBenchmarks = benchmarks.filter { entry -> entry.value.first!!.meanBenchmark.metric == metric } - if (filteredBenchmarks.isEmpty()) { - error("There is no benchmarks for metric $metric") - } - val results = filteredBenchmarks.map { entry -> - val score = entry.value.first!!.meanBenchmark.score + val results = benchmarks.map { entry -> val name = entry.key.removeSuffix(metric.suffix) - val value = normalizeData?.let { - it.get(name)?.get("$metric")?.let { score / it } - ?: error("No normalization data for benchmark $name and metric $metric") - } ?: score - name to value }.toMap() + if (entry.value.first!!.metric == metric) { + val score = entry.value.first!!.score + val value = normalizeData?.let { + it.get(name)?.get("$metric")?.let { score / it } + ?: error("No normalization data for benchmark $name and metric $metric") + } ?: score + name to value + } else name to null + }.toMap() if (getGeoMean) { - return listOf(geometricMean(results.values)) + return listOf(geometricMean(results.values.filterNotNull())) } - return filter?.let { it.map { results[it] ?: error("Benchmark $it for metric $metric doesn't exist.") }.toList() } ?: results.values.toList() + return filter?.let { it.map { results[it] }.toList() } ?: results.values.toList() } private fun getMaximumChange(bucket: Map): Double = - // Maps of regressions and improvements are sorted. - if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first() + // Maps of regressions and improvements are sorted. + if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first() private fun getGeometricMeanOfChanges(bucket: Map): Double { if (bucket.isEmpty()) @@ -185,11 +178,9 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, // Create geometric mean. private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark { val geoMeanBenchmarkName = "Geometric mean" - val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.meanBenchmark.score }) - val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.varianceBenchmark.score }) - val meanBenchmark = BenchmarkResult(geoMeanBenchmarkName, geoMean) - val varianceBenchmark = BenchmarkResult(geoMeanBenchmarkName, varianceGeoMean) - return MeanVarianceBenchmark(meanBenchmark, varianceBenchmark) + val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score }) + val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance }) + return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean) } // Generate map with summary durations of each benchmark. @@ -209,16 +200,15 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, val mergedTable = mutableMapOf() mergedTable.apply { currentBenchmarks.forEach { (name, current) -> - val currentBenchmark = current.meanBenchmark // Check existance of benchmark in previous results. if (previousBenchmarks == null || name !in previousBenchmarks) { getOrPut(name) { SummaryBenchmark(current, null) } } else { - val previousBenchmark = previousBenchmarks.getValue(name).meanBenchmark + val previousBenchmark = previousBenchmarks.getValue(name) getOrPut(name) { SummaryBenchmark(current, previousBenchmarks[name]) } // Explore change of status. - if (previousBenchmark.status != currentBenchmark.status) { - val statusChange = FieldChange("$name", previousBenchmark.status, currentBenchmark.status) + if (previousBenchmark.status != current.status) { + val statusChange = FieldChange("$name", previousBenchmark.status, current.status) benchmarksWithChangedStatus.add(statusChange) } } @@ -240,7 +230,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, SummaryBenchmark { // Calculate geometric mean. val currentGeoMean = createGeoMeanBenchmark(currentBenchmarks) - val previousGeoMean = previousBenchmarks?. let { createGeoMeanBenchmark(previousBenchmarks) } + val previousGeoMean = previousBenchmarks?.let { createGeoMeanBenchmark(previousBenchmarks) } return SummaryBenchmark(currentGeoMean, previousGeoMean) } @@ -261,7 +251,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, // Analyze and collect changes in performance between same becnhmarks. private fun analyzePerformanceChanges() { - val performanceChanges = mergedReport.asSequence().map {(name, element) -> + val performanceChanges = mergedReport.asSequence().map { (name, element) -> getBenchmarkPerfomanceChange(name, element) }.filterNotNull().groupBy { if (it.second.first.mean > 0) "regressions" else "improvements" @@ -277,8 +267,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, // Calculate change for geometric mean. val (current, previous) = geoMeanBenchmark - geoMeanScoreChange = current?. let { - previous?. let { + geoMeanScoreChange = current?.let { + previous?.let { Pair(current.calcPercentageDiff(previous), current.calcRatio(previous)) } } diff --git a/tools/benchmarks/shared/src/main/kotlin/analyzer/Utils.kt b/tools/benchmarks/shared/src/main/kotlin/analyzer/Utils.kt new file mode 100644 index 00000000000..1f36fe6abc0 --- /dev/null +++ b/tools/benchmarks/shared/src/main/kotlin/analyzer/Utils.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.analyzer + +expect fun readFile(fileName: String): String +expect fun Double.format(decimalNumber: Int = 4): String +expect fun writeToFile(fileName: String, text: String) +expect fun assert(value: Boolean, lazyMessage: () -> Any) +expect fun sendGetRequest(url: String, user: String? = null, password: String? = null, + followLocation: Boolean = false) : String \ No newline at end of file diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt b/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt similarity index 63% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt rename to tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt index 9c9e2eccb63..7448d89117b 100644 --- a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt +++ b/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt @@ -1,26 +1,22 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.report import org.jetbrains.report.json.* interface JsonSerializable { - fun toJson(): String + fun serializeFields(): String + + fun toJson(): String { + return """ + { + ${serializeFields()} + } + """ + } // Convert iterable objects arrays, lists to json. fun arrayToJson(data: Iterable): String { @@ -30,51 +26,63 @@ interface JsonSerializable { } } -interface EntityFromJsonFactory: ConvertedFromJson { +interface EntityFromJsonFactory : ConvertedFromJson { fun create(data: JsonElement): T } -// Class for benchcmarks report with all information of run. -class BenchmarksReport(val env: Environment, benchmarksList: List, val compiler: Compiler): +// Parse array with benchmarks to list +fun parseBenchmarksArray(data: JsonElement): List { + if (data is JsonArray) { + return data.jsonArray.map { + if (MeanVarianceBenchmark.isMeanVarianceBenchmark(it)) + MeanVarianceBenchmark.create(it as JsonObject) + else BenchmarkResult.create(it as JsonObject) + } + } else { + error("Benchmarks field is expected to be an array. Please, check origin files.") + } +} + +// Class for benchmarks report with all information of run. +open class BenchmarksReport(val env: Environment, benchmarksList: List, val compiler: Compiler) : JsonSerializable { - companion object: EntityFromJsonFactory { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): BenchmarksReport { if (data is JsonObject) { val env = Environment.create(data.getRequiredField("env")) val benchmarksObj = data.getRequiredField("benchmarks") val compiler = Compiler.create(data.getRequiredField("kotlin")) + val buildNumberField = data.getOptionalField("buildNumber") val benchmarksList = parseBenchmarksArray(benchmarksObj) - return BenchmarksReport(env, benchmarksList, compiler) + val report = BenchmarksReport(env, benchmarksList, compiler) + buildNumberField?.let { report.buildNumber = (it as JsonLiteral).unquoted() } + return report } else { error("Top level entity is expected to be an object. Please, check origin files.") } } - // Parse array with benchmarks to list - fun parseBenchmarksArray(data: JsonElement): List { - if (data is JsonArray) { - return data.jsonArray.map { BenchmarkResult.create(it as JsonObject) } - } else { - error("Benchmarks field is expected to be an array. Please, check origin files.") - } - } - // Made a map of becnhmarks with name as key from list. private fun structBenchmarks(benchmarksList: List) = - benchmarksList.groupBy{ it.name } + benchmarksList.groupBy { it.name } } - val benchmarks: Map> = structBenchmarks(benchmarksList) + val benchmarks = structBenchmarks(benchmarksList) - override fun toJson(): String { + var buildNumber: String? = null + + override fun serializeFields(): String { + val buildNumberField = buildNumber?.let { + """, + "buildNumber": "$buildNumber" + """ + } ?: "" return """ - { "env": ${env.toJson()}, "kotlin": ${compiler.toJson()}, - "benchmarks": ${arrayToJson(benchmarks.flatMap{it.value})} - } - """ + "benchmarks": ${arrayToJson(benchmarks.flatMap { it.value })}$buildNumberField + """.trimIndent() } fun merge(other: BenchmarksReport): BenchmarksReport { @@ -85,27 +93,27 @@ class BenchmarksReport(val env: Environment, benchmarksList: List { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): Compiler { if (data is JsonObject) { val backend = Backend.create(data.getRequiredField("backend")) @@ -121,13 +129,14 @@ data class Compiler(val backend: Backend, val kotlinVersion: String): JsonSerial } // Class for compiler backend - data class Backend(val type: BackendType, val version: String, val flags: List): JsonSerializable { - companion object: EntityFromJsonFactory { + data class Backend(val type: BackendType, val version: String, val flags: List) : JsonSerializable { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): Backend { if (data is JsonObject) { val typeElement = data.getRequiredField("type") if (typeElement is JsonLiteral) { - val type = backendTypeFromString(typeElement.unquoted()) ?: error("Backend type should be 'jvm' or 'native'") + val type = backendTypeFromString(typeElement.unquoted()) + ?: error("Backend type should be 'jvm' or 'native'") val version = elementToString(data.getRequiredField("version"), "version") val flagsArray = data.getOptionalField("flags") var flags: List = emptyList() @@ -144,40 +153,35 @@ data class Compiler(val backend: Backend, val kotlinVersion: String): JsonSerial } } - override fun toJson(): String { + override fun serializeFields(): String { val result = """ - { "type": "${type.type}", "version": "${version}"""" // Don't print flags field if there is no one. if (flags.isEmpty()) { return """$result - }""" - } - else { + """ + } else { return """ $result, - "flags": ${arrayToJson(flags)} - } + "flags": ${arrayToJson(flags.map { if (it.startsWith("\"")) it else "\"$it\"" })} """ } } } - override fun toJson(): String { + override fun serializeFields(): String { return """ - { "backend": ${backend.toJson()}, "kotlinVersion": "${kotlinVersion}" - } """ } } // Class for description of environment of benchmarks run -data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializable { +data class Environment(val machine: Machine, val jdk: JDKInstance) : JsonSerializable { - companion object: EntityFromJsonFactory { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): Environment { if (data is JsonObject) { val machine = Machine.create(data.getRequiredField("machine")) @@ -191,8 +195,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ } // Class for description of machine used for benchmarks run. - data class Machine(val cpu: String, val os: String): JsonSerializable { - companion object: EntityFromJsonFactory { + data class Machine(val cpu: String, val os: String) : JsonSerializable { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): Machine { if (data is JsonObject) { val cpu = elementToString(data.getRequiredField("cpu"), "cpu") @@ -205,19 +209,17 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ } } - override fun toJson(): String { + override fun serializeFields(): String { return """ - { "cpu": "$cpu", "os": "$os" - } """ } } // Class for description of jdk used for benchmarks run. - data class JDKInstance(val version: String, val vendor: String): JsonSerializable { - companion object: EntityFromJsonFactory { + data class JDKInstance(val version: String, val vendor: String) : JsonSerializable { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): JDKInstance { if (data is JsonObject) { val version = elementToString(data.getRequiredField("version"), "version") @@ -230,47 +232,44 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ } } - override fun toJson(): String { + override fun serializeFields(): String { return """ - { "version": "$version", "vendor": "$vendor" - } """ } } - override fun toJson(): String { + override fun serializeFields(): String { return """ - { "machine": ${machine.toJson()}, "jdk": ${jdk.toJson()} - } """ } } -class BenchmarkResult(val name: String, val status: Status, - val score: Double, val metric: Metric, val runtimeInUs: Double, - val repeat: Int, val warmup: Int): JsonSerializable { +open class BenchmarkResult(val name: String, val status: Status, + val score: Double, val metric: Metric, val runtimeInUs: Double, + val repeat: Int, val warmup: Int) : JsonSerializable { enum class Metric(val suffix: String, val value: String) { EXECUTION_TIME("", "EXECUTION_TIME"), CODE_SIZE(".codeSize", "CODE_SIZE"), - COMPILE_TIME(".compileTime", "COMPILE_TIME") + COMPILE_TIME(".compileTime", "COMPILE_TIME"), + BUNDLE_SIZE(".bundleSize", "BUNDLE_SIZE") } constructor(name: String, score: Double) : this(name, Status.PASSED, score, Metric.EXECUTION_TIME, 0.0, 0, 0) - companion object: EntityFromJsonFactory { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): BenchmarkResult { if (data is JsonObject) { var name = elementToString(data.getRequiredField("name"), "name") val metricElement = data.getOptionalField("metric") val metric = if (metricElement != null && metricElement is JsonLiteral) - metricFromString(metricElement.unquoted()) ?: Metric.EXECUTION_TIME - else Metric.EXECUTION_TIME + metricFromString(metricElement.unquoted()) ?: Metric.EXECUTION_TIME + else Metric.EXECUTION_TIME name += metric.suffix val statusElement = data.getRequiredField("status") if (statusElement is JsonLiteral) { @@ -300,9 +299,8 @@ class BenchmarkResult(val name: String, val status: Status, FAILED("FAILED") } - override fun toJson(): String { + override fun serializeFields(): String { return """ - { "name": "${name.removeSuffix(metric.suffix)}", "status": "${status.value}", "score": ${score}, @@ -310,10 +308,44 @@ class BenchmarkResult(val name: String, val status: Status, "runtimeInUs": ${runtimeInUs}, "repeat": ${repeat}, "warmup": ${warmup} - } """ } val shortName: String get() = name.removeSuffix(metric.suffix) +} + +// Entity to describe avarage values which conssists of mean and variance values. +data class MeanVariance(val mean: Double, val variance: Double) + +// Processed benchmark result with calculated mean and variance value. +open class MeanVarianceBenchmark(name: String, status: BenchmarkResult.Status, score: Double, metric: BenchmarkResult.Metric, + runtimeInUs: Double, repeat: Int, warmup: Int, val variance: Double) : + BenchmarkResult(name, status, score, metric, runtimeInUs, repeat, warmup) { + + constructor(name: String, score: Double, variance: Double) : this(name, BenchmarkResult.Status.PASSED, score, + BenchmarkResult.Metric.EXECUTION_TIME, 0.0, 0, 0, variance) + + companion object : EntityFromJsonFactory { + + fun isMeanVarianceBenchmark(data: JsonElement) = data is JsonObject && data.getOptionalField("variance") != null + + override fun create(data: JsonElement): MeanVarianceBenchmark { + if (data is JsonObject) { + val baseBenchmark = BenchmarkResult.create(data) + val variance = elementToDouble(data.getRequiredField("variance"), "variance") + return MeanVarianceBenchmark(baseBenchmark.name, baseBenchmark.status, baseBenchmark.score, baseBenchmark.metric, + baseBenchmark.runtimeInUs, baseBenchmark.repeat, baseBenchmark.warmup, variance) + } else { + error("Benchmark entity is expected to be an object. Please, check origin files.") + } + } + } + + override fun serializeFields(): String { + return """ + ${super.serializeFields()}, + "variance": $variance + """ + } } \ No newline at end of file diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt b/tools/benchmarks/shared/src/main/kotlin/report/json/ConvertedFromJson.kt similarity index 100% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt rename to tools/benchmarks/shared/src/main/kotlin/report/json/ConvertedFromJson.kt diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt b/tools/benchmarks/shared/src/main/kotlin/report/json/JsonElement.kt similarity index 100% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt rename to tools/benchmarks/shared/src/main/kotlin/report/json/JsonElement.kt diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonExceptions.kt b/tools/benchmarks/shared/src/main/kotlin/report/json/JsonExceptions.kt similarity index 100% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonExceptions.kt rename to tools/benchmarks/shared/src/main/kotlin/report/json/JsonExceptions.kt diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonParser.kt b/tools/benchmarks/shared/src/main/kotlin/report/json/JsonParser.kt similarity index 99% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonParser.kt rename to tools/benchmarks/shared/src/main/kotlin/report/json/JsonParser.kt index 1f2b13a483f..42f51e67083 100644 --- a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonParser.kt +++ b/tools/benchmarks/shared/src/main/kotlin/report/json/JsonParser.kt @@ -121,10 +121,12 @@ internal class Parser(val source: String) { TC_BEGIN_LIST, TC_BEGIN_OBJ, TC_OTHER, TC_STRING, TC_NULL -> true else -> false } + + @OptIn(ExperimentalStdlibApi::class) fun takeStr(): String { if (tc != TC_OTHER && tc != TC_STRING) fail(tokenPos, "Expected string or non-null literal") val prevStr = if (offset < 0) - String(buf, 0, length) else + buf.concatToString(0, length) else source.substring(offset, offset + length) nextToken() return prevStr diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonTreeParser.kt b/tools/benchmarks/shared/src/main/kotlin/report/json/JsonTreeParser.kt similarity index 100% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonTreeParser.kt rename to tools/benchmarks/shared/src/main/kotlin/report/json/JsonTreeParser.kt diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/StringOps.kt b/tools/benchmarks/shared/src/main/kotlin/report/json/StringOps.kt similarity index 100% rename from tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/StringOps.kt rename to tools/benchmarks/shared/src/main/kotlin/report/json/StringOps.kt diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt index 1e6fad63d76..996a676b1fd 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt @@ -1,17 +1,6 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ package org.jetbrains.analyzer diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt index 0bf8b023a1a..6878ff341f0 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt @@ -1,17 +1,6 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ package org.jetbrains.analyzer @@ -29,7 +18,7 @@ actual fun readFile(fileName: String): String { } actual fun Double.format(decimalNumber: Int): String = - "%.${decimalNumber}f".format(this) + "%.${decimalNumber}f".format(this) actual fun writeToFile(fileName: String, text: String) { File(fileName).printWriter().use { out -> diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt index f5e4d72c0ad..7c32e581e7c 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt @@ -1,17 +1,6 @@ /* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ package org.jetbrains.analyzer diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index 7db7abfd088..0bfe5f6cf24 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -1,23 +1,12 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ -@file:OptIn(ExperimentalCli::class) + +import kotlinx.cli.* import org.jetbrains.analyzer.sendGetRequest import org.jetbrains.analyzer.readFile import org.jetbrains.analyzer.SummaryBenchmarksReport -import kotlinx.cli.* import org.jetbrains.renders.* import org.jetbrains.report.* import org.jetbrains.report.json.* @@ -56,7 +45,7 @@ object ArtifactoryConnector : Connector() { } } -object TeamCityConnector: Connector() { +object TeamCityConnector : Connector() { override val connectorPrefix = "teamcity:" val teamCityUrl = "http://buildserver.labs.intellij.net" @@ -76,11 +65,28 @@ object TeamCityConnector: Connector() { } } +object DBServerConnector : Connector() { + override val connectorPrefix = "" + val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg" + + override fun getFileContent(fileLocation: String, user: String?): String { + val buildNumber = fileLocation.substringBefore(':') + val target = fileLocation.substringAfter(':') + if (target == buildNumber) { + error("To get file from database, please, specify, target and build number" + + " in format target:build_number") + } + val accessFileUrl = "$serverUrl/report/$target/$buildNumber" + return sendGetRequest(accessFileUrl) + } +} + fun getFileContent(fileName: String, user: String? = null): String { return when { ArtifactoryConnector.isCompatible(fileName) -> ArtifactoryConnector.getFileContent(fileName, user) TeamCityConnector.isCompatible(fileName) -> TeamCityConnector.getFileContent(fileName, user) - else -> readFile(fileName) + fileName.endsWith(".json") -> readFile(fileName) + else -> DBServerConnector.getFileContent(fileName, user) } } @@ -108,12 +114,16 @@ fun parseNormalizeResults(results: String): Map> { return parsedNormalizeResults } -fun mergeCompilerFlags(reports: List) = - reports.map { +fun mergeCompilerFlags(reports: List): List { + val flagsMap = mutableMapOf>() + reports.forEach { val benchmarks = it.benchmarks.values.flatten().asSequence().filter { it.metric == BenchmarkResult.Metric.COMPILE_TIME } - .map { it.shortName }.distinct().sorted().joinToString() - "${it.compiler.backend.flags.joinToString()} for [$benchmarks]" + .map { it.shortName }.toList() + if (benchmarks.isNotEmpty()) + (flagsMap.getOrPut("${it.compiler.backend.flags.joinToString()}") { mutableListOf() }).addAll(benchmarks) } + return flagsMap.map { (flags, benchmarks) -> "$flags for [${benchmarks.distinct().sorted().joinToString()}]" } +} fun mergeReportsWithDetailedFlags(reports: List) = if (reports.size > 1) { @@ -129,75 +139,9 @@ fun mergeReportsWithDetailedFlags(reports: List) = } fun main(args: Array) { - class Summary: Subcommand("summary", "Output summary information") { - val exec by option(ArgType.Choice(listOf("samples", "geomean")), - description = "Execution time way of calculation").default("geomean") - val execSamples by option(ArgType.String, "exec-samples", - description = "Samples used for execution time metric (value 'all' allows use all samples)") - .delimiter(",") - val execNormalize by option(ArgType.String, "exec-normalize", - description = "File with golden results which should be used for normalization") - val compile by option(ArgType.Choice(listOf("samples", "geomean")), - description = "Compile time way of calculation").default("geomean") - val compileSamples by option(ArgType.String, "compile-samples", - description = "Samples used for compile time metric (value 'all' allows use all samples)") - .delimiter(",") - val compileNormalize by option(ArgType.String, "compile-normalize", - description = "File with golden results which should be used for normalization") - val codesize by option(ArgType.Choice(listOf("samples", "geomean")), - description = "Code size way of calculation").default("geomean") - val codesizeSamples by option(ArgType.String, "codesize-samples", - description = "Samples used for code size metric (value 'all' allows use all samples)").delimiter(",") - val codesizeNormalize by option(ArgType.String, "codesize-normalize", - description = "File with golden results which should be used for normalization") - val user by option(ArgType.String, shortName = "u", description = "User access information for authorization") - val mainReport by argument(ArgType.String, description = "Main report for analysis") - - override fun execute() { - val reportsList = getBenchmarkReport(mainReport, user) - val report = reportsList.reduce { result, it -> - result.merge(it) - } - val benchsReport = SummaryBenchmarksReport(report) - val results = mutableListOf() - val executionNormalize = execNormalize?.let { - parseNormalizeResults(getFileContent(it)) - } - val compileNormalize = compileNormalize?.let { - parseNormalizeResults(getFileContent(it)) - } - val codesizeNormalize = codesizeNormalize?.let { - parseNormalizeResults(getFileContent(it)) - } - - results.apply { - add(benchsReport.failedBenchmarks.size.toString()) - if (!execSamples.isEmpty()) { - val filterExec = if (execSamples.first() == "all") null else execSamples - add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.EXECUTION_TIME, - exec == "geomean", filterExec, executionNormalize).joinToString(";")) - } - - if (!compileSamples.isEmpty()) { - val filterCompile = if (compileSamples.first() == "all") null else compileSamples - add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.COMPILE_TIME, - compile == "geomean", filterCompile, compileNormalize).joinToString(";")) - } - - if (!codesizeSamples.isEmpty()) { - val filterCodesize = if (codesizeSamples.first() == "all") null else codesizeSamples - add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.CODE_SIZE, - codesize == "geomean", filterCodesize, codesizeNormalize).joinToString(";")) - } - - } - println(results.joinToString()) - } - } - val action = Summary() // Parse args. val argParser = ArgParser("benchmarksAnalyzer") - argParser.subcommands(action) + val mainReport by argParser.argument(ArgType.String, description = "Main report for analysis") val compareToReport by argParser.argument(ArgType.String, description = "Report to compare to").optional() @@ -207,26 +151,25 @@ fun main(args: Array) { val useShortForm by argParser.option(ArgType.Boolean, "short", "s", "Show short version of report").default(false) val renders by argParser.option(ArgType.Choice(listOf("text", "html", "teamcity", "statistics", "metrics")), - shortName = "r", description = "Renders for showing information").multiple().default(listOf("text")) + shortName = "r", description = "Renders for showing information").multiple().default(listOf("text")) val user by argParser.option(ArgType.String, shortName = "u", description = "User access information for authorization") - if (argParser.parse(args).commandName == "benchmarksAnalyzer") { - // Read contents of file. - val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user)) + argParser.parse(args) + // Read contents of file. + val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user)) - var compareToBenchsReport = compareToReport?.let { - mergeReportsWithDetailedFlags(getBenchmarkReport(it, user)) - } + var compareToBenchsReport = compareToReport?.let { + mergeReportsWithDetailedFlags(getBenchmarkReport(it, user)) + } - // Generate comparasion report. - val summaryReport = SummaryBenchmarksReport(mainBenchsReport, - compareToBenchsReport, - epsValue) + // Generate comparasion report. + val summaryReport = SummaryBenchmarksReport(mainBenchsReport, + compareToBenchsReport, + epsValue) - var outputFile = output - renders.forEach { - Render.getRenderByName(it).print(summaryReport, useShortForm, outputFile) - outputFile = null - } + var outputFile = output + renders.forEach { + Render.getRenderByName(it).print(summaryReport, useShortForm, outputFile) + outputFile = null } } \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt deleted file mode 100644 index 2ee516a44b7..00000000000 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.jetbrains.analyzer -import org.jetbrains.report.BenchmarkResult -import kotlin.math.abs -import kotlin.math.max -import kotlin.math.min -import kotlin.math.pow -import kotlin.math.sqrt - -// Entity to describe avarage values which conssists of mean and variance values. -data class MeanVariance(val mean: Double, val variance: Double) { - override fun toString(): String { - val format = { number: Double -> number.format(2)} - return "${format(mean)} ± ${format(variance)}" - } -} - -// Composite benchmark which descibes avarage result for several runs and contains mean and variance value. -data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianceBenchmark: BenchmarkResult) { - - // Calculate difference in percentage compare to another. - fun calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance { - assert(other.meanBenchmark.score >= 0 && - other.varianceBenchmark.score >= 0 && - other.meanBenchmark.score - other.varianceBenchmark.score != 0.0, - { "Mean and variance should be positive and not equal!" }) - val exactMean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score - // Analyze intervals. Calculate difference between border points. - val (bigValue, smallValue) = if (meanBenchmark.score > other.meanBenchmark.score) Pair(this, other) else Pair(other, this) - val bigValueIntervalStart = bigValue.meanBenchmark.score - bigValue.varianceBenchmark.score - val bigValueIntervalEnd = bigValue.meanBenchmark.score + bigValue.varianceBenchmark.score - val smallValueIntervalStart = smallValue.meanBenchmark.score - smallValue.varianceBenchmark.score - val smallValueIntervalEnd = smallValue.meanBenchmark.score + smallValue.varianceBenchmark.score - if (smallValueIntervalEnd > bigValueIntervalStart) { - // Interval intersect. - return MeanVariance(0.0, 0.0) - } - val mean = ((smallValueIntervalEnd - bigValueIntervalStart) / bigValueIntervalStart) * - (if (meanBenchmark.score > other.meanBenchmark.score) -1 else 1) - - val maxValueChange = ((bigValueIntervalEnd - smallValueIntervalEnd) / bigValueIntervalEnd) - val minValueChange = ((bigValueIntervalStart - smallValueIntervalStart) / bigValueIntervalStart) - val variance = abs(abs(mean) - max(minValueChange, maxValueChange)) - return MeanVariance(mean * 100, variance * 100) - } - - // Calculate ratio value compare to another. - fun calcRatio(other: MeanVarianceBenchmark): MeanVariance { - assert(other.meanBenchmark.score >= 0 && - other.varianceBenchmark.score >= 0 && - other.meanBenchmark.score - other.varianceBenchmark.score != 0.0, - { "Mean and variance should be positive and not equal!" }) - val mean = meanBenchmark.score / other.meanBenchmark.score - val minRatio = (meanBenchmark.score - varianceBenchmark.score) / (other.meanBenchmark.score + other.varianceBenchmark.score) - val maxRatio = (meanBenchmark.score + varianceBenchmark.score) / (other.meanBenchmark.score - other.varianceBenchmark.score) - val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean)) - return MeanVariance(mean, ratioConfInt) - } - - override fun toString(): String = - "${meanBenchmark.score.format()} ± ${varianceBenchmark.score.format()}" - -} - -fun geometricMean(values: Collection, totalNumber: Int = values.size) = - with(values.asSequence().filter { it != 0.0 }) { - if (count() == 0) { - 0.0 - } else { - map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b } - } - } - -fun computeMeanVariance(samples: List): MeanVariance { - val zStar = 1.67 // Critical point for 90% confidence of normal distribution. - val mean = samples.sum() / samples.size - val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size - val confidenceInterval = sqrt(variance / samples.size) * zStar - return MeanVariance(mean, confidenceInterval) -} - -// Calculate avarage results for bencmarks (each becnhmark can be run several times). -fun collectMeanResults(benchmarks: Map>): BenchmarksTable { - return benchmarks.map {(name, resultsSet) -> - val repeatedSequence = IntArray(resultsSet.size) - var metric = BenchmarkResult.Metric.EXECUTION_TIME - var currentStatus = BenchmarkResult.Status.PASSED - var currentWarmup = -1 - - // Collect common becnhmark values and check them. - resultsSet.forEachIndexed { index, result -> - // If there was at least one failure, summary is marked as failure. - if (result.status == BenchmarkResult.Status.FAILED) { - currentStatus = result.status - } - repeatedSequence[index] = result.repeat - if (currentWarmup != -1) - if (result.warmup != currentWarmup) - println("Check data consistency. Warmup value for benchmark '${result.name}' differs.") - currentWarmup = result.warmup - metric = result.metric - } - - repeatedSequence.sort() - // Check if there are missed loop during running benchmarks. - repeatedSequence.forEachIndexed { index, element -> - if (index != 0) - if ((element - repeatedSequence[index - 1]) != 1) - println("Check data consistency. For benchmark '$name' there is no run" + - " between ${repeatedSequence[index - 1]} and $element.") - } - - // Create mean and variance benchmarks result. - val scoreMeanVariance = computeMeanVariance(resultsSet.map { it.score }) - val runtimeInUsMeanVariance = computeMeanVariance(resultsSet.map { it.runtimeInUs }) - val meanBenchmark = BenchmarkResult(name, currentStatus, scoreMeanVariance.mean, metric, - runtimeInUsMeanVariance.mean, repeatedSequence[resultsSet.size - 1], - currentWarmup) - val varianceBenchmark = BenchmarkResult(name, currentStatus, scoreMeanVariance.variance, metric, - runtimeInUsMeanVariance.variance, repeatedSequence[resultsSet.size - 1], - currentWarmup) - name to MeanVarianceBenchmark(meanBenchmark, varianceBenchmark) - }.toMap() -} - -fun collectBenchmarksDurations(benchmarks: Map>): Map = - benchmarks.map { (name, resultsSet) -> - name to resultsSet.sumByDouble { it.runtimeInUs } - }.toMap() \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt deleted file mode 100644 index dc21d68c94d..00000000000 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.analyzer - -expect fun readFile(fileName: String): String -expect fun Double.format(decimalNumber: Int = 4): String -expect fun writeToFile(fileName: String, text: String) -expect fun assert(value: Boolean, lazyMessage: () -> Any) -expect fun sendGetRequest(url: String, user: String? = null, password: String? = null, - followLocation: Boolean = false) : String \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt index 9874913eeab..d9423e3fc7d 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt @@ -1,20 +1,8 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.renders import org.jetbrains.analyzer.* @@ -556,7 +544,7 @@ class HTMLRender: Render() { bucket: Map? = null, rowStyle: String? = null) { if (bucket != null && !bucket.isEmpty()) { // Find max ratio. - val maxRatio = bucket.values.map { it.second.mean }.max()!! + val maxRatio = bucket.values.map { it.second.mean }.maxOrNull()!! // There are changes in performance. // Output changed benchmarks. for ((name, change) in bucket) { @@ -565,19 +553,19 @@ class HTMLRender: Render() { attributes["style"] = rowStyle } th { +name } - td { +"${fullSet.getValue(name).first}" } - td { +"${fullSet.getValue(name).second}" } + td { +"${fullSet.getValue(name).first?.description}" } + td { +"${fullSet.getValue(name).second?.description}" } td { attributes["bgcolor"] = ColoredCell(if (bucket.values.first().first.mean == 0.0) null else change.first.mean / abs(bucket.values.first().first.mean)) .backgroundStyle - +"${change.first.toString() + " %"}" + +"${change.first.description + " %"}" } td { val scaledRatio = if (maxRatio == 0.0) null else change.second.mean / maxRatio attributes["bgcolor"] = ColoredCell(scaledRatio, borderPositive = { cellValue -> cellValue > 1.0 / maxRatio }).backgroundStyle - +"${change.second}" + +"${change.second.description}" } } } @@ -587,8 +575,8 @@ class HTMLRender: Render() { for ((name, value) in fullSet) { tr { th { +name } - td { +"${value.first?.toString() ?: placeholder}" } - td { +"${value.second?.toString() ?: placeholder}" } + td { +"${value.first?.description ?: placeholder}" } + td { +"${value.second?.description ?: placeholder}" } td { +placeholder } td { +placeholder } } @@ -619,11 +607,11 @@ class HTMLRender: Render() { } } val geoMeanChangeMap = report.geoMeanScoreChange?. - let { mapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanScoreChange!!) } + let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) } tbody { renderBenchmarksDetails( - mutableMapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanBenchmark), + mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark), geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black") renderBenchmarksDetails(report.mergedReport, report.regressions) renderBenchmarksDetails(report.mergedReport, report.improvements) diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/JsonResultsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/JsonResultsRender.kt new file mode 100644 index 00000000000..60c47b2a5a1 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/JsonResultsRender.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.renders + +import org.jetbrains.analyzer.* +import org.jetbrains.report.* + +// Report render to text format. +class JsonResultsRender: Render() { + override val name: String + get() = "json" + + override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean) = + report.getBenchmarksReport().toJson() +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt index a0f8b4620cf..bab962287c6 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt @@ -1,20 +1,8 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.renders import org.jetbrains.analyzer.* @@ -28,10 +16,10 @@ class MetricResultsRender: Render() { override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { val results = report.mergedReport.map { entry -> buildString { - val metric = entry.value.first!!.meanBenchmark.metric + val metric = entry.value.first!!.metric append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",") append("\"metric\": \"${metric}\",") - append("\"value\": \"${entry.value.first!!.meanBenchmark.score}\" }") + append("\"value\": \"${entry.value.first!!.score}\" }") } }.joinToString(", ") return "[ $results ]" diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt index 6d02346eda1..1342f8bc52e 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt @@ -1,24 +1,12 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.renders import org.jetbrains.analyzer.* -import org.jetbrains.report.BenchmarkResult +import org.jetbrains.report.* import kotlin.math.abs @@ -150,23 +138,23 @@ class TextRender: Render() { private fun printBenchmarksDetails(fullSet: Map, bucket: Map? = null) { + val placeholder = "-" if (bucket != null) { // There are changes in performance. // Output changed benchmarks. for ((name, change) in bucket) { append(formatColumn(name, true) + - formatColumn(fullSet.getValue(name).first.toString()) + - formatColumn(fullSet.getValue(name).second.toString()) + - formatColumn(change.first.toString() + " %") + - formatColumn(change.second.toString())) + formatColumn(fullSet.getValue(name).first?.description ?: placeholder) + + formatColumn(fullSet.getValue(name).second?.description ?: placeholder) + + formatColumn(change.first.description + " %") + + formatColumn(change.second.description)) } } else { // Output all values without performance changes. - val placeholder = "-" for ((name, value) in fullSet) { append(formatColumn(name, true) + - formatColumn(value.first?.toString() ?: placeholder) + - formatColumn(value.second?.toString() ?: placeholder) + + formatColumn(value.first?.description ?: placeholder) + + formatColumn(value.second?.description ?: placeholder) + formatColumn(placeholder) + formatColumn(placeholder)) } @@ -201,9 +189,9 @@ class TextRender: Render() { val tableWidth = printPerformanceTableHeader() // Print geometric mean. val geoMeanChangeMap = report.geoMeanScoreChange?. - let { mapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanScoreChange!!) } + let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) } printBenchmarksDetails( - mutableMapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanBenchmark), + mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark), geoMeanChangeMap) printTableLineSeparator(tableWidth) printBenchmarksDetails(report.mergedReport, report.regressions) diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt index a4a73252b56..abccbb5808c 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt @@ -1,20 +1,8 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.renders import org.jetbrains.analyzer.* diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt index d6f343002dc..548a421a65d 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt @@ -1,24 +1,13 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.renders import org.jetbrains.analyzer.* import org.jetbrains.report.BenchmarkResult +import org.jetbrains.report.MeanVarianceBenchmark // Report render to text format. class TeamCityStatisticsRender: Render() { @@ -34,9 +23,8 @@ class TeamCityStatisticsRender: Render() { // For current benchmarks print score as TeamCity Test Metadata report.currentMeanVarianceBenchmarks.forEach { benchmark -> - renderBenchmark(benchmark.meanBenchmark, currentDurations[benchmark.meanBenchmark.name]!!) - renderSummaryBecnhmarkValue(benchmark.meanBenchmark, "Mean") - renderSummaryBecnhmarkValue(benchmark.varianceBenchmark, "Variance") + renderBenchmark(benchmark, currentDurations[benchmark.name]!!) + renderSummaryBecnhmarkValue(benchmark) } content.append("##teamcity[testSuiteFinished name='Benchmarks']\n") @@ -46,9 +34,12 @@ class TeamCityStatisticsRender: Render() { return content.toString() } - private fun renderSummaryBecnhmarkValue(benchmark: BenchmarkResult, metric: String) = - content.append("##teamcity[testMetadata testName='${benchmark.name}' name='$metric'" + - " type='number' value='${benchmark.score}']\n") + private fun renderSummaryBecnhmarkValue(benchmark: MeanVarianceBenchmark) { + content.append("##teamcity[testMetadata testName='${benchmark.name}' name='Mean'" + + " type='number' value='${benchmark.score}']\n") + content.append("##teamcity[testMetadata testName='${benchmark.name}' name='Variance'" + + " type='number' value='${benchmark.variance}']\n") + } // Produce benchmark as test in TeamCity private fun renderBenchmark(benchmark: BenchmarkResult , duration: Double) { @@ -61,7 +52,7 @@ class TeamCityStatisticsRender: Render() { } private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) { - content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n") - content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n") + content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.score}']\n") + content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.variance}']\n") } } \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt index aaa07ca9af6..506244686af 100644 --- a/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt +++ b/tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt @@ -1,37 +1,21 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - package org.jetbrains.analyzer import kotlin.test.* import kotlin.math.abs import org.jetbrains.report.BenchmarkResult +import org.jetbrains.report.MeanVarianceBenchmark class AnalyzerTests { private val eps = 0.000001 private fun createMeanVarianceBenchmarks(): Pair { - val firstMean = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 9.0, BenchmarkResult.Metric.EXECUTION_TIME, 9.0, 10, 10) - val firstVariance = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 0.0001, BenchmarkResult.Metric.EXECUTION_TIME, 0.0001, 10, 10) - val first = MeanVarianceBenchmark(firstMean, firstVariance) - - val secondMean = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 10.0, BenchmarkResult.Metric.EXECUTION_TIME, 10.0, 10, 10) - val secondVariance = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 0.0001, BenchmarkResult.Metric.EXECUTION_TIME, 0.0001, 10, 10) - val second = MeanVarianceBenchmark(secondMean, secondVariance) + val first = MeanVarianceBenchmark("testBenchmark", BenchmarkResult.Status.PASSED, 9.0, BenchmarkResult.Metric.EXECUTION_TIME, 9.0, 10, 10, 0.0001) + val second = MeanVarianceBenchmark("testBenchmark", BenchmarkResult.Status.PASSED, 10.0, BenchmarkResult.Metric.EXECUTION_TIME, 10.0, 10, 10, 0.0001) return Pair(first, second) } diff --git a/tools/performance-server/build.gradle b/tools/performance-server/build.gradle index 4648999932c..a6d0020106e 100644 --- a/tools/performance-server/build.gradle +++ b/tools/performance-server/build.gradle @@ -18,6 +18,8 @@ buildscript { } } +apply plugin: 'kotlin-multiplatform' + repositories { maven { url 'https://cache-redirector.jetbrains.com/jcenter' @@ -30,22 +32,33 @@ repositories { } } -apply plugin: 'kotlin2js' +kotlin { + sourceSets { + commonMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion" + } + kotlin.srcDir '../benchmarks/shared/src' + } + jsMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-js:$kotlinVersion" + implementation(npm("aws-sdk", "~2.670.0")) + } + kotlin.srcDir 'src/main/kotlin' + kotlin.srcDir 'src/main/kotlin-js' + kotlin.srcDir 'shared/src/main/kotlin' + } + } -dependencies { - compile "org.jetbrains.kotlin:kotlin-stdlib-js:$kotlinVersion" -} - - -sourceSets { - main.resources.srcDirs += "resources" - main.kotlin.srcDirs += "src" - main.kotlin.srcDirs += "../benchmarks/shared/src" - main.kotlin.srcDirs += "shared/src" -} - -compileKotlin2Js { - kotlinOptions.outputFile = "${projectDir}/server/app.js" - kotlinOptions.moduleKind = "commonjs" - kotlinOptions.sourceMap = true + targets { + fromPreset(presets.js, 'js') { + nodejs() + compilations.main.kotlinOptions { + outputFile = "${projectDir}/server/app.js" + moduleKind = "commonjs" + sourceMap = true + } + } + } } \ No newline at end of file diff --git a/tools/performance-server/gradle/wrapper/gradle-wrapper.jar b/tools/performance-server/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000000..f3d88b1c2fa Binary files /dev/null and b/tools/performance-server/gradle/wrapper/gradle-wrapper.jar differ diff --git a/tools/performance-server/gradle/wrapper/gradle-wrapper.properties b/tools/performance-server/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..1b16c34a71c --- /dev/null +++ b/tools/performance-server/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/tools/performance-server/gradlew b/tools/performance-server/gradlew new file mode 100755 index 00000000000..2fe81a7d95e --- /dev/null +++ b/tools/performance-server/gradlew @@ -0,0 +1,183 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/tools/performance-server/gradlew.bat b/tools/performance-server/gradlew.bat new file mode 100644 index 00000000000..24467a141f7 --- /dev/null +++ b/tools/performance-server/gradlew.bat @@ -0,0 +1,100 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/tools/performance-server/package-lock.json b/tools/performance-server/package-lock.json index aad66f40392..a7adb37e4a5 100644 --- a/tools/performance-server/package-lock.json +++ b/tools/performance-server/package-lock.json @@ -4,32 +4,6 @@ "lockfileVersion": 1, "requires": true, "dependencies": { - "@types/concat-stream": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz", - "integrity": "sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0=", - "requires": { - "@types/node": "*" - } - }, - "@types/form-data": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-0.0.33.tgz", - "integrity": "sha1-yayFsqX9GENbjIXZ7LUObWyJP/g=", - "requires": { - "@types/node": "*" - } - }, - "@types/node": { - "version": "9.6.44", - "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.44.tgz", - "integrity": "sha512-hR+jK4kSOX+u5aQ6Zj6az5N4l1AeXH37/sOenbgxJvkO450C5qQL4/1twdufIMLC3o3hNfJNrPfQn0ivMdKfPg==" - }, - "@types/qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-47kAAs3yV/hROraCTQYDMh4p/6zI9+gtssjD0kq9OWsGdLcBge59rl49FnCuJ+iWxEKiqFz6KXzeGH5DRVjNJA==" - }, "accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", @@ -44,15 +18,26 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, - "asap": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + "aws-sdk": { + "version": "2.670.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.670.0.tgz", + "integrity": "sha512-hGRnZtp1wDUh6hZRBHO0Ki7thx/xbRlIEiTKlWes+f/0E1Nhm3KpelsBZ3L/Q6y1ragwkQd4Q720AmWEqemLyA==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "body-parser": { "version": "1.18.3", @@ -81,40 +66,21 @@ } } }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } }, "bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, "content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -135,11 +101,6 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -155,11 +116,6 @@ } } }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -195,6 +151,11 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "express": { "version": "4.16.4", "resolved": "https://registry.npmjs.org/express/-/express-4.16.4.tgz", @@ -276,16 +237,6 @@ } } }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -296,24 +247,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, - "get-port": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", - "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=" - }, - "http-basic": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/http-basic/-/http-basic-7.0.0.tgz", - "integrity": "sha1-gvClBr6UJzLsje6+6A50bvVzbbo=", - "requires": { - "@types/concat-stream": "^1.6.0", - "@types/node": "^9.4.1", - "caseless": "~0.12.0", - "concat-stream": "^1.4.6", - "http-response-object": "^3.0.1", - "parse-cache-control": "^1.0.1" - } - }, "http-errors": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -325,14 +258,6 @@ "statuses": ">= 1.4.0 < 2" } }, - "http-response-object": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.1.tgz", - "integrity": "sha512-6L0Fkd6TozA8kFSfh9Widst0wfza3U1Ex2RjJ6zNDK0vR1U1auUR6jY4Nn2Xl7CCy0ikFmxW1XcspVpb9RvwTg==", - "requires": { - "@types/node": "^9.3.0" - } - }, "iconv-lite": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", @@ -341,6 +266,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", @@ -356,10 +286,15 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "kotlin": { - "version": "1.3.21", - "resolved": "https://registry.npmjs.org/kotlin/-/kotlin-1.3.21.tgz", - "integrity": "sha512-JqxY9quNzwWl8iYvNj/ycfNBEbSFDKvmeK2rDVU89q0/X8BRjVzibJ5RFOq/9efvvvi+VBnIRhL7GSOGzmP2nQ==" + "version": "1.4.0-M3", + "resolved": "https://registry.npmjs.org/kotlin/-/kotlin-1.4.0-M3.tgz", + "integrity": "sha512-y6dl1t36tI1hAJlJTxlOB/fj/0iOga+XC8+eg9hUSTu5BDJ9MIUR7XnA8Ld+tQA9IakAPaUmgZZfX2lsHaAdIw==" }, "media-typer": { "version": "0.3.0", @@ -404,6 +339,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -412,11 +352,6 @@ "ee-first": "1.1.1" } }, - "parse-cache-control": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", - "integrity": "sha1-juqz5U+laSD+Fro493+iGqzC104=" - }, "parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", @@ -427,19 +362,6 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "promise": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/promise/-/promise-8.0.2.tgz", - "integrity": "sha512-EIyzM39FpVOMbqgzEHhxdrEhtOSDOtjMZQ0M6iVfCE+kWNgCkAyOdnuCWqfmflylftfadU6FkiMgHZA2kUzwRw==", - "requires": { - "asap": "~2.0.6" - } - }, "proxy-addr": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", @@ -449,11 +371,21 @@ "ipaddr.js": "1.8.0" } }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "range-parser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", @@ -470,20 +402,6 @@ "unpipe": "1.0.0" } }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -494,6 +412,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, "send": { "version": "0.16.2", "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", @@ -550,57 +473,6 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "sync-request": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-6.0.0.tgz", - "integrity": "sha512-jGNIAlCi9iU4X3Dm4oQnNQshDD3h0/1A7r79LyqjbjUnj69sX6mShAXlhRXgImsfVKtTcnra1jfzabdZvp+Lmw==", - "requires": { - "http-response-object": "^3.0.1", - "sync-rpc": "^1.2.1", - "then-request": "^6.0.0" - } - }, - "sync-rpc": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.4.tgz", - "integrity": "sha512-Iug+t1ICVFenUcTnDu8WXFnT+k8IVoLKGh8VA3eXUtl2Rt9SjKX3YEv33OenABqpTPL9QEaHv1+CNn2LK8vMow==", - "requires": { - "get-port": "^3.1.0" - } - }, - "then-request": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/then-request/-/then-request-6.0.0.tgz", - "integrity": "sha512-xA+7uEMc+jsQIoyySJ93Ad08Kuqnik7u6jLS5hR91Z3smAoCfL3M8/MqMlobAa9gzBfO9pA88A/AntfepkkMJQ==", - "requires": { - "@types/concat-stream": "^1.6.0", - "@types/form-data": "0.0.33", - "@types/node": "^8.0.0", - "@types/qs": "^6.2.31", - "caseless": "~0.12.0", - "concat-stream": "^1.6.0", - "form-data": "^2.2.0", - "http-basic": "^7.0.0", - "http-response-object": "^3.0.1", - "promise": "^8.0.0", - "qs": "^6.4.0" - }, - "dependencies": { - "@types/node": { - "version": "8.10.42", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.42.tgz", - "integrity": "sha512-8LCqostMfYwQs9by1k21/P4KZp9uFQk3Q528y3qtPKQnCJmKz0Em3YzgeNjTNV1FVrG/7n/6j12d4UKg9zSgDw==" - } - } - }, "type-is": { "version": "1.6.16", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", @@ -610,30 +482,48 @@ "mime-types": "~2.1.18" } }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" } } } diff --git a/tools/performance-server/package.json b/tools/performance-server/package.json index ded496a07ed..dbbec39fc8d 100644 --- a/tools/performance-server/package.json +++ b/tools/performance-server/package.json @@ -6,11 +6,12 @@ "start": "node server/app.js" }, "dependencies": { + "aws-sdk": "~2.670.0", "body-parser": "~1.18.3", "debug": "~4.1.1", + "ejs": "~2.6.1", "express": "~4.16.4", - "kotlin": "^1.3.21", - "sync-request": "~6.0.0", - "ejs": "~2.6.1" + "kotlin": "~1.4.0-M3", + "node-fetch": "~2.6.0" } } diff --git a/tools/performance-server/shared/src/main/kotlin/org/jetbrains/build/Build.kt b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/buildInfo/BuildInfo.kt similarity index 54% rename from tools/performance-server/shared/src/main/kotlin/org/jetbrains/build/Build.kt rename to tools/performance-server/shared/src/main/kotlin/org/jetbrains/buildInfo/BuildInfo.kt index 28eb2c51624..2ced13b02a9 100644 --- a/tools/performance-server/shared/src/main/kotlin/org/jetbrains/build/Build.kt +++ b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/buildInfo/BuildInfo.kt @@ -1,45 +1,26 @@ /* - * Copyright 2010-2019 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ - -package org.jetbrains.build +package org.jetbrains.buildInfo import org.jetbrains.report.* import org.jetbrains.report.json.* data class Build(val buildNumber: String, val startTime: String, val finishTime: String, val branch: String, - val commits: String, val buildType: String, val failuresNumber: Int, val executionTime: String, - val compileTime: String, val codeSize: String, val bundleSize: String?) { + val commits: String, val failuresNumber: Int) { - companion object: EntityFromJsonFactory { + companion object : EntityFromJsonFactory { override fun create(data: JsonElement): Build { if (data is JsonObject) { - val buildNumber = elementToString(data.getRequiredField("buildNumber"), "buildNumber") - val startTime = elementToString(data.getRequiredField("startTime"), "startTime") - val finishTime = elementToString(data.getRequiredField("finishTime"), "finishTime") - val branch = elementToString(data.getRequiredField("branch"), "branch") + val buildNumber = elementToString(data.getRequiredField("buildNumber"), "buildNumber").replace("\"", "") + val startTime = elementToString(data.getRequiredField("startTime"), "startTime").replace("\"", "") + val finishTime = elementToString(data.getRequiredField("finishTime"), "finishTime").replace("\"", "") + val branch = elementToString(data.getRequiredField("branch"), "branch").replace("\"", "") val commits = elementToString(data.getRequiredField("commits"), "commits") - val buildType = elementToString(data.getRequiredField("buildType"), "buildType") val failuresNumber = elementToInt(data.getRequiredField("failuresNumber"), "failuresNumber") - val executionTime = elementToString(data.getRequiredField("executionTime"), "executionTime") - val compileTime = elementToString(data.getRequiredField("compileTime"), "compileTime") - val codeSize = elementToString(data.getRequiredField("codeSize"), "codeSize") - val bundleSize = elementToStringOrNull(data.getRequiredField("bundleSize"), "bundleSize") - return Build(buildNumber, startTime, finishTime, branch, commits, buildType, failuresNumber, executionTime, - compileTime, codeSize, bundleSize) + return Build(buildNumber, startTime, finishTime, branch, commits, failuresNumber) } else { error("Top level entity is expected to be an object. Please, check origin files.") } @@ -58,6 +39,7 @@ data class Build(val buildNumber: String, val startTime: String, val finishTime: return "${if (hours < 10) "0$hours" else "$hours"}:${matchResult[2]}" } ?: error { "Wrong format of time $startTime" } } + val date: String by lazy { val matchResult = "^(\\d{4})(\\d{2})(\\d{2})".toRegex().find(startTime)?.groupValues matchResult?.let { "${matchResult[3]}/${matchResult[2]}/${matchResult[1]}" } diff --git a/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchConnector.kt b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchConnector.kt new file mode 100644 index 00000000000..041fc4b686d --- /dev/null +++ b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchConnector.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.elastic + +import kotlin.js.Promise // TODO - migrate to multiplatform. +import org.jetbrains.report.json.* +import org.jetbrains.network.* + +// Connector with InfluxDB. +class ElasticSearchConnector(private val connector: NetworkConnector, + private val user: String? = null, private val password: String? = null) { + // Execute ElasticSearch request. + fun request(method: RequestMethod, path: String, acceptJsonContentType: Boolean = true, body: String? = null) = + connector.sendRequest(method, path, user, password, acceptJsonContentType, body) +} \ No newline at end of file diff --git a/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt new file mode 100644 index 00000000000..eeb9f9f218a --- /dev/null +++ b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt @@ -0,0 +1,224 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.elastic + +import org.jetbrains.report.* +import org.jetbrains.report.json.* +import org.jetbrains.report.MeanVarianceBenchmark +import org.jetbrains.network.* +import kotlin.js.Promise // TODO - migrate to multiplatform. + +data class Commit(val revision: String, val developer: String) : JsonSerializable { + override fun toString() = "$revision by $developer" + + override fun serializeFields() = """ + "revision": "$revision", + "developer": "$developer" + """ + + companion object : EntityFromJsonFactory { + fun parse(description: String) = if (description != "...") { + description.split(" by ").let { + val (currentRevision, currentDeveloper) = it + Commit(currentRevision, currentDeveloper) + } + } else { + Commit("unknown", "unknown") + } + + override fun create(data: JsonElement): Commit { + if (data is JsonObject) { + val revision = elementToString(data.getRequiredField("revision"), "revision") + val developer = elementToString(data.getRequiredField("developer"), "developer") + return Commit(revision, developer) + } else { + error("Top level entity is expected to be an object. Please, check origin files.") + } + } + } +} + +// List of commits. +class CommitsList : ConvertedFromJson, JsonSerializable { + + val commits: List + + constructor(data: JsonElement) { + if (data !is JsonObject) { + error("Commits description is expected to be a JSON object!") + } + val changesElement = data.getOptionalField("change") + commits = changesElement?.let { + if (changesElement !is JsonArray) { + error("Change field is expected to be an array. Please, check source.") + } + changesElement.jsonArray.map { + with(it as JsonObject) { + Commit(elementToString(getRequiredField("version"), "version"), + elementToString(getRequiredField("username"), "username") + ) + } + } + } ?: listOf() + } + + constructor(_commits: List) { + commits = _commits + } + + override fun toString(): String = + commits.toString() + + companion object { + fun parse(description: String) = CommitsList(description.split(";").filter { it.isNotEmpty() }.map { + Commit.parse(it) + }) + } + + override fun serializeFields() = """ + "commits": ${arrayToJson(commits)} + """ +} + +data class BuildInfo(val buildNumber: String, val startTime: String, val endTime: String, val commitsList: CommitsList, + val branch: String, + val agentInfo: String /* Important agent information often used in requests.*/) : JsonSerializable { + override fun serializeFields() = """ + "buildNumber": "$buildNumber", + "startTime": "$startTime", + "endTime": "$endTime", + ${commitsList.serializeFields()}, + "branch": "$branch", + "agentInfo": "$agentInfo" + """ + + companion object : EntityFromJsonFactory { + override fun create(data: JsonElement): BuildInfo { + if (data is JsonObject) { + val buildNumber = elementToString(data.getRequiredField("buildNumber"), "buildNumber") + val startTime = elementToString(data.getRequiredField("startTime"), "startTime") + val endTime = elementToString(data.getRequiredField("endTime"), "endTime") + val branch = elementToString(data.getRequiredField("branch"), "branch") + val commitsList = data.getRequiredField("commits") + val commits = if (commitsList is JsonArray) { + commitsList.jsonArray.map { Commit.create(it as JsonObject) } + } else { + error("benchmarksSets field is expected to be an array. Please, check origin files.") + } + val agentInfoElement = data.getOptionalField("agentInfo") + val agentInfo = agentInfoElement?.let { + elementToString(agentInfoElement, "agentInfo") + } ?: "" + return BuildInfo(buildNumber, startTime, endTime, CommitsList(commits), branch, agentInfo) + } else { + error("Top level entity is expected to be an object. Please, check origin files.") + } + } + } +} + +enum class ElasticSearchType { + TEXT, KEYWORD, DATE, LONG, DOUBLE, BOOLEAN, OBJECT, NESTED +} + +abstract class ElasticSearchIndex(val indexName: String, val connector: ElasticSearchConnector) { + var nextId = 0L + + // Insert data. + fun insert(data: JsonSerializable): Promise { + val description = data.toJson() + val writePath = "$indexName/_doc/$nextId?pretty" + nextId++ + return connector.request(RequestMethod.PUT, writePath, body = description) + } + + // Delete data. + fun delete(data: String): Promise { + val writePath = "$indexName/_delete_by_query" + return connector.request(RequestMethod.POST, writePath, body = data) + } + + // Make search request. + fun search(requestJson: String, filterPathes: List = emptyList()): Promise { + val path = "$indexName/_search?pretty${if (filterPathes.isNotEmpty()) + "&filter_path=" + filterPathes.joinToString(",") else ""}" + return connector.request(RequestMethod.POST, path, body = requestJson) + } + + init { + // Get latest id for index. + val queryBody = """{ + "_source": ["_id"], + "size": 1, + "query": { + "match_all": {} + } + }""" + search(queryBody, listOf("hits.total.value")).then { responseString -> + val response = JsonTreeParser.parse(responseString).jsonObject + val value = response.getObjectOrNull("hits")?.getObjectOrNull("total")?.getPrimitiveOrNull("value")?.content + ?: error("Error response from ElasticSearch:\n$responseString") + nextId = value.toLong() + }.catch { errorMessage -> + error(errorMessage.message ?: "Failed getting next id for index $indexName") + } + } + + abstract val mapping: Map + + val mappingDescription: String + get() = """ + { + "mappings": { + "properties": { + ${mapping.map { (property, type) -> + "\"${property}\": { \"type\": \"${type.name.toLowerCase()}\"${if (type == ElasticSearchType.DATE) "," + + "\"format\": \"basic_date_time_no_millis\"" else ""} }" + }.joinToString()}} + } + } + """.trimIndent() + + fun createMapping() = + connector.request(RequestMethod.PUT, indexName, body = mappingDescription) +} + +class BenchmarksIndex(name: String, connector: ElasticSearchConnector) : ElasticSearchIndex(name, connector) { + override val mapping: Map + get() = mapOf("buildNumber" to ElasticSearchType.KEYWORD, + "benchmarks" to ElasticSearchType.NESTED, + "env" to ElasticSearchType.NESTED, + "kotlin" to ElasticSearchType.NESTED) +} + +class GoldenResultsIndex(connector: ElasticSearchConnector) : ElasticSearchIndex("golden", connector) { + override val mapping: Map + get() = mapOf("buildNumber" to ElasticSearchType.KEYWORD, + "benchmarks" to ElasticSearchType.NESTED, + "env" to ElasticSearchType.NESTED, + "kotlin" to ElasticSearchType.NESTED) +} + +class BuildInfoIndex(connector: ElasticSearchConnector) : ElasticSearchIndex("builds", connector) { + override val mapping: Map + get() = mapOf("buildNumber" to ElasticSearchType.KEYWORD, + "startTime" to ElasticSearchType.DATE, + "endTime" to ElasticSearchType.DATE, + "commits" to ElasticSearchType.NESTED) +} + +// Processed benchmark result with calculated mean, variance and normalized reult. +class NormalizedMeanVarianceBenchmark(name: String, status: BenchmarkResult.Status, score: Double, metric: BenchmarkResult.Metric, + runtimeInUs: Double, repeat: Int, warmup: Int, variance: Double, val normalizedScore: Double) : + MeanVarianceBenchmark(name, status, score, metric, runtimeInUs, repeat, warmup, variance) { + + override fun serializeFields(): String { + return """ + ${super.serializeFields()}, + "normalizedScore": $normalizedScore + """ + } +} \ No newline at end of file diff --git a/tools/performance-server/shared/src/main/kotlin/org/jetbrains/network/NetworkConnector.kt b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/network/NetworkConnector.kt new file mode 100644 index 00000000000..7865a059df4 --- /dev/null +++ b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/network/NetworkConnector.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.network + +import kotlin.js.Promise // TODO - migrate to multiplatform. +import kotlin.js.json // TODO - migrate to multiplatform. + +// Now implemenation for network connection only for Node.js. TODO - multiplatform. +external fun require(module: String): dynamic + +enum class RequestMethod { + POST, GET, PUT +} + +// Abstract class for working with network. +abstract class NetworkConnector { + fun getAuth(user: String, password: String): String { + val buffer = js("Buffer").from(user + ":" + password) + val based64String = buffer.toString("base64") + return "Basic " + based64String + } + + protected abstract fun sendBaseRequest(method: RequestMethod, path: String, user: String? = null, + password: String? = null, acceptJsonContentType: Boolean = true, + body: String? = null, + errorHandler: (url: String, response: dynamic) -> Nothing?): Promise + + open fun sendRequest(method: RequestMethod, path: String, user: String? = null, password: String? = null, + acceptJsonContentType: Boolean = true, body: String? = null): Promise = + sendBaseRequest(method, path, user, password, acceptJsonContentType, body) { url, response -> + error("Error during getting response from $url\n$response") + } + + open fun sendOptionalRequest(method: RequestMethod, path: String, user: String? = null, password: String? = null, + acceptJsonContentType: Boolean = true, body: String? = null): Promise = + sendBaseRequest(method, path, user, password, acceptJsonContentType, body) { url, response -> + println("Error during getting response from $url\n$response") + null + } +} + + + diff --git a/tools/performance-server/shared/src/main/kotlin/org/jetbrains/network/UrlNetworkConnector.kt b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/network/UrlNetworkConnector.kt new file mode 100644 index 00000000000..f594570f5ad --- /dev/null +++ b/tools/performance-server/shared/src/main/kotlin/org/jetbrains/network/UrlNetworkConnector.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.network + +import kotlin.js.Promise // TODO - migrate to multiplatform. +import kotlin.js.json // TODO - migrate to multiplatform. + +// Network connector to work with basic url requests. +class UrlNetworkConnector(private val host: String, private val port: Int? = null) : NetworkConnector() { + + private val url = "$host${port?.let { ":$port" } ?: ""}" + + override fun sendBaseRequest(method: RequestMethod, path: String, user: String?, password: String?, + acceptJsonContentType: Boolean, body: String?, + errorHandler: (url: String, response: dynamic) -> Nothing?): Promise { + val fullUrl = "$url/$path" + val request = require("node-fetch") + val headers = mutableListOf>() + if (user != null && password != null) { + headers.add("Authorization" to getAuth(user, password)) + } + if (acceptJsonContentType) { + headers.add("Accept" to "application/json") + headers.add("Content-Type" to "application/json") + } + + return request(fullUrl, + json( + "method" to method.toString(), + "headers" to json(*(headers.toTypedArray())), + "body" to body + ) + ).then { response -> + if (!response.ok) { + errorHandler(fullUrl, response) + } else { + response.text() + } + } + } +} + + + diff --git a/tools/performance-server/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt b/tools/performance-server/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt new file mode 100644 index 00000000000..5494cd550b0 --- /dev/null +++ b/tools/performance-server/src/main/kotlin-js/org/jetbrains/analyzer/Utils.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.analyzer + +import org.w3c.xhr.* +import kotlin.browser.* +import kotlin.js.* + +actual fun readFile(fileName: String): String { + error("Reading from local file for JS isn't supported") +} + +actual fun writeToFile(fileName: String, text: String) { + error("Writing to local file for JS isn't supported") +} + +actual fun Double.format(decimalNumber: Int): String = + this.asDynamic().toFixed(decimalNumber) + +actual fun assert(value: Boolean, lazyMessage: () -> Any) { + if (!value) error(lazyMessage) +} + +actual fun sendGetRequest(url: String, user: String?, password: String?, followLocation: Boolean) : String { + error("Unsupported") +} \ No newline at end of file diff --git a/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt b/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt new file mode 100644 index 00000000000..e5db2561e50 --- /dev/null +++ b/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt @@ -0,0 +1,365 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.database + +import kotlin.js.Promise +import org.jetbrains.elastic.* +import org.jetbrains.utils.* +import org.jetbrains.report.json.* +import org.jetbrains.report.* + +fun Iterable.isEmpty() = count() == 0 +fun Iterable.isNotEmpty() = !isEmpty() + +inline fun T?.str(block: (T) -> String): String = + if (this != null) block(this) + else "" + +// Dispatcher to create and control benchmarks indexes separated by some feature. +// Feature can be choosen as often used as filtering entity in case there is no need in separate indexes. +// Default behaviour of dispatcher is working with one index (case when separating isn't needed). +class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature: String, + featureValues: Iterable = emptyList()) { + // Becnhmarks indexes to work with in case of existing feature values. + private val benchmarksIndexes = + if (featureValues.isNotEmpty()) + featureValues.map { it to BenchmarksIndex("benchmarks_${it.replace(" ", "_").toLowerCase()}", connector) } + .toMap() + else emptyMap() + + // Single benchmark index. + private val benchmarksSingleInstance = + if (featureValues.isEmpty()) BenchmarksIndex("benchmarks", connector) else null + + // Get right index in ES. + private fun getIndex(featureValue: String = "") = + benchmarksSingleInstance ?: benchmarksIndexes[featureValue] + ?: error("Used wrong feature value $featureValue. Indexes are separated using next values: ${benchmarksIndexes.keys}") + + // Used filter to get data with needed feature value. + var featureFilter: ((String) -> String)? = null + + // Get benchmark reports corresponding to needed build number. + fun getBenchmarksReports(buildNumber: String, featureValue: String): Promise> { + val queryDescription = """ + { + "size": 1000, + "query": { + "bool": { + "must": [ + { "match": { "buildNumber": "$buildNumber" } } + ] + } + } + } + """ + + return getIndex(featureValue).search(queryDescription, listOf("hits.hits._source")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits")?.let { results -> + results.map { + val element = it as JsonObject + element.getObject("_source").toString() + } + } ?: emptyList() + } + } + + // Get benchmarkes names corresponding to needed build number. + fun getBenchmarksList(buildNumber: String, featureValue: String): Promise> { + return getBenchmarksReports(buildNumber, featureValue).then { reports -> + reports.map { + val dbResponse = JsonTreeParser.parse(it).jsonObject + parseBenchmarksArray(dbResponse.getArray("benchmarks")) + .map { it.name } + }.flatten() + } + } + + // Delete benchmarks from database. + fun deleteBenchmarks(featureValue: String, buildNumber: String? = null): Promise { + // Delete all or for choosen build number. + val matchQuery = buildNumber?.let { + """"match": { "buildNumber": "$it" }""" + } ?: """"match_all": {}""" + + val queryDescription = """ + { + "query": { + $matchQuery + } + } + """.trimIndent() + return getIndex(featureValue).delete(queryDescription) + } + + // Get benchmarks values of needed metric for choosen build number. + fun getSamples(metricName: String, featureValue: String = "", samples: List, + buildNumbers: Iterable? = null, + normalize: Boolean = false): Promise>>> { + val queryDescription = """ + { + "_source": ["buildNumber"], + "size": 1000, + "query": { + "bool": { + "must": [ + ${buildNumbers.str { builds -> + """ + { "terms" : { "buildNumber" : [${builds.map { "\"$it\"" }.joinToString()}] } },""" } + } + ${featureFilter.str { "${it(featureValue)}," } } + {"nested" : { + "path" : "benchmarks", + "query" : { + "bool": { + "must": [ + { "match": { "benchmarks.metric": "$metricName" } }, + { "terms": { "benchmarks.name": [${samples.map { "\"${it.toLowerCase()}\"" }.joinToString()}] }} + ] + } + }, "inner_hits": { + "size": ${samples.size}, + "_source": ["benchmarks.name", + "benchmarks.${if (normalize) "normalizedScore" else "score"}"] + } + } + } + ] + } + } + }""" + + return getIndex(featureValue).search(queryDescription, listOf("hits.hits._source", "hits.hits.inner_hits")) + .then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + val results = dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits") + ?: error("Wrong response:\n$responseString") + // Get indexes for provided samples. + val indexesMap = samples.mapIndexed { index, it -> it to index }.toMap() + val valuesMap = buildNumbers?.map { + it to arrayOfNulls(samples.size) + }?.toMap()?.toMutableMap() ?: mutableMapOf>() + // Parse and save values in requested order. + results.forEach { + val element = it as JsonObject + val build = element.getObject("_source").getPrimitive("buildNumber").content + buildNumbers?.let { valuesMap.getOrPut(build) { arrayOfNulls(samples.size) } } + element + .getObject("inner_hits") + .getObject("benchmarks") + .getObject("hits") + .getArray("hits").forEach { + val source = (it as JsonObject).getObject("_source") + valuesMap[build]!![indexesMap[source.getPrimitive("name").content]!!] = + source.getPrimitive(if (normalize) "normalizedScore" else "score").double + } + + } + valuesMap.toList() + } + } + + fun insert(data: JsonSerializable, featureValue: String = "") = + getIndex(featureValue).insert(data) + + fun delete(data: String, featureValue: String = "") = + getIndex(featureValue).delete(data) + + // Get failures number happned during build. + fun getFailuresNumber(featureValue: String = "", buildNumbers: Iterable? = null): Promise> { + val queryDescription = """ + { + "_source": false, + ${featureFilter.str { + """ + "query": { + "bool": { + "must": [ ${it(featureValue)} ] + } + }, """ + } } + ${buildNumbers.str { builds -> + """ + "aggs" : { + "builds": { + "filters" : { + "filters": { + ${builds.map { "\"$it\": { \"match\" : { \"buildNumber\" : \"$it\" }}" } + .joinToString(",\n")} + } + },""" + } } + "aggs" : { + "metric_build" : { + "nested" : { + "path" : "benchmarks" + }, + "aggs" : { + "metric_samples": { + "filters" : { + "filters": { "samples": { "match": { "benchmarks.status": "FAILED" } } } + }, + "aggs" : { + "failed_count": { + "value_count": { + "field" : "benchmarks.score" + } + } + } + } + } + ${buildNumbers.str { + """ } + }""" + } } + } + } +} +""" + return getIndex(featureValue).search(queryDescription, listOf("aggregations")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + val aggregations = dbResponse.getObjectOrNull("aggregations") ?: error("Wrong response:\n$responseString") + buildNumbers?.let { + // Get failed number for each provided build. + val buckets = aggregations + .getObjectOrNull("builds") + ?.getObjectOrNull("buckets") + ?: error("Wrong response:\n$responseString") + buildNumbers.map { + it to buckets + .getObject(it) + .getObject("metric_build") + .getObject("metric_samples") + .getObject("buckets") + .getObject("samples") + .getObject("failed_count") + .getPrimitive("value") + .int + }.toMap() + } ?: listOf("golden" to aggregations + .getObject("metric_build") + .getObject("metric_samples") + .getObject("buckets") + .getObject("samples") + .getObject("failed_count") + .getPrimitive("value") + .int + ).toMap() + } + } + + // Get geometric mean for benchmarks values of needed metric. + fun getGeometricMean(metricName: String, featureValue: String = "", + buildNumbers: Iterable? = null, normalize: Boolean = false, + excludeNames: List = emptyList()): Promise>>> { + // Filter only with metric or also with names. + val filterBenchmarks = if (excludeNames.isEmpty()) + """ + "match": { "benchmarks.metric": "$metricName" } + """ + else """ + "bool": { + "must": { "match": { "benchmarks.metric": "$metricName" } }, + "must_not": { "terms" : { "benchmarks.name" : [${excludeNames.map { "\"$it\"" }.joinToString()}] } } + } + """.trimIndent() + val queryDescription = """ + { + "_source": false, + ${featureFilter.str { + """ + "query": { + "bool": { + "must": [ ${it(featureValue)} ] + } + }, """ + } } + ${buildNumbers.str { builds -> + """ + "aggs" : { + "builds": { + "filters" : { + "filters": { + ${builds.map { "\"$it\": { \"match\" : { \"buildNumber\" : \"$it\" }}" } + .joinToString(",\n")} + } + },""" + } } + "aggs" : { + "metric_build" : { + "nested" : { + "path" : "benchmarks" + }, + "aggs" : { + "metric_samples": { + "filters" : { + "filters": { "samples": { $filterBenchmarks } } + }, + "aggs" : { + "sum_log_x": { + "sum": { + "field" : "benchmarks.${if (normalize) "normalizedScore" else "score"}", + "script" : { + "source": "if (_value == 0) { 0.0 } else { Math.log(_value) }" + } + } + }, + "geom_mean": { + "bucket_script": { + "buckets_path": { + "sum_log_x": "sum_log_x", + "x_cnt": "_count" + }, + "script": "Math.exp(params.sum_log_x/params.x_cnt)" + } + } + } + } + } + + ${buildNumbers.str { + """ } + }""" + } } + } + } + } + """ + + return getIndex(featureValue).search(queryDescription, listOf("aggregations")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + val aggregations = dbResponse.getObjectOrNull("aggregations") ?: error("Wrong response:\n$responseString") + buildNumbers?.let { + val buckets = aggregations + .getObjectOrNull("builds") + ?.getObjectOrNull("buckets") + ?: error("Wrong response:\n$responseString") + buildNumbers.map { + it to listOf(buckets + .getObject(it) + .getObject("metric_build") + .getObject("metric_samples") + .getObject("buckets") + .getObject("samples") + .getObject("geom_mean") + .getPrimitive("value") + .double + ) + } + } ?: listOf("golden" to listOf(aggregations + .getObject("metric_build") + .getObject("metric_samples") + .getObject("buckets") + .getObject("samples") + .getObject("geom_mean") + .getPrimitive("value") + .double + ) + ) + } + } +} \ No newline at end of file diff --git a/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt b/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt new file mode 100644 index 00000000000..04f5c97919c --- /dev/null +++ b/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt @@ -0,0 +1,133 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.database + +import kotlin.js.Promise +import org.jetbrains.elastic.* +import org.jetbrains.utils.* +import org.jetbrains.report.json.* +import org.jetbrains.report.* + +// Delete build information from ES index. +internal fun deleteBuildInfo(agentInfo: String, buildInfoIndex: ElasticSearchIndex, + buildNumber: String? = null): Promise { + val queryDescription = """ + { + "query": { + "bool": { + "must": [ + { "match": { "agentInfo": "$agentInfo" } } + ${buildNumber?.let { + """, + {"match": { "buildNumber": "$it" }} + """ + } ?: ""} + ] + } + } + } + """.trimIndent() + return buildInfoIndex.delete(queryDescription) +} + +// Get infromation about builds details from database. +internal fun getBuildsDescription(type: String?, branch: String?, agentInfo: String, buildInfoIndex: ElasticSearchIndex, + onlyNumbers: Boolean = false): Promise { + val queryDescription = """ + { "size": 10000, + ${if (onlyNumbers) """"_source": ["buildNumber"],""" else ""} + "sort": {"_id": "desc" }, + "query": { + "bool": { + "must": [ + { "match": { "agentInfo": "$agentInfo" } } + ${type?.let { + """, + { "regexp": { "buildNumber": { "value": "${if (it == "release") + ".*eap.*|.*release.*|.*rc.*" else ".*dev.*"}" } } + } + """ + } ?: ""} + ${branch?.let { + """, + {"match": { "branch": "$it" }} + """ + } ?: ""} + ] + } + } + } + """.trimIndent() + return buildInfoIndex.search(queryDescription, listOf("hits.hits._source")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits") ?: error("Wrong response:\n$responseString") + } +} + +// Check if current build already exists. +suspend fun buildExists(buildInfo: BuildInfo, buildInfoIndex: ElasticSearchIndex): Boolean { + val queryDescription = """ + { "size": 1, + "_source": ["buildNumber"], + "query": { + "bool": { + "must": [ + { "match": { "buildNumber": "${buildInfo.buildNumber}" } }, + { "match": { "agentInfo": "${buildInfo.agentInfo}" } }, + { "match": { "branch": "${buildInfo.branch}" } } + ] + } + } + } + """.trimIndent() + + return buildInfoIndex.search(queryDescription, listOf("hits.total.value")).then { responseString -> + val response = JsonTreeParser.parse(responseString).jsonObject + val value = response.getObjectOrNull("hits")?.getObjectOrNull("total")?.getPrimitiveOrNull("value")?.content + ?: error("Error response from ElasticSearch:\n$responseString") + value.toInt() > 0 + }.await() +} + +// Get builds numbers corresponding to machine and branch. +fun getBuildsNumbers(type: String?, branch: String?, agentInfo: String, buildInfoIndex: ElasticSearchIndex) = + getBuildsDescription(type, branch, agentInfo, buildInfoIndex, true).then { responseArray -> + responseArray.map { (it as JsonObject).getObject("_source").getPrimitive("buildNumber").content } + } + +// Get full builds information corresponding to machine and branch. +fun getBuildsInfo(type: String?, branch: String?, agentInfo: String, buildInfoIndex: ElasticSearchIndex) = + getBuildsDescription(type, branch, agentInfo, buildInfoIndex).then { responseArray -> + responseArray.map { BuildInfo.create((it as JsonObject).getObject("_source")) } + } + +// Get golden results from database. +fun getGoldenResults(goldenResultsIndex: GoldenResultsIndex): Promise>> { + return goldenResultsIndex.search("", listOf("hits.hits._source")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits")?.map { + val reportDescription = (it as JsonObject).getObject("_source") + BenchmarksReport.create(reportDescription).benchmarks + }?.reduce { acc, it -> acc + it } ?: error("Wrong format of response:\n $responseString") + } +} + +// Get distinct values for needed field from database. +fun distinctValues(field: String, index: ElasticSearchIndex): Promise> { + val queryDescription = """ + { + "aggs": { + "unique": {"terms": {"field": "$field", "size": 1000}} + } + } + """.trimIndent() + return index.search(queryDescription, listOf("aggregations.unique.buckets")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + dbResponse.getObjectOrNull("aggregations")?.getObjectOrNull("unique")?.getArrayOrNull("buckets") + ?.map { (it as JsonObject).getPrimitiveOrNull("key")?.content }?.filterNotNull() + ?: error("Wrong response:\n$responseString") + } +} \ No newline at end of file diff --git a/tools/performance-server/src/main/kotlin/network/CachableResponseDispatcher.kt b/tools/performance-server/src/main/kotlin/network/CachableResponseDispatcher.kt new file mode 100644 index 00000000000..ee86444bfe2 --- /dev/null +++ b/tools/performance-server/src/main/kotlin/network/CachableResponseDispatcher.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +@file:OptIn(ExperimentalTime::class) + +package org.jetbrains.network + +import kotlin.js.Promise // TODO - migrate to multiplatform. +import kotlin.time.* + +// Response saved in cache. +data class CachedResponse(val cachedResult: Any, val time: TimeMark) + +// Dispatcher for work with cachable responses. +object CachableResponseDispatcher { + // Storage of cached responses. + private val cachedResponses = mutableMapOf() + + // Get response. If response isn't cached, use provided action to get response. + fun getResponse(request: dynamic, response: dynamic, + action: (success: (result: Any) -> Unit, reject: () -> Unit) -> Unit) { + cachedResponses[request.url]?.let { + // Update cache value if needed. Update only if last result was get later than 2 minutes. + if (it.time.elapsedNow().inMinutes > 2.0) { + println("Cache update for ${request.url}...") + action({ result: Any -> + cachedResponses[request.url] = CachedResponse(result, TimeSource.Monotonic.markNow()) + }, { println("Cache update for ${request.url} failed!") }) + } + response.json(it.cachedResult) + } ?: run { + action({ result: Any -> + cachedResponses[request.url] = CachedResponse(result, TimeSource.Monotonic.markNow()) + response.json(result) + }, { response.sendStatus(400) }) + } + } + + fun clear(): Unit { + cachedResponses.clear() + } +} \ No newline at end of file diff --git a/tools/performance-server/src/main/kotlin/network/aws/AWSNetworkUtils.kt b/tools/performance-server/src/main/kotlin/network/aws/AWSNetworkUtils.kt new file mode 100644 index 00000000000..2e4ad8398a6 --- /dev/null +++ b/tools/performance-server/src/main/kotlin/network/aws/AWSNetworkUtils.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.network + +import kotlin.js.Promise // TODO - migrate to multiplatform. +import kotlin.js.json // TODO - migrate to multiplatform. +import kotlin.js.Date +import org.jetbrains.report.json.* + +// Placeholder as analog for indexable type in TS. +external interface `T$0` { + @nativeGetter + operator fun get(key: String): String? + + @nativeSetter + operator fun set(key: String, value: String) +} + +@JsModule("aws-sdk") +@JsNonModule +external object AWSInstance { + + // Replace dynamic with some real type + class Endpoint(domain: String) + open class HttpRequest(endpoint: Endpoint, region: String) { + open fun pathname(): String + open var search: String + open var body: String? + open var endpoint: Endpoint + open var headers: `T$0` + open var method: String + open var path: String + } + + class HttpClient() { + val handleRequest: dynamic + } + + class SharedIniFileCredentials(options: Map) { + val accessKeyId: String + } + + class Signers() { + class V4(request: HttpRequest, subsystem: String) { + fun addAuthorization(credentials: SharedIniFileCredentials, date: Date) + } + } +} + +// Network connector to work with AWS resources. +class AWSNetworkConnector : NetworkConnector() { + val AWSDomain = "vpc-kotlin-perf-service-5e6ldakkdv526ii5hbclzcmpny.eu-west-1.es.amazonaws.com" + val AWSRegion = "eu-west-1" + + override fun sendBaseRequest(method: RequestMethod, path: String, user: String?, password: String?, + acceptJsonContentType: Boolean, body: String?, + errorHandler: (url: String, response: dynamic) -> Nothing?): Promise { + val AWSEndpoint = AWSInstance.Endpoint(AWSDomain) + var request = AWSInstance.HttpRequest(AWSEndpoint, AWSRegion) + request.method = method.toString() + request.path += path + request.body = body + + request.headers["host"] = this.AWSDomain + + if (acceptJsonContentType) { + request.headers["Content-Type"] = "application/json" + request.headers["Content-Length"] = js("Buffer").byteLength(request.body) + } + + val credentials = AWSInstance.SharedIniFileCredentials(mapOf()) + val signer = AWSInstance.Signers.V4(request, "es") + signer.addAuthorization(credentials, Date()) + + val client = AWSInstance.HttpClient() + return Promise { resolve, reject -> + client.handleRequest(request, null, { response -> + var responseBody = "" + response.on("data") { chunk -> + responseBody += chunk + chunk + } + response.on("end") { _ -> + val dbResponse = JsonTreeParser.parse(responseBody).jsonObject + // Response can fail and return 400 error for ES. + if (dbResponse.getPrimitiveOrNull("status")?.let { it.content != "200" } ?: false) { + println(dbResponse) + val errorMessage = dbResponse.getObject("error").toString() + reject(Throwable(errorMessage)) + } + resolve(responseBody as T) + } + }, { error -> + reject(error) + }) + } + } +} \ No newline at end of file diff --git a/tools/performance-server/src/main/kotlin/routes/route.kt b/tools/performance-server/src/main/kotlin/routes/route.kt index 17480d5fde9..5134e238d5e 100644 --- a/tools/performance-server/src/main/kotlin/routes/route.kt +++ b/tools/performance-server/src/main/kotlin/routes/route.kt @@ -1,330 +1,551 @@ /* - * Copyright 2010-2019 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ import org.w3c.xhr.* import kotlin.js.json import kotlin.js.Date +import kotlin.js.Promise +import org.jetbrains.database.* import org.jetbrains.report.json.* -import org.jetbrains.build.Build +import org.jetbrains.elastic.* +import org.jetbrains.network.* +import org.jetbrains.buildInfo.Build +import org.jetbrains.analyzer.* +import org.jetbrains.report.* +import org.jetbrains.utils.* + +// TODO - create DSL for ES requests? const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest" const val artifactoryUrl = "https://repo.labs.intellij.net/kotlin-native-benchmarks" -const val buildsFileName = "buildsSummary.csv" -const val goldenResultsFileName = "goldenResults.csv" -const val artifactoryBuildsDirectory = "builds" -const val buildsInfoPartsNumber = 11 operator fun Map?.get(key: K) = this?.get(key) fun getArtifactoryHeader(artifactoryApiKey: String) = Pair("X-JFrog-Art-Api", artifactoryApiKey) -// Local cache for saving information about builds got from Artifactory. -object LocalCache { - private val knownTargets = listOf("Linux", "MacOSX", "Windows10") - private val buildsInfo = mutableMapOf>() - - fun clean(onlyTarget: String? = null) { - onlyTarget?.let { - buildsInfo[onlyTarget]?.clear() - } ?: buildsInfo.clear() +// Convert saved old report to expected new format. +internal fun convertToNewFormat(data: JsonObject): List { + val env = Environment.create(data.getRequiredField("env")) + val benchmarksObj = data.getRequiredField("benchmarks") + val compilerDescription = data.getRequiredField("kotlin") + val compiler = Compiler.create(compilerDescription) + val backend = (compilerDescription as JsonObject).getRequiredField("backend") + val flagsArray = (backend as JsonObject).getOptionalField("flags") + var flags: List = emptyList() + if (flagsArray != null && flagsArray is JsonArray) { + flags = flagsArray.jsonArray.map { (it as JsonLiteral).unquoted() } } + val benchmarksList = parseBenchmarksArray(benchmarksObj) - fun fill(onlyTarget: String? = null) { - onlyTarget?.let { - val buildsDescription = getBuildsInfoFromArtifactory(onlyTarget).lines().drop(1) - buildsInfo[onlyTarget] = mutableMapOf() - buildsDescription.forEach { - if (!it.isEmpty()) { - val buildNumber = it.substringBefore(',') - if (!"\\d+(\\.\\d+)+(-M\\d)?-\\w+-\\d+".toRegex().matches(buildNumber)) { - error("Build number $buildNumber differs from expected format. File with data for " + - "target $onlyTarget could be corrupted.") - } - buildsInfo[onlyTarget]!![buildNumber] = it - } - } - } ?: run { - knownTargets.forEach { - fill(it) - } - } - } - - fun buildExists(target: String, buildNumber: String) = - buildsInfo[target][buildNumber]?.let { true } ?: false - - fun delete(target: String, builds: Iterable, apiKey: String): Boolean { - // Delete from Artifactory. - val buildsDescription = getBuildsInfoFromArtifactory(target).lines() - - val newBuildsDescription = buildsDescription.filter { - val buildNumber = it.substringBefore(',') - buildNumber !in builds - } - if (newBuildsDescription.size < buildsDescription.size) { - // Upload new version of file. - val uploadUrl = "$artifactoryUrl/$artifactoryBuildsDirectory/$target/$buildsFileName" - sendUploadRequest(uploadUrl, newBuildsDescription.joinToString("\n"), - extraHeaders = listOf(getArtifactoryHeader(apiKey))) - - // Reload values. - clean(target) - fill(target) - return true - } - return false - } - - private fun getBuilds(target: String, buildNumber: String? = null) = - buildsInfo[target]?.let { buildsList -> - buildNumber?.let { - // Check if interesting build id is in cache. - buildsList[it]?.let { buildsList.values } - } ?: buildsList.values - } - - operator fun get(target: String, buildId: String? = null): Collection { - val builds = getBuilds(target, buildId) - - if (builds.isNullOrEmpty()) { - // No suitable builds were found. - // Refill cache. - clean(target) - fill(target) - return getBuilds(target, buildId) ?: listOf() - } - - return builds - } + return listOf(env, compiler, benchmarksList, flags) } +// Convert data results to expected format. +internal fun convert(json: String, buildNumber: String, target: String): List { + val data = JsonTreeParser.parse(json) + val reports = if (data is JsonArray) { + data.map { convertToNewFormat(it as JsonObject) } + } else listOf(convertToNewFormat(data as JsonObject)) + + // Restored flags for old reports. + val knownFlags = mapOf( + "Cinterop" to listOf("-opt"), + "FrameworkBenchmarksAnalyzer" to listOf("-g"), + "HelloWorld" to if (target == "Mac OS X") + listOf("-Xcache-directory=/Users/teamcity/buildAgent/work/c104dee5223a31c5/test_dist/klib/cache/macos_x64-gSTATIC", "-g") + else listOf("-g"), + "Numerical" to listOf("-opt"), + "ObjCInterop" to listOf("-opt"), + "Ring" to listOf("-opt"), + "Startup" to listOf("-opt"), + "swiftInterop" to listOf("-opt"), + "Videoplayer" to if (target == "Mac OS X") + listOf("-Xcache-directory=/Users/teamcity/buildAgent/work/c104dee5223a31c5/test_dist/klib/cache/macos_x64-gSTATIC", "-g") + else listOf("-g") + ) + + return reports.map { elements -> + val benchmarks = (elements[2] as List).groupBy { it.name.substringBefore('.').substringBefore(':') } + val parsedFlags = elements[3] as List + benchmarks.map { (setName, results) -> + val flags = if (parsedFlags.isNotEmpty() && parsedFlags[0] == "-opt") knownFlags[setName]!! else parsedFlags + val savedCompiler = elements[1] as Compiler + val compiler = Compiler(Compiler.Backend(savedCompiler.backend.type, savedCompiler.backend.version, flags), + savedCompiler.kotlinVersion) + val newReport = BenchmarksReport(elements[0] as Environment, results, compiler) + newReport.buildNumber = buildNumber + newReport + } + }.flatten() +} + +// Golden result value used to get normalized results. data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double) -data class GoldenResultsInfo(val apiKey: String, val goldenResults: Array) +data class GoldenResultsInfo(val goldenResults: Array) + +// Convert information about golden results to benchmarks report format. +fun GoldenResultsInfo.toBenchmarksReport(): BenchmarksReport { + val benchmarksSamples = goldenResults.map { + BenchmarkResult(it.benchmarkName, BenchmarkResult.Status.PASSED, + it.value, BenchmarkResult.metricFromString(it.metric)!!, it.value, 1, 0) + } + val compiler = Compiler(Compiler.Backend(Compiler.BackendType.NATIVE, "golden", emptyList()), "golden") + val environment = Environment(Environment.Machine("golden", "golden"), Environment.JDKInstance("golden", "golden")) + return BenchmarksReport(environment, + benchmarksSamples, compiler) +} // Build information provided from request. -data class BuildInfo(val buildNumber: String, val branch: String, val startTime: String, - val finishTime: String) +data class TCBuildInfo(val buildNumber: String, val branch: String, val startTime: String, + val finishTime: String) data class BuildRegister(val buildId: String, val teamCityUser: String, val teamCityPassword: String, - val artifactoryApiKey: String, val target: String, val buildType: String, val failuresNumber: Int, - val executionTime: String, val compileTime: String, val codeSize: String, - val bundleSize: String?) { + val bundleSize: String?, val fileWithResult: String) { companion object { fun create(json: String): BuildRegister { val requestDetails = JSON.parse(json) // Parse method doesn't create real instance with all methods. So create it by hands. return BuildRegister(requestDetails.buildId, requestDetails.teamCityUser, requestDetails.teamCityPassword, - requestDetails.artifactoryApiKey, requestDetails.target, requestDetails.buildType, - requestDetails.failuresNumber, requestDetails.executionTime, requestDetails.compileTime, - requestDetails.codeSize, requestDetails.bundleSize) + requestDetails.bundleSize, requestDetails.fileWithResult) } } - private val teamCityBuildUrl: String by lazy { "$teamCityUrl/builds/id:$buildId" } + private val teamCityBuildUrl: String by lazy { "builds/id:$buildId" } val changesListUrl: String by lazy { - "$teamCityUrl/changes/?locator=build:id:$buildId" + "changes/?locator=build:id:$buildId" } - private fun sendTeamCityRequest(url: String) = sendGetRequest(url, teamCityUser, teamCityPassword) + val teamCityArtifactsUrl: String by lazy { "builds/id:$buildId/artifacts/content/$fileWithResult" } + + fun sendTeamCityRequest(url: String, json: Boolean = false) = + UrlNetworkConnector(teamCityUrl).sendRequest(RequestMethod.GET, url, teamCityUser, teamCityPassword, json) private fun format(timeValue: Int): String = if (timeValue < 10) "0$timeValue" else "$timeValue" - fun getBuildInformation(): BuildInfo { - val buildNumber = sendTeamCityRequest("$teamCityBuildUrl/number") - val branch = sendTeamCityRequest("$teamCityBuildUrl/branchName") - val startTime = sendTeamCityRequest("$teamCityBuildUrl/startDate") - val currentTime = Date() - val timeZone = currentTime.getTimezoneOffset() / -60 // Convert to hours. - // Get finish time as current time, because buid on TeamCity isn't finished. - val finishTime = "${format(currentTime.getUTCFullYear())}" + - "${format(currentTime.getUTCMonth() + 1)}" + - "${format(currentTime.getUTCDate())}" + - "T${format(currentTime.getUTCHours())}" + - "${format(currentTime.getUTCMinutes())}" + - "${format(currentTime.getUTCSeconds())}" + - "${if (timeZone > 0) "+" else "-"}${format(timeZone)}${format(0)}" - return BuildInfo(buildNumber, branch, startTime, finishTime) - } -} - -data class Commit(val revision: String, val developer: String) - -// List of commits. -class CommitsList(data: JsonElement): ConvertedFromJson { - - val commits: List - - init { - if (data !is JsonObject) { - error("Commits description is expected to be a json object!") - } - val changesElement = data.getOptionalField("change") - commits = changesElement?.let { - if (changesElement !is JsonArray) { - error("Change field is expected to be an array. Please, check source.") - } - changesElement.jsonArray.map { - with(it as JsonObject) { - Commit(elementToString(getRequiredField("version"), "version"), - elementToString(getRequiredField("username"), "username") - ) - } - } - } ?: listOf() - } -} - -fun getBuildsInfoFromArtifactory(target: String) = - sendGetRequest("$artifactoryUrl/$artifactoryBuildsDirectory/$target/$buildsFileName") - -fun checkBuildType(currentType: String, targetType: String): Boolean { - val releasesBuildTypes = listOf("release", "eap", "rc1", "rc2") - return if (targetType == "release") currentType in releasesBuildTypes else currentType == targetType -} - -// Parse and postprocess result of response with build description. -fun prepareBuildsResponse(builds: Collection, type: String, branch: String, buildNumber: String? = null): List { - val buildsObjects = mutableListOf() - builds.forEach { - val tokens = buildDescriptionToTokens(it) - if ((checkBuildType(tokens[5], type) || type == "day") && (branch == tokens[3] || branch == "all") - || tokens[0] == buildNumber) { - buildsObjects.add(Build(tokens[0], tokens[1], tokens[2], tokens[3], - tokens[4], tokens[5], tokens[6].toInt(), tokens[7], tokens[8], tokens[9], - if (tokens[10] == "-") null else tokens[10])) + fun getBuildInformation(): Promise { + return Promise.all(arrayOf(sendTeamCityRequest("$teamCityBuildUrl/number"), + sendTeamCityRequest("$teamCityBuildUrl/branchName"), + sendTeamCityRequest("$teamCityBuildUrl/startDate"))).then { results -> + val (buildNumber, branch, startTime) = results + val currentTime = Date() + val timeZone = currentTime.getTimezoneOffset() / -60 // Convert to hours. + // Get finish time as current time, because buid on TeamCity isn't finished. + val finishTime = "${format(currentTime.getUTCFullYear())}" + + "${format(currentTime.getUTCMonth() + 1)}" + + "${format(currentTime.getUTCDate())}" + + "T${format(currentTime.getUTCHours())}" + + "${format(currentTime.getUTCMinutes())}" + + "${format(currentTime.getUTCSeconds())}" + + "${if (timeZone > 0) "+" else "-"}${format(timeZone)}${format(0)}" + TCBuildInfo(buildNumber, branch, startTime, finishTime) } } - return buildsObjects } -fun buildDescriptionToTokens(buildDescription: String): List { - val tokens = buildDescription.split(",").map { it.trim() } - if (tokens.size != buildsInfoPartsNumber) { - error("Build description $buildDescription doesn't contain all necessary information. " + - "File with data could be corrupted.") - } - return tokens -} +// Get builds numbers in right order. +internal fun orderedValues(values: List, buildElement: (T) -> String = { it -> it.toString() }, + skipMilestones: Boolean = false) = + values.sortedWith( + compareBy({ buildElement(it).substringBefore(".").toInt() }, + { buildElement(it).substringAfter(".").substringBefore("-").toDouble() }, + { + if (skipMilestones) 0 + else if (buildElement(it).substringAfter("-").startsWith("M")) + buildElement(it).substringAfter("M").substringBefore("-").toInt() + else + Int.MAX_VALUE + }, + { buildElement(it).substringAfterLast("-").toDouble() } + ) + ) + +// ElasticSearch connector for work with custom instance. +internal val localHostElasticConnector = UrlNetworkConnector("http://localhost", 9200) +// ElasticSearch connector for work with AWS instance. +internal val awsElasticConnector = AWSNetworkConnector() +internal val networkConnector = awsElasticConnector + +fun urlParameterToBaseFormat(value: dynamic) = + value.toString().replace("_", " ") // Routing of requests to current server. fun router() { val express = require("express") val router = express.Router() + val connector = ElasticSearchConnector(networkConnector) + val benchmarksDispatcher = BenchmarksIndexesDispatcher(connector, "env.machine.os", + listOf("Linux", "Mac OS X", "Windows 10") + ) + val goldenIndex = GoldenResultsIndex(connector) + val buildInfoIndex = BuildInfoIndex(connector) + + router.get("/createMapping") { request, response -> + buildInfoIndex.createMapping().then { _ -> + response.sendStatus(200) + }.catch { _ -> + response.sendStatus(400) + } + } + + // Get consistent build information in cases of rerunning the same build. + suspend fun getConsistentBuildInfo(buildInfoInstance: BuildInfo, reports: List, + rerunNumber: Int = 1): BuildInfo { + var currentBuildInfo = buildInfoInstance + if (buildExists(currentBuildInfo, buildInfoIndex)) { + // Check if benchmarks aren't repeated. + val existingBecnhmarks = benchmarksDispatcher.getBenchmarksList(currentBuildInfo.buildNumber, + currentBuildInfo.agentInfo).await() + val benchmarksToRegister = reports.map { it.benchmarks.keys }.flatten() + if (existingBecnhmarks.toTypedArray().intersect(benchmarksToRegister).isNotEmpty()) { + // Build was rerun. + val buildNumber = "${currentBuildInfo.buildNumber}.$rerunNumber" + currentBuildInfo = BuildInfo(buildNumber, currentBuildInfo.startTime, currentBuildInfo.endTime, + currentBuildInfo.commitsList, currentBuildInfo.branch, currentBuildInfo.agentInfo) + return getConsistentBuildInfo(currentBuildInfo, reports, rerunNumber + 1) + } + } + return currentBuildInfo + } // Register build on Artifactory. - router.post("/register", { request, response -> - val maxCommitsNumber = 5 + router.post("/register") { request, response -> val register = BuildRegister.create(JSON.stringify(request.body)) // Get information from TeamCity. - val buildInfo = register.getBuildInformation() - val changes = sendGetRequest(register.changesListUrl, register.teamCityUser, - register.teamCityPassword, true) - val commitsList = CommitsList(JsonTreeParser.parse(changes)) - val commitsDescription = buildString { - if (commitsList.commits.size > maxCommitsNumber) { - append("${commitsList.commits.get(0).revision} by ${commitsList.commits.get(0).developer};") - append("${commitsList.commits.get(1).revision} by ${commitsList.commits.get(1).developer};") - append("...;") - val beforeLast = commitsList.commits.lastIndex - 1 - append("${commitsList.commits.get(beforeLast).revision} by ${commitsList.commits.get(beforeLast).developer};") - append("${commitsList.commits.last().revision} by ${commitsList.commits.last().developer};") - } else { - commitsList.commits.forEach { - append("${it.revision} by ${it.developer};") + register.getBuildInformation().then { buildInfo -> + register.sendTeamCityRequest(register.changesListUrl, true).then { changes -> + val commitsList = CommitsList(JsonTreeParser.parse(changes)) + // Get artifact. + register.sendTeamCityRequest(register.teamCityArtifactsUrl).then { resultsContent -> + launch { + val reportData = JsonTreeParser.parse(resultsContent) + val reports = if (reportData is JsonArray) { + reportData.map { BenchmarksReport.create(it as JsonObject) } + } else listOf(BenchmarksReport.create(reportData as JsonObject)) + val goldenResultPromise = getGoldenResults(goldenIndex) + val goldenResults = goldenResultPromise.await() + // Register build information. + var buildInfoInstance = getConsistentBuildInfo( + BuildInfo(buildInfo.buildNumber, buildInfo.startTime, buildInfo.finishTime, + commitsList, buildInfo.branch, reports[0].env.machine.os), + reports + ) + if (register.bundleSize != null) { + // Add bundle size. + val bundleSizeBenchmark = BenchmarkResult("KotlinNative", + BenchmarkResult.Status.PASSED, register.bundleSize.toDouble(), + BenchmarkResult.Metric.BUNDLE_SIZE, 0.0, 1, 0) + val bundleSizeReport = BenchmarksReport(reports[0].env, + listOf(bundleSizeBenchmark), reports[0].compiler) + bundleSizeReport.buildNumber = buildInfoInstance.buildNumber + benchmarksDispatcher.insert(bundleSizeReport, reports[0].env.machine.os).then { _ -> + println("[BUNDLE] Success insert ${buildInfoInstance.buildNumber}") + }.catch { errorResponse -> + println("Failed to insert data for build") + println(errorResponse) + } + } + val insertResults = reports.map { + val benchmarksReport = SummaryBenchmarksReport(it).getBenchmarksReport() + .normalizeBenchmarksSet(goldenResults) + benchmarksReport.buildNumber = buildInfoInstance.buildNumber + // Save results in database. + benchmarksDispatcher.insert(benchmarksReport, benchmarksReport.env.machine.os) + } + if (!buildExists(buildInfoInstance, buildInfoIndex)) { + buildInfoIndex.insert(buildInfoInstance).then { _ -> + println("Success insert build information for ${buildInfoInstance.buildNumber}") + }.catch { + response.sendStatus(400) + } + } + Promise.all(insertResults.toTypedArray()).then { _ -> + response.sendStatus(200) + }.catch { + response.sendStatus(400) + } + } } } } - - // Get summary file from Artifactory. - var buildsDescription = getBuildsInfoFromArtifactory(register.target) - // Add information about new build. - //var buildsDescription = "build, start time, finish time, branch, commits, type, failuresNumber, execution time, compile time, code size, bundle size\n" - buildsDescription += "${buildInfo.buildNumber}, ${buildInfo.startTime}, ${buildInfo.finishTime}, " + - "${buildInfo.branch}, $commitsDescription, ${register.buildType}, ${register.failuresNumber}, " + - "${register.executionTime}, ${register.compileTime}, ${register.codeSize}, " + - "${register.bundleSize ?: "-"}\n" - - // Upload new version of file. - val uploadUrl = "$artifactoryUrl/$artifactoryBuildsDirectory/${register.target}/${buildsFileName}" - sendUploadRequest(uploadUrl, buildsDescription, extraHeaders = listOf(getArtifactoryHeader(register.artifactoryApiKey))) - - LocalCache.clean(register.target) - LocalCache.fill(register.target) - - // Send response. - response.sendStatus(200) - }) + } // Register golden results to normalize on Artifactory. router.post("/registerGolden", { request, response -> - val goldenResultsInfo = JSON.parse(JSON.stringify(request.body)) - val buildsDescription = StringBuilder(sendGetRequest("$artifactoryUrl/$artifactoryBuildsDirectory/$goldenResultsFileName")) - goldenResultsInfo.goldenResults.forEach { - buildsDescription.append("${it.benchmarkName}, ${it.metric}, ${it.value}\n") + val goldenResultsInfo: GoldenResultsInfo = JSON.parse(JSON.stringify(request.body)) + val goldenReport = goldenResultsInfo.toBenchmarksReport() + goldenIndex.insert(goldenReport).then { _ -> + response.sendStatus(200) + }.catch { + response.sendStatus(400) } - // Upload new version of file. - val uploadUrl = "$artifactoryUrl/$artifactoryBuildsDirectory/$goldenResultsFileName" - sendUploadRequest(uploadUrl, buildsDescription.toString(), - extraHeaders = listOf(getArtifactoryHeader(goldenResultsInfo.apiKey))) - // Send response. - response.sendStatus(200) }) - // Get list of builds. - router.get("/builds/:target/:type/:branch/:id", { request, response -> - val builds = LocalCache[request.params.target, request.params.id] - response.json(prepareBuildsResponse(builds, request.params.type, request.params.branch, request.params.id)) + // Get builds description with additional information. + router.get("/buildsDesc/:target", { request, response -> + CachableResponseDispatcher.getResponse(request, response) { success, reject -> + val target = request.params.target.toString().replace('_', ' ') + + var branch: String? = null + var type: String? = null + if (request.query != undefined) { + if (request.query.branch != undefined) { + branch = request.query.branch + } + if (request.query.type != undefined) { + type = request.query.type + } + } + + getBuildsInfo(type, branch, target, buildInfoIndex).then { buildsInfo -> + val buildNumbers = buildsInfo.map { it.buildNumber } + // Get number of failed benchmarks for each build. + benchmarksDispatcher.getFailuresNumber(target, buildNumbers).then { failures -> + success(orderedValues(buildsInfo, { it -> it.buildNumber }, branch == "master").map { + Build(it.buildNumber, it.startTime, it.endTime, it.branch, + it.commitsList.serializeFields(), failures[it.buildNumber] ?: 0) + }) + }.catch { errorResponse -> + println("Error during getting failures numbers") + println(errorResponse) + reject() + } + }.catch { + reject() + } + } }) - router.get("/builds/:target/:type/:branch", { request, response -> - val builds = LocalCache[request.params.target] - response.json(prepareBuildsResponse(builds, request.params.type, request.params.branch)) + // Get values of current metric. + router.get("/metricValue/:target/:metric", { request, response -> + CachableResponseDispatcher.getResponse(request, response) { success, reject -> + val metric = request.params.metric + val target = request.params.target.toString().replace('_', ' ') + var samples: List = emptyList() + var aggregation = "geomean" + var normalize = false + var branch: String? = null + var type: String? = null + var excludeNames: List = emptyList() + + // Parse parameters from request if it exists. + if (request.query != undefined) { + if (request.query.samples != undefined) { + samples = request.query.samples.toString().split(",").map { it.trim() } + } + if (request.query.agr != undefined) { + aggregation = request.query.agr.toString() + } + if (request.query.normalize != undefined) { + normalize = true + } + if (request.query.branch != undefined) { + branch = request.query.branch + } + if (request.query.type != undefined) { + type = request.query.type + } + if (request.query.exclude != undefined) { + excludeNames = request.query.exclude.toString().split(",").map { it.trim() } + } + } + + getBuildsNumbers(type, branch, target, buildInfoIndex).then { buildNumbers -> + if (aggregation == "geomean") { + // Get geometric mean for samples. + benchmarksDispatcher.getGeometricMean(metric, target, buildNumbers, normalize, + excludeNames).then { geoMeansValues -> + success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master")) + }.catch { errorResponse -> + println("Error during getting geometric mean") + println(errorResponse) + reject() + } + } else { + benchmarksDispatcher.getSamples(metric, target, samples, buildNumbers, normalize).then { geoMeansValues -> + success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master")) + }.catch { + println("Error during getting samples") + reject() + } + } + }.catch { + println("Error during getting builds information") + reject() + } + } }) - router.get("/branches/:target", { request, response -> - val builds = LocalCache[request.params.target] - response.json(builds.map { buildDescriptionToTokens(it)[3] }.distinct()) + // Get branches for [target]. + router.get("/branches", { request, response -> + CachableResponseDispatcher.getResponse(request, response) { success, reject -> + distinctValues("branch", buildInfoIndex).then { results -> + success(results) + }.catch { errorMessage -> + error(errorMessage.message ?: "Failed getting branches list.") + reject() + } + } }) + // Get build numbers for [target]. router.get("/buildsNumbers/:target", { request, response -> - val builds = LocalCache[request.params.target] - response.json(builds.map { buildDescriptionToTokens(it)[0] }.distinct()) + CachableResponseDispatcher.getResponse(request, response) { success, reject -> + distinctValues("buildNumber", buildInfoIndex).then { results -> + success(results) + }.catch { errorMessage -> + println(errorMessage.message ?: "Failed getting branches list.") + reject() + } + } }) - router.get("/clean", { _, response -> - LocalCache.clean() - response.sendStatus(200) - }) + // Conert data and migrate it from Artifactory to DB. + router.get("/migrate/:target", { request, response -> + val target = urlParameterToBaseFormat(request.params.target) + val targetPathName = target.replace(" ", "") + var buildNumber: String? = null + if (request.query != undefined) { + if (request.query.buildNumber != undefined) { + buildNumber = request.query.buildNumber + } + } + getBuildsInfoFromArtifactory(targetPathName).then { buildInfo -> + launch { + val buildsDescription = buildInfo.lines().drop(1) + var shouldConvert = buildNumber?.let { false } ?: true + val goldenResultPromise = getGoldenResults(goldenIndex) + val goldenResults = goldenResultPromise.await() + val buildsSet = mutableSetOf() + buildsDescription.forEach { + if (!it.isEmpty()) { + val currentBuildNumber = it.substringBefore(',') + if (!"\\d+(\\.\\d+)+(-M\\d)?-\\w+-\\d+(\\.\\d+)?".toRegex().matches(currentBuildNumber)) { + error("Build number $currentBuildNumber differs from expected format. File with data for " + + "target $target could be corrupted.") + } + if (!shouldConvert && buildNumber != null && buildNumber == currentBuildNumber) { + shouldConvert = true + } + if (shouldConvert) { + // Save data from Artifactory into database. + val artifactoryUrlConnector = UrlNetworkConnector(artifactoryUrl) + val fileName = "nativeReport.json" + val accessFileUrl = "$targetPathName/$currentBuildNumber/$fileName" + val extrenalFileName = if (target == "Linux") "externalReport.json" else "spaceFrameworkReport.json" + val accessExternalFileUrl = "$targetPathName/$currentBuildNumber/$extrenalFileName" + val infoParts = it.split(", ") + if ((infoParts[3] == "master" || "eap" in currentBuildNumber || "release" in currentBuildNumber) && + currentBuildNumber !in buildsSet) { + try { + buildsSet.add(currentBuildNumber) + val jsonReport = artifactoryUrlConnector.sendRequest(RequestMethod.GET, accessFileUrl).await() + var reports = convert(jsonReport, currentBuildNumber, target) + val buildInfoRecord = BuildInfo(currentBuildNumber, infoParts[1], infoParts[2], + CommitsList.parse(infoParts[4]), infoParts[3], target) - router.get("/fill", { _, response -> - LocalCache.fill() - response.sendStatus(200) + val externalJsonReport = artifactoryUrlConnector.sendOptionalRequest(RequestMethod.GET, accessExternalFileUrl).await() + buildInfoIndex.insert(buildInfoRecord).then { _ -> + externalJsonReport?.let { + var externalReports = convert(externalJsonReport, currentBuildNumber, target) + externalReports.forEach { externalReport -> + val extrenalAdditionalReport = SummaryBenchmarksReport(externalReport) + .getBenchmarksReport().normalizeBenchmarksSet(goldenResults) + extrenalAdditionalReport.buildNumber = currentBuildNumber + benchmarksDispatcher.insert(extrenalAdditionalReport, target).then { _ -> + println("[External] Success insert ${buildInfoRecord.buildNumber}") + }.catch { errorResponse -> + println("Failed to insert data for build") + println(errorResponse) + } + } + } + + val bundleSize = if (infoParts[10] != "-") infoParts[10] else null + if (bundleSize != null) { + // Add bundle size. + val bundleSizeBenchmark = BenchmarkResult("KotlinNative", + BenchmarkResult.Status.PASSED, bundleSize.toDouble(), + BenchmarkResult.Metric.BUNDLE_SIZE, 0.0, 1, 0) + val bundleSizeReport = BenchmarksReport(reports[0].env, + listOf(bundleSizeBenchmark), reports[0].compiler) + bundleSizeReport.buildNumber = currentBuildNumber + benchmarksDispatcher.insert(bundleSizeReport, target).then { _ -> + println("[BUNDLE] Success insert ${buildInfoRecord.buildNumber}") + }.catch { errorResponse -> + println("Failed to insert data for build") + println(errorResponse) + } + } + + reports.forEach { report -> + val summaryReport = SummaryBenchmarksReport(report).getBenchmarksReport() + .normalizeBenchmarksSet(goldenResults) + summaryReport.buildNumber = currentBuildNumber + // Save results in database. + benchmarksDispatcher.insert(summaryReport, target).then { _ -> + println("Success insert ${buildInfoRecord.buildNumber}") + }.catch { errorResponse -> + println("Failed to insert data for build") + println(errorResponse.message) + } + } + + + }.catch { errorResponse -> + println("Failed to insert data for build") + println(errorResponse) + } + } catch (e: Exception) { + println(e) + } + } + } + } + } + } + response.sendStatus(200) + }.catch { + response.sendStatus(400) + } }) router.get("/delete/:target", { request, response -> - val buildsToDelete: List = request.query.builds.toString().split(",").map { it.trim() } - val result = LocalCache.delete(request.params.target, buildsToDelete, request.query.key) - if (result) { - response.sendStatus(200) - } else { - response.sendStatus(404) + val target = urlParameterToBaseFormat(request.params.target) + var buildNumber: String? = null + if (request.query != undefined) { + if (request.query.buildNumber != undefined) { + buildNumber = request.query.buildNumber + } } + benchmarksDispatcher.deleteBenchmarks(target, buildNumber).then { + deleteBuildInfo(target, buildInfoIndex, buildNumber).then { + response.sendStatus(200) + }.catch { + response.sendStatus(400) + } + }.catch { + response.sendStatus(400) + } + }) + + router.get("/report/:target/:buildNumber", { request, response -> + val target = urlParameterToBaseFormat(request.params.target) + val buildNumber = request.params.buildNumber.toString() + benchmarksDispatcher.getBenchmarksReports(buildNumber, target).then { reports -> + response.send(reports.joinToString(", ", "[", "]")) + }.catch { + response.sendStatus(400) + } + }) + + router.get("/clear", { _, response -> + CachableResponseDispatcher.clear() + response.sendStatus(200) }) // Main page. @@ -335,52 +556,20 @@ fun router() { return router } -fun getAuth(user: String, password: String): String { - val buffer = js("Buffer").from(user + ":" + password) - val based64String = buffer.toString("base64") - return "Basic " + based64String +fun getBuildsInfoFromArtifactory(target: String): Promise { + val buildsFileName = "buildsSummary.csv" + val artifactoryBuildsDirectory = "builds" + return UrlNetworkConnector(artifactoryUrl).sendRequest(RequestMethod.GET, + "$artifactoryBuildsDirectory/$target/$buildsFileName") } -fun sendGetRequest(url: String, user: String? = null, password: String? = null, jsonContentType: Boolean = false) : String { - val request = require("sync-request") - val headers = mutableListOf>() - if (user != null && password != null) { - headers.add("Authorization" to getAuth(user, password)) - } - if (jsonContentType) { - headers.add("Accept" to "application/json") - } - val response = request("GET", url, - json( - "headers" to json(*(headers.toTypedArray())) - ) - ) - if (response.statusCode != 200) { - error("Error during getting response from $url\n" + - "${response.getBody()}") - } - - return response.getBody().toString() -} - -fun sendUploadRequest(url: String, fileContent: String, user: String? = null, password: String? = null, - extraHeaders: List> = emptyList()) { - val request = require("sync-request") - val headers = mutableListOf>("Content-type" to "text/plain") - if (user != null && password != null) { - headers.add("Authorization" to getAuth(user, password)) - } - extraHeaders.forEach { - headers.add(it.first to it.second) - } - val response = request("PUT", url, - json( - "headers" to json(*(headers.toTypedArray())), - "body" to fileContent - ) - ) - if (response.statusCode != 201) { - error("Error during uploading to $url\n" + - "${response}") - } +fun BenchmarksReport.normalizeBenchmarksSet(dataForNormalization: Map>): BenchmarksReport { + val resultBenchmarksList = benchmarks.map { benchmarksList -> + benchmarksList.value.map { + NormalizedMeanVarianceBenchmark(it.name, it.status, it.score, it.metric, + it.runtimeInUs, it.repeat, it.warmup, (it as MeanVarianceBenchmark).variance, + dataForNormalization[benchmarksList.key]?.get(0)?.score?.let { golden -> it.score / golden } ?: 0.0) + } + }.flatten() + return BenchmarksReport(env, resultBenchmarksList, compiler) } \ No newline at end of file diff --git a/tools/performance-server/src/main/kotlin/utils/AsyncUtils.kt b/tools/performance-server/src/main/kotlin/utils/AsyncUtils.kt new file mode 100644 index 00000000000..d3c7d6353b3 --- /dev/null +++ b/tools/performance-server/src/main/kotlin/utils/AsyncUtils.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.utils + +import kotlin.coroutines.* +import kotlin.js.Promise + +suspend fun Promise.await(): T = suspendCoroutine { cont -> + then({ cont.resume(it) }, { cont.resumeWithException(it) }) +} + +fun launch(context: CoroutineContext = EmptyCoroutineContext, block: suspend () -> Unit) = + block.startCoroutine(Continuation(context) { result -> + result.onFailure { exception -> + throw exception + } + }) \ No newline at end of file diff --git a/tools/performance-server/ui/build.gradle b/tools/performance-server/ui/build.gradle index cc94fb1fafd..e77c736428d 100644 --- a/tools/performance-server/ui/build.gradle +++ b/tools/performance-server/ui/build.gradle @@ -1,3 +1,5 @@ +import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig + buildscript { ext.rootBuildDirectory = file('../../..') @@ -11,6 +13,12 @@ buildscript { maven { url kotlinCompilerRepo } + maven { + url "http://dl.bintray.com/kotlin/kotlin-eap" + } + maven { + url "http://dl.bintray.com/kotlin/kotlin-dev" + } } dependencies { @@ -28,42 +36,40 @@ repositories { maven { url buildKotlinCompilerRepo } + maven { + url "http://dl.bintray.com/kotlin/kotlin-eap" + } + maven { + url "http://dl.bintray.com/kotlin/kotlin-dev" + } } -apply plugin: 'kotlin2js' +apply plugin: 'kotlin-multiplatform' -dependencies { - compile "org.jetbrains.kotlin:kotlin-stdlib-js:$kotlinVersion" -} - -compileKotlin2Js { - kotlinOptions.outputFile = "${projectDir}/js/main.js" - kotlinOptions.main = "call" -} - -sourceSets { - main.kotlin.srcDirs += "src" - main.kotlin.srcDirs += "../../benchmarks/shared/src" - main.kotlin.srcDirs += "../shared/src" - main.output.resourcesDir = "build/js/resources" -} - -task copyResources(type: Copy) { - from sourceSets.main.resources.srcDirs - into file(buildDir.path + "/js") -} - -build.doLast { - configurations.compile.each { File file -> - copy { - includeEmptyDirs = false - - from zipTree(file.absolutePath) - into "${projectDir}/lib/kotlin" - include { fileTreeElement -> - def path = fileTreeElement.path - path.endsWith(".js") && (path.startsWith("META-INF/resources/") || !path.startsWith("META-INF/")) +kotlin { + js { + browser { + binaries.executable() + distribution { + directory = new File("$projectDir/js/") } } } + + sourceSets { + commonMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion" + } + kotlin.srcDir '../../benchmarks/shared/src' + } + jsMain { + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-js:$kotlinVersion" + } + kotlin.srcDir 'src/main/kotlin' + kotlin.srcDir '../shared/src/main/kotlin' + kotlin.srcDir '../src/main/kotlin-js' + } + } } \ No newline at end of file diff --git a/tools/performance-server/ui/css/style.css b/tools/performance-server/ui/css/style.css index 952a6724da2..4e53a519c18 100644 --- a/tools/performance-server/ui/css/style.css +++ b/tools/performance-server/ui/css/style.css @@ -68,11 +68,6 @@ border-color: darkcyan; } -.ct-legend .ct-series-5:before { - background-color: #453d3f; - border-color: #453d3f; -} - .ct-series-b .ct-line, .ct-series-b .ct-point { /* Set the colour of this series line */ diff --git a/tools/performance-server/ui/index.ejs b/tools/performance-server/ui/index.ejs index 6fe0205b0d3..b1c9181aee7 100644 --- a/tools/performance-server/ui/index.ejs +++ b/tools/performance-server/ui/index.ejs @@ -33,8 +33,8 @@ @@ -110,7 +110,6 @@ - - + \ No newline at end of file diff --git a/tools/performance-server/ui/src/main/kotlin/main.kt b/tools/performance-server/ui/src/main/kotlin/main.kt index fcfb38143d0..5b917862322 100644 --- a/tools/performance-server/ui/src/main/kotlin/main.kt +++ b/tools/performance-server/ui/src/main/kotlin/main.kt @@ -1,23 +1,12 @@ /* - * Copyright 2010-2019 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. */ import kotlin.browser.* -import org.w3c.xhr.* +import org.w3c.fetch.* import org.jetbrains.report.json.* -import org.jetbrains.build.Build +import org.jetbrains.buildInfo.Build import kotlin.js.* import kotlin.math.ceil import org.w3c.dom.* @@ -30,63 +19,40 @@ external class ChartistPlugins { external object Chartist { class Svg(form: String, parameters: dynamic, chartArea: String) + val plugins: ChartistPlugins val Interpolation: dynamic fun Line(query: String, data: dynamic, options: dynamic): dynamic } -fun sendGetRequest(url: String) : String { - val request = XMLHttpRequest() +data class Commit(val revision: String, val developer: String) - request.open("GET", url, false) - request.send() - if (request.status == 200.toShort()) { - return request.responseText +fun sendGetRequest(url: String) = window.fetch(url, RequestInit("GET")).then { response -> + if (!response.ok) + error("Error during getting response from $url\n" + + "${response}") + else + response.text() +}.then { text -> text } + +// Get groups of builds for different zoom values. +fun getBuildsGroup(builds: List) = buildsNumberToShow?.let { + val buildsGroups = builds.chunked(buildsNumberToShow!!) + val expectedGroup = buildsGroups.size - 1 + stageToShow + val index = when { + expectedGroup < 0 -> 0 + expectedGroup >= buildsGroups.size -> buildsGroups.size - 1 + else -> expectedGroup } - error("Request to $url has status ${request.status}") -} - -// Parse description with values for metrics. -fun separateValues(values: String, valuesContainer: MutableMap>, convert: (String) -> T = { it as T }) { - val existingSamples = mutableListOf() - val splittedValues = values.split(";") - val insertedList = valuesContainer.values.firstOrNull()?.let { MutableList(it.size) { null } } ?: mutableListOf() - splittedValues.forEach { it -> - val valueParts = it.split("-", limit = 2) - if (valueParts.size != 2) { - error("Wrong format of value $it.") - } - val (sampleName, value) = valueParts - existingSamples.add(sampleName) - val currentList = mutableListOf() - currentList.addAll(insertedList) - valuesContainer.getOrPut(sampleName) { currentList }.add(convert(value)) - } - // Check if there are other keys that are absent in current record. - val missedSamples = valuesContainer.keys - existingSamples - missedSamples.forEach { - valuesContainer[it]!!.add(null) - } -} - -fun getBuildsGroup(builds: List) = buildsNumberToShow?.let { - val buildsGroups = builds.chunked(buildsNumberToShow!!) - val expectedGroup = buildsGroups.size - 1 + stageToShow - val index = when { - expectedGroup < 0 -> 0 - expectedGroup >= buildsGroups.size -> buildsGroups.size - 1 - else -> expectedGroup - } - buildsGroups[index] - } ?: builds - + buildsGroups[index] +} ?: builds +// Get data for chart in needed format. fun getChartData(labels: List, valuesList: Collection>, stageToShow: Int = 0, buildsNumber: Int? = null, classNames: Array? = null): dynamic { - val chartData: dynamic = object{} + val chartData: dynamic = object {} // Show only some part of data. val (labelsData, valuesData) = buildsNumber?.let { - println("Divide on ${labels.size / buildsNumber}") val labelsGroups = labels.chunked(buildsNumber) val valuesListGroups = valuesList.map { it.chunked(buildsNumber) } val expectedGroup = labelsGroups.size - 1 + stageToShow @@ -95,11 +61,11 @@ fun getChartData(labels: List, valuesList: Collection>, stageToS expectedGroup >= labelsGroups.size -> labelsGroups.size - 1 else -> expectedGroup } - Pair(labelsGroups[index], valuesListGroups.map {it[index]}) + Pair(labelsGroups[index], valuesListGroups.map { it[index] }) } ?: Pair(labels, valuesList) chartData["labels"] = labelsData.toTypedArray() chartData["series"] = valuesData.mapIndexed { index, it -> - val series: dynamic = object{} + val series: dynamic = object {} series["data"] = it.toTypedArray() classNames?.let { series["className"] = classNames[index] } series @@ -107,32 +73,33 @@ fun getChartData(labels: List, valuesList: Collection>, stageToS return chartData } +// Create object with options of chart. fun getChartOptions(samples: Array, yTitle: String, classNames: Array? = null): dynamic { - val chartOptions: dynamic = object{} + val chartOptions: dynamic = object {} chartOptions["fullWidth"] = true - val paddingObject: dynamic = object{} + val paddingObject: dynamic = object {} paddingObject["right"] = 40 chartOptions["chartPadding"] = paddingObject - val axisXObject: dynamic = object{} + val axisXObject: dynamic = object {} axisXObject["offset"] = 40 axisXObject["labelInterpolationFnc"] = { value, index, labels -> val labelsCount = 30 val skipNumber = ceil((labels.length as Int).toDouble() / labelsCount).toInt() if (skipNumber > 1) { - if (index % skipNumber == 0) value else null + if (index % skipNumber == 0) value else null } else { value } } chartOptions["axisX"] = axisXObject - val axisYObject: dynamic = object{} + val axisYObject: dynamic = object {} axisYObject["offset"] = 90 chartOptions["axisY"] = axisYObject - val legendObject: dynamic = object{} + val legendObject: dynamic = object {} legendObject["legendNames"] = samples - classNames?.let { legendObject["classNames"] = classNames } - val titleObject: dynamic = object{} - val axisYTitle: dynamic = object{} + classNames?.let { legendObject["classNames"] = classNames.sliceArray(0 until samples.size) } + val titleObject: dynamic = object {} + val axisYTitle: dynamic = object {} axisYTitle["axisTitle"] = yTitle axisYTitle["axisClass"] = "ct-axis-title" val titleOffset: dynamic = {} @@ -153,7 +120,8 @@ fun redirect(url: String) { window.location.href = url } -fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynamic, builds: List, +// Set customizations rules for chart. +fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynamic, builds: List, parameters: Map) { chart.on("draw", { data -> var element = data.element @@ -161,59 +129,74 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam val buildsGroup = getBuildsGroup(builds) val pointSize = 12 val currentBuild = buildsGroup.get(data.index) - // Higlight builds with failures. - if (currentBuild.failuresNumber > 0) { - val svgParameters: dynamic = object{} - svgParameters["d"] = arrayOf("M", data.x, data.y - pointSize, - "L", data.x - pointSize, data.y + pointSize/2, - "L", data.x + pointSize, data.y + pointSize/2, "z").joinToString(" ") - svgParameters["style"] = "fill:rgb(255,0,0);stroke-width:0" - val triangle = Chartist.Svg("path", svgParameters, chartContainer) - element = data.element.replace(triangle) - } else if (currentBuild.buildNumber == parameters["build"]) { - // Higlight choosen build. - val svgParameters: dynamic = object{} - svgParameters["x"] = data.x - pointSize/2 - svgParameters["y"] = data.y - pointSize/2 - svgParameters["height"] = pointSize - svgParameters["width"] = pointSize - svgParameters["style"] = "fill:rgb(0,0,255);stroke-width:0" - val rectangle = Chartist.Svg("rect", svgParameters, "ct-point") - element = data.element.replace(rectangle) - } - // Add tooltips. - val linkToDetailedInfo = "https://kotlin-native-performance.labs.jb.gg/?report=artifactory:" + - "${currentBuild.buildNumber}:${parameters["target"]}:nativeReport.json" + - "${if (data.index - 1 >= 0) - "&compareTo=artifactory:${buildsGroup.get(data.index - 1).buildNumber}:${parameters["target"]}:nativeReport.json" - else ""}" - val information = buildString { - append("${currentBuild.buildNumber}
") - append("Value: ${data.value.y.toFixed(4)}
") + currentBuild?.let { currentBuild -> + // Higlight builds with failures. if (currentBuild.failuresNumber > 0) { - append("failures: ${currentBuild.failuresNumber}
") + val svgParameters: dynamic = object {} + svgParameters["d"] = arrayOf("M", data.x, data.y - pointSize, + "L", data.x - pointSize, data.y + pointSize / 2, + "L", data.x + pointSize, data.y + pointSize / 2, "z").joinToString(" ") + svgParameters["style"] = "fill:rgb(255,0,0);stroke-width:0" + val triangle = Chartist.Svg("path", svgParameters, chartContainer) + element = data.element.replace(triangle) + } else if (currentBuild.buildNumber == parameters["build"]) { + // Higlight choosen build. + val svgParameters: dynamic = object {} + svgParameters["x"] = data.x - pointSize / 2 + svgParameters["y"] = data.y - pointSize / 2 + svgParameters["height"] = pointSize + svgParameters["width"] = pointSize + svgParameters["style"] = "fill:rgb(0,0,255);stroke-width:0" + val rectangle = Chartist.Svg("rect", svgParameters, "ct-point") + element = data.element.replace(rectangle) } - append("branch: ${currentBuild.branch}
") - append("date: ${currentBuild.date}
") - append("time: ${currentBuild.formattedStartTime}-${currentBuild.formattedFinishTime}
") - append("Commits:
") - val commitsList = currentBuild.commits.split(";") - commitsList.forEach { - if (!it.isEmpty()) - append("${it.substringBefore("by").substring(0, 7)} by ${it.substringAfter("by ")}
") + // Add tooltips. + var shift = 1 + var previousBuild: Build? = null + while (previousBuild == null && data.index - shift >= 0) { + previousBuild = buildsGroup.get(data.index - shift) + shift++ } - + val linkToDetailedInfo = "https://kotlin-native-performance.labs.jb.gg/?report=" + + "${currentBuild.buildNumber}:${parameters["target"]}" + + "${previousBuild?.let { + "&compareTo=${previousBuild.buildNumber}:${parameters["target"]}" + } ?: ""}" + val information = buildString { + append("${currentBuild.buildNumber}
") + append("Value: ${data.value.y.toFixed(4)}
") + if (currentBuild.failuresNumber > 0) { + append("failures: ${currentBuild.failuresNumber}
") + } + append("branch: ${currentBuild.branch}
") + append("date: ${currentBuild.date}
") + append("time: ${currentBuild.formattedStartTime}-${currentBuild.formattedFinishTime}
") + append("Commits:
") + val commitsList = (JsonTreeParser.parse("{${currentBuild.commits}}") as JsonObject).getArray("commits").map { + Commit( + (it as JsonObject).getPrimitive("revision").content, + (it as JsonObject).getPrimitive("developer").content + ) + } + val commits = if (commitsList.size > 3) commitsList.slice(0..2) else commitsList + commits.forEach { + append("${it.revision.substring(0, 7)} by ${it.developer}
") + } + if (commitsList.size > 3) { + append("...") + } + } + element._node.setAttribute("title", information) + element._node.setAttribute("data-chart-tooltip", chartContainer) + element._node.addEventListener("click", { + redirect(linkToDetailedInfo) + }) } - element._node.setAttribute("title", information) - element._node.setAttribute("data-chart-tooltip", chartContainer) - element._node.addEventListener("click", { - redirect(linkToDetailedInfo) - }) } }) chart.on("created", { val currentChart = jquerySelector - val parameters: dynamic = object{} + val parameters: dynamic = object {} parameters["selector"] = "[data-chart-tooltip=\"$chartContainer\"]" parameters["container"] = "#$chartContainer" parameters["html"] = true @@ -225,6 +208,12 @@ var stageToShow = 0 var buildsNumberToShow: Int? = null +fun waitForElements(elements: Iterable, action: () -> Unit) { + if (elements.filterNotNull().size == elements.count()) + action() + else window.setTimeout(waitForElements(elements, action), 500) +} + fun main(args: Array) { val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg" buildsNumberToShow = null @@ -243,45 +232,52 @@ fun main(args: Array) { } } - // Get builds. - val buildsUrl = buildString { - append("$serverUrl/builds") - append("/${parameters["target"]}") - append("/${parameters["type"]}") - append("/${parameters["branch"]}") - append("/${parameters["build"]}") + // Get branches. + val branchesUrl = "$serverUrl/branches" + sendGetRequest(branchesUrl).then { response -> + val branches: Array = JSON.parse(response) + // Add release branches to selector. + branches.filter { it != "master" }.forEach { + if ("v(\\d|\\.)+(-M\\d)?-fixes".toRegex().matches(it)) { + val option = Option(it, it) + js("$('#inputGroupBranch')").append(js("$(option)")) + } + } + document.querySelector("#inputGroupBranch [value=\"${parameters["branch"]}\"]")?.setAttribute("selected", "true") } - val response = sendGetRequest(buildsUrl) - val data = JsonTreeParser.parse(response) - if (data !is JsonArray) { - error("Response is expected to be an array.") - } - val builds = data.jsonArray.map { Build.create(it as JsonObject) } - .sortedWith(compareBy ( { it.buildNumber.substringBefore(".").toInt() }, { it.buildNumber.substringAfter(".").substringBefore("-").toDouble() }, { it.buildNumber.substringAfterLast("-").toInt() })) - val branchesUrl = "$serverUrl/branches/${parameters["target"]}" - - val branches: Array = JSON.parse(sendGetRequest(branchesUrl)) - val releaseBranches = branches.filter { "^v\\d+(\\.\\d+)+(-M\\d)?-fixes$".toRegex().find(it) != null } - - // Fill autocomplete list. + // Fill autocomplete list with build numbers. val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}" - val buildsNumbers: Array = JSON.parse(sendGetRequest(buildsNumbersUrl)) - - // Add release branches to selector. - releaseBranches.forEach { - val option = Option(it, it) - js("$('#inputGroupBranch')").append(js("$(option)")) + sendGetRequest(buildsNumbersUrl).then { response -> + val buildsNumbers: Array = JSON.parse(response) + val autocompleteParameters: dynamic = object {} + autocompleteParameters["lookup"] = buildsNumbers + autocompleteParameters["onSelect"] = { suggestion -> + if (suggestion.value != parameters["build"]) { + val newLink = "http://${window.location.host}/?target=${parameters["target"]}&type=${parameters["type"]}" + + "${if ((suggestion.value as String).isEmpty()) "" else "&build=${suggestion.value}"}" + window.location.href = newLink + } + } + js("$( \"#highligted_build\" )").autocomplete(autocompleteParameters) + js("$('#highligted_build')").change({ value -> + val newValue = js("$(this).val()").toString() + if (newValue.isEmpty() || newValue in buildsNumbers) { + val newLink = "http://${window.location.host}/?target=${parameters["target"]}&type=${parameters["type"]}" + + "${if (newValue.isEmpty()) "" else "&build=$newValue"}" + window.location.href = newLink + } + }) } // Change inputs values connected with parameters and add events listeners. document.querySelector("#inputGroupTarget [value=\"${parameters["target"]}\"]")?.setAttribute("selected", "true") document.querySelector("#inputGroupBuildType [value=\"${parameters["type"]}\"]")?.setAttribute("selected", "true") - document.querySelector("#inputGroupBranch [value=\"${parameters["branch"]}\"]")?.setAttribute("selected", "true") (document.getElementById("highligted_build") as HTMLInputElement).value = parameters["build"]!! // Add onChange events for fields. + // Don't use AJAX to have opportunity to share results with simple links. js("$('#inputGroupTarget')").change({ val newValue = js("$(this).val()") if (newValue != parameters["target"]) { @@ -307,69 +303,160 @@ fun main(args: Array) { } }) - val autocompleteParameters: dynamic = object{} - autocompleteParameters["lookup"] = buildsNumbers - autocompleteParameters["onSelect"] = { suggestion -> - if (suggestion.value != parameters["build"]) { - val newLink = "http://${window.location.host}/?target=${parameters["target"]}&type=${parameters["type"]}" + - "${if ((suggestion.value as String).isEmpty()) "" else "&build=${suggestion.value}"}" - window.location.href = newLink - } - } - js("$( \"#highligted_build\" )").autocomplete(autocompleteParameters) - js("$('#highligted_build')").change({ value -> - val newValue = js("$(this).val()").toString() - if (newValue.isEmpty() || newValue in builds.map {it.buildNumber}) { - val newLink = "http://${window.location.host}/?target=${parameters["target"]}&type=${parameters["type"]}" + - "${if (newValue.isEmpty()) "" else "&build=$newValue"}" - window.location.href = newLink - } - }) - + val platformSpecificBenchs = if (parameters["target"] == "Mac_OS_X") ",FrameworkBenchmarksAnalyzer,circlet_iosX64" else + if (parameters["target"] == "Linux") ",kotlinx.coroutines" else "" // Collect information for charts library. - val labels = mutableListOf() - val executionTime = mutableMapOf>() - val compileTime = mutableMapOf>() - val codeSize = mutableMapOf>() - val bundleSize = mutableListOf() + val valuesToShow = mapOf("EXECUTION_TIME" to arrayOf(mapOf( + "normalize" to "true" + )), + "COMPILE_TIME" to arrayOf(mapOf( + "samples" to "HelloWorld,Videoplayer$platformSpecificBenchs", + "agr" to "samples" + )), + "CODE_SIZE" to arrayOf(mapOf( + "normalize" to "true", + "exclude" to if (parameters["target"] == "Linux") + "kotlinx.coroutines" + else if (parameters["target"] == "Mac_OS_X") + "circlet_iosX64" + else "" + ), mapOf( + "normalize" to "true", + "agr" to "samples", + "samples" to platformSpecificBenchs.removePrefix(",") + )), + "BUNDLE_SIZE" to arrayOf(mapOf("samples" to "KotlinNative", + "agr" to "samples")) + ) - builds.forEach { - // Choose labels on x axis. - if (parameters["type"] == "day") { - labels.add(it.date) - } else { - labels.add(it.buildNumber) - } - separateValues(it.executionTime, executionTime) { value -> value.toDouble() } - separateValues(it.compileTime, compileTime) { value -> value.toDouble() / 1000 } - separateValues(it.codeSize, codeSize) { value -> value.toDouble() } - bundleSize.add(it.bundleSize?.toInt()?. let { it / 1024 / 1024 }) - } + var execData = listOf() to listOf>() + var compileData = listOf() to listOf>() + var codeSizeData = listOf() to listOf>() + var bundleSizeData = listOf() to listOf>() val sizeClassNames = arrayOf("ct-series-e", "ct-series-f", "ct-series-g") // Draw charts. - val execChart = Chartist.Line("#exec_chart", getChartData(labels, executionTime.values, stageToShow, buildsNumberToShow), - getChartOptions(executionTime.keys.toTypedArray(), "Normalized time")) - val compileChart = Chartist.Line("#compile_chart", getChartData(labels, compileTime.values, stageToShow, buildsNumberToShow), - getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds")) - val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, stageToShow, buildsNumberToShow, sizeClassNames), - getChartOptions(codeSize.keys.toTypedArray(), "Normalized size", arrayOf("ct-series-4", "ct-series-5", "ct-series-6"))) - val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), stageToShow, buildsNumberToShow, sizeClassNames), - getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4"))) + var execChart: dynamic = null + var compileChart: dynamic = null + var codeSizeChart: dynamic = null + var bundleSizeChart: dynamic = null - // Tooltips and higlights. - customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters) - customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters) - customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters) - customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters) + val descriptionUrl = "$serverUrl/buildsDesc/${parameters["target"]}?type=${parameters["type"]}" + + "${if (parameters["branch"] != "all") "&branch=${parameters["branch"]}" else ""}" + val metricUrl = "$serverUrl/metricValue/${parameters["target"]}/" + + // Send requests to get all needed metric values. + valuesToShow.map { (metric, arrayOfSettings) -> + val resultValues = arrayOfSettings.map { settings -> + val getParameters = with(StringBuilder()) { + if (settings.isNotEmpty()) { + append("?") + } + var prefix = "" + settings.forEach { (key, value) -> + if (value.isNotEmpty()) { + append("$prefix$key=$value") + prefix = "&" + } + } + toString() + } + val branchParameter = if (parameters["branch"] != "all") + (if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}" + else "" + + val url = "$metricUrl$metric$getParameters$branchParameter${ + if (parameters["type"] != "all") + (if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}" + else "" + }" + sendGetRequest(url) + }.toTypedArray() + + // Get metrics values for charts. + Promise.all(resultValues).then { responses -> + val valuesList = responses.map { response -> + val results = (JsonTreeParser.parse(response) as JsonArray).map { + (it as JsonObject).getPrimitive("first").content to + it.getArray("second").map { (it as JsonPrimitive).doubleOrNull } + } + + val labels = results.map { it.first } + val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } } + ?: emptyList() + labels to values + } + val labels = valuesList[0].first + val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart } + + when (metric) { + // Update chart with gotten data. + "COMPILE_TIME" -> { + compileData = labels to values.map { it.map { it?.let { it / 1000 } } } + compileChart = Chartist.Line("#compile_chart", + getChartData(labels, compileData.second, stageToShow, buildsNumberToShow), + getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(), + "Time, milliseconds")) + } + "EXECUTION_TIME" -> { + execData = labels to values + execChart = Chartist.Line("#exec_chart", + getChartData(labels, execData.second, stageToShow, buildsNumberToShow), + getChartOptions(arrayOf("Geometric Mean"), "Normalized time")) + } + "CODE_SIZE" -> { + codeSizeData = labels to values + codeSizeChart = Chartist.Line("#codesize_chart", + getChartData(labels, codeSizeData.second, + stageToShow, buildsNumberToShow, sizeClassNames), + getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',') + .filter { it.isNotEmpty() }, + "Normalized size", + arrayOf("ct-series-4", "ct-series-5", "ct-series-6"))) + } + "BUNDLE_SIZE" -> { + bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } } + bundleSizeChart = Chartist.Line("#bundlesize_chart", + getChartData(labels, + bundleSizeData.second, stageToShow, + buildsNumberToShow, sizeClassNames), + getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4"))) + } + else -> error("No chart for metric $metric") + } + true + } + } + + // Update all charts with using same data. val updateAllCharts: () -> Unit = { - execChart.update(getChartData(labels, executionTime.values, stageToShow, buildsNumberToShow)) - compileChart.update(getChartData(labels, compileTime.values, stageToShow, buildsNumberToShow)) - codeSizeChart.update(getChartData(labels, codeSize.values, stageToShow, buildsNumberToShow, sizeClassNames)) - bundleSizeChart.update(getChartData(labels, listOf(bundleSize), stageToShow, buildsNumberToShow, sizeClassNames)) + execChart.update(getChartData(execData.first, execData.second, stageToShow, buildsNumberToShow)) + compileChart.update(getChartData(compileData.first, compileData.second, stageToShow, buildsNumberToShow)) + codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, stageToShow, buildsNumberToShow, sizeClassNames)) + bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, stageToShow, buildsNumberToShow, sizeClassNames)) + } + + // Get builds description. + sendGetRequest(descriptionUrl).then { response -> + val buildsInfo = response as String + val data = JsonTreeParser.parse(buildsInfo) + if (data !is JsonArray) { + error("Response is expected to be an array.") + } + val builds = data.jsonArray.map { + val element = it as JsonElement + if (element.isNull) null else Build.create(element as JsonObject) + } + waitForElements(listOf(execChart, compileChart, codeSizeChart, bundleSizeChart)) { + customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters) + customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters) + customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters) + customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters) + updateAllCharts() + } } js("$('#plusBtn')").click({ @@ -379,14 +466,13 @@ fun main(args: Array) { } else { it } - } ?: labels.size / zoomRatio - println(buildsNumberToShow) + } ?: execData.first.size / zoomRatio updateAllCharts() }) js("$('#minusBtn')").click({ buildsNumberToShow = buildsNumberToShow?.let { - if (it * zoomRatio <= labels.size) { + if (it * zoomRatio <= execData.first.size) { it * zoomRatio } else { null @@ -397,11 +483,11 @@ fun main(args: Array) { js("$('#prevBtn')").click({ buildsNumberToShow?.let { - val bottomBorder = -labels.size / (buildsNumberToShow as Int) + val bottomBorder = -execData.first.size / (buildsNumberToShow as Int) if (stageToShow - 1 > bottomBorder) { stageToShow-- } - } ?: run { stageToShow = 0} + } ?: run { stageToShow = 0 } updateAllCharts() })