From 130fd8311ee8722bdd355fc8c956f648521c4707 Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Tue, 7 Sep 2021 12:01:54 +0300 Subject: [PATCH] KT-48620: add readable output size metric into build scan --- .../incremental/IncrementalCompilerRunner.kt | 12 +++++ .../gradle/plugin/AbiSnapshotDiffService.kt | 54 ------------------- .../plugin/KotlinGradleBuildServices.kt | 2 +- .../statistics/KotlinBuildEsStatListener.kt | 3 +- .../statistics/ReportStatisticsToBuildScan.kt | 28 +++++++--- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 12 ----- 6 files changed, 36 insertions(+), 75 deletions(-) delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 8289a459c32..8bfa97c3e24 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -174,6 +174,18 @@ abstract class IncrementalCompilerRunner< // Here we should analyze exit code of compiler. E.g. compiler failure should lead to caches rebuild, // but now JsKlib compiler reports invalid exit code. cachesMayBeCorrupted = false + + reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) { + reporter.addMetric( + BuildPerformanceMetric.SNAPSHOT_SIZE, + buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length() + ) + if (cacheDirectory.exists() && cacheDirectory.isDirectory()) { + cacheDirectory.walkTopDown().filter { it.isFile }.map { it.length() }.sum().let { + reporter.addMetric(BuildPerformanceMetric.OUTPUT_SIZE, it) + } + } + } return exitCode } catch (e: Exception) { // todo: catch only cache corruption // todo: warn? diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt deleted file mode 100644 index 4a258319df9..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin - -import org.gradle.api.services.BuildService -import org.gradle.api.services.BuildServiceParameters -import org.jetbrains.kotlin.build.report.BuildReporter -import org.jetbrains.kotlin.incremental.* - -abstract class AbiSnapshotDiffService() : BuildService { - abstract class Parameters : BuildServiceParameters { - abstract val caches: IncrementalCachesManager<*> - abstract val reporter: BuildReporter - abstract val sourceFilesExtensions: List - } - - val caches: IncrementalCachesManager<*> = parameters.caches - val reporter: BuildReporter = parameters.reporter - val sourceFilesExtensions: List = parameters.sourceFilesExtensions - - companion object { -// //Store list of changed lookups -// val diffCache: MutableMap, DirtyFilesContainer> = mutableMapOf() -// @TestOnly -// fun compareJarsInternal(caches: IncrementalCachesManager<*>, reporter: ICReporter, sourceFilesExtensions: List, -// snapshot: AbiSnapshot, newJar: AbiSnapshot) = -// diffCache.computeIfAbsent(Pair(snapshot, newJar)) { (snapshotJar, actualJar) -> -// DirtyFilesContainer(caches, reporter, sourceFilesExtensions) -// .also { -// it.addByDirtyClasses(snapshotJar.fqNames.minus(actualJar.fqNames)) -// it.addByDirtyClasses(actualJar.fqNames.minus(snapshotJar.fqNames)) -// it.addByDirtySymbols(snapshotJar.symbols.minus(actualJar.symbols)) -// it.addByDirtySymbols(actualJar.symbols.minus(snapshotJar.symbols)) -// } -// } - } - -// @Synchronized -// fun compareJarsInternal(snapshot: AbiSnapshot, newJar: AbiSnapshot): DirtyFilesContainer { -// return diffCache.computeIfAbsent(Pair(snapshot, newJar)) { (snapshotJar, actualJar) -> -// DirtyFilesContainer(caches, reporter, sourceFilesExtensions) -// .also { -// it.addByDirtyClasses(snapshotJar.fqNames.minus(actualJar.fqNames)) -// it.addByDirtyClasses(actualJar.fqNames.minus(snapshotJar.fqNames)) -// it.addByDirtySymbols(snapshotJar.symbols.minus(actualJar.symbols)) -// it.addByDirtySymbols(actualJar.symbols.minus(snapshotJar.symbols)) -// } -// -// } -// } -} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt index 2411340227f..ffc8364c464 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt @@ -79,7 +79,7 @@ internal class KotlinGradleBuildServices private constructor( val listeners = project.rootProject.objects.listProperty(ReportStatistics::class.java) .value(listOf(ReportStatisticsToElasticSearch)) project.rootProject.extensions.findByName("buildScan") - ?.also { listeners.add(ReportStatisticsToBuildScan(it as BuildScanExtension, UUID.randomUUID().toString(), "kotlin_version")) } + ?.also { listeners.add(ReportStatisticsToBuildScan(it as BuildScanExtension)) } KotlinBuildEsStatListener(project.rootProject.name, listeners.get(), UUID.randomUUID().toString()) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildEsStatListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildEsStatListener.kt index 76a0f0ee5f8..9befc975566 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildEsStatListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildEsStatListener.kt @@ -68,8 +68,7 @@ class KotlinBuildEsStatListener(val projectName: String, val reportStatistics: L } val taskExecutionResult = TaskExecutionResults[taskPath] val timeData = taskExecutionResult?.buildMetrics?.buildTimes?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap() - val perfData = taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.mapValues { it.value / 1000 } - ?.filterValues { value -> value != 0L } ?: emptyMap() + val perfData = taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap() val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) { is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } is ChangedFiles.Dependencies -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt index e9689cfc70d..4a8ec4ce8ae 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt @@ -10,7 +10,15 @@ import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics import org.jetbrains.kotlin.konan.file.File -class ReportStatisticsToBuildScan(private val buildScan: BuildScanExtension, private val buildUuid: String, private val kotlinVersion: String) : ReportStatistics { +class ReportStatisticsToBuildScan( + private val buildScan: BuildScanExtension +) : ReportStatistics { + companion object { + const val kbSize = 1024 + const val mbSize = kbSize * kbSize + const val gbSize = kbSize * mbSize + } + override fun report(data: CompileStatData) { buildScan.value(data.taskName, readableString(data)) data.tags.forEach { buildScan.tag(it) } @@ -26,10 +34,18 @@ class ReportStatisticsToBuildScan(private val buildScan: BuildScanExtension, pri } data.changes.joinTo(readableString, prefix = "Changes: [", postfix = "]; ") { it.substringAfterLast(File.separator) } - readableString.append("Performance: [") - data.timeData.forEach { (key, value) -> readableString.append(key.readableString).append(": ").append(value).append("ms, ") } - data.perfData.forEach { (key, value) -> readableString.append(key.readableString).append(": ").append(value).append("kb, ") } - readableString.append("]; ") + val timeData = data.timeData.map { (key, value) -> "${key.readableString}: ${value}ms"} //sometimes it is better to have separate variable to be able debug + val perfData = data.perfData.map { (key, value) -> "$key: ${readableFileLength(value)}"} + timeData.union(perfData).joinTo(readableString, ",", "Performance: [", "]") return readableString.toString() } -} \ No newline at end of file + + + private fun readableFileLength(length: Long): String = + when { + length / gbSize > 0 -> "${length / gbSize} GB" + length / mbSize > 0 -> "${length / mbSize} MB" + length / kbSize > 0 -> "${length / kbSize} KB" + else -> "$length B" + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index 8919fa0186c..52613cfb6dd 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -345,18 +345,6 @@ abstract class AbstractKotlinCompile : AbstractKotl try { executeImpl(inputChanges) - metrics.measure(BuildTime.CALCULATE_OUTPUT_SIZE) { - metrics.addMetric( - BuildPerformanceMetric.SNAPSHOT_SIZE, - taskBuildDirectory.file("build-history.bin").get().asFile.length() + - taskBuildDirectory.file("last-build.bin").get().asFile.length() + - taskBuildDirectory.file("abi-snapshot.bin").get().asFile.length() - ) - metrics.addMetric(BuildPerformanceMetric.OUTPUT_SIZE, - taskBuildDirectory.dir("caches-jvm").get().asFileTree.files.filter { it.isFile }.map { it.length() } - .sum() - ) - } } catch (t: Throwable) { if (outputsBackup != null) { metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {