From a34657195b2cbdbb5cdf25f6ee1824474d83c4f8 Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Mon, 27 Feb 2023 18:59:00 +0100 Subject: [PATCH] Add GC metric to build statistic --- .../build/report/metrics/BuildMetrics.kt | 4 ++- .../report/metrics/BuildMetricsReporter.kt | 20 +++++++++++ .../metrics/BuildMetricsReporterImpl.kt | 23 ++++++++++++- .../report/metrics/BuildPerformanceMetric.kt | 3 ++ .../metrics/DoNothingBuildMetricsReporter.kt | 9 +++++ .../kotlin/build/report/metrics/GcMetrics.kt | 33 +++++++++++++++++++ .../kotlin/daemon/CompileServiceImpl.kt | 6 ++++ .../jetbrains/kotlin/gradle/BuildReportsIT.kt | 3 ++ .../statistics/CompileStatisticsData.kt | 2 ++ .../gradle/report/BuildMetricsService.kt | 2 +- .../report/PlainTextBuildReportWriter.kt | 17 ++++++++++ .../kotlin/gradle/report/reportDataUtil.kt | 4 ++- 12 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 build-common/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt index 68d92fe9d00..47a3e456a9a 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt @@ -10,12 +10,14 @@ import java.io.Serializable data class BuildMetrics( val buildTimes: BuildTimes = BuildTimes(), val buildPerformanceMetrics: BuildPerformanceMetrics = BuildPerformanceMetrics(), - val buildAttributes: BuildAttributes = BuildAttributes() + val buildAttributes: BuildAttributes = BuildAttributes(), + val gcMetrics: GcMetrics = GcMetrics() ) : Serializable { fun addAll(other: BuildMetrics) { buildTimes.addAll(other.buildTimes) buildPerformanceMetrics.addAll(other.buildPerformanceMetrics) buildAttributes.addAll(other.buildAttributes) + gcMetrics.addAll(other.gcMetrics) } companion object { diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt index 0034bccecc6..39d01c7bc1b 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.build.report.metrics +import java.lang.management.ManagementFactory + interface BuildMetricsReporter { fun startMeasure(time: BuildTime) fun endMeasure(time: BuildTime) @@ -14,6 +16,11 @@ interface BuildMetricsReporter { fun addMetric(metric: BuildPerformanceMetric, value: Long) fun addTimeMetric(metric: BuildPerformanceMetric) + //Change metric to enum if possible + fun addGcMetric(metric: String, value: GcMetric) + fun startGcMetric(name: String, value: GcMetric) + fun endGcMetric(name: String, value: GcMetric) + fun addAttribute(attribute: BuildAttribute) fun getMetrics(): BuildMetrics @@ -27,4 +34,17 @@ inline fun BuildMetricsReporter.measure(time: BuildTime, fn: () -> T): T { } finally { endMeasure(time) } +} + + +fun BuildMetricsReporter.startMeasureGc() { + ManagementFactory.getGarbageCollectorMXBeans().forEach { + startGcMetric(it.name, GcMetric(it.collectionTime, it.collectionCount)) + } +} + +fun BuildMetricsReporter.endMeasureGc() { + ManagementFactory.getGarbageCollectorMXBeans().forEach { + endGcMetric(it.name, GcMetric(it.collectionTime, it.collectionCount)) + } } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt index 5de3a45e37f..a38664d2c33 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt @@ -13,9 +13,11 @@ class BuildMetricsReporterImpl : BuildMetricsReporter, Serializable { EnumMap( BuildTime::class.java ) + private val myGcPerformance = HashMap() private val myBuildTimes = BuildTimes() private val myBuildMetrics = BuildPerformanceMetrics() private val myBuildAttributes = BuildAttributes() + private val myGcMetrics = GcMetrics() override fun startMeasure(time: BuildTime) { if (time in myBuildTimeStartNs) { @@ -30,6 +32,19 @@ class BuildMetricsReporterImpl : BuildMetricsReporter, Serializable { myBuildTimes.addTimeNs(time, durationNs) } + override fun startGcMetric(name: String, value: GcMetric) { + if (name in myGcPerformance) { + error("$name was restarted before it finished") + } + myGcPerformance[name] = value + } + + override fun endGcMetric(name: String, value: GcMetric) { + val startValue = myGcPerformance.remove(name) ?: error("$name finished before it started") + val diff = value - startValue + myGcMetrics.add(name, diff) + } + override fun addTimeMetricNs(time: BuildTime, durationNs: Long) { myBuildTimes.addTimeNs(time, durationNs) } @@ -47,6 +62,10 @@ class BuildMetricsReporterImpl : BuildMetricsReporter, Serializable { } + override fun addGcMetric(metric: String, value: GcMetric) { + myGcMetrics.add(metric, value) + } + override fun addAttribute(attribute: BuildAttribute) { myBuildAttributes.add(attribute) } @@ -55,12 +74,14 @@ class BuildMetricsReporterImpl : BuildMetricsReporter, Serializable { BuildMetrics( buildTimes = myBuildTimes, buildPerformanceMetrics = myBuildMetrics, - buildAttributes = myBuildAttributes + buildAttributes = myBuildAttributes, + gcMetrics = myGcMetrics ) override fun addMetrics(metrics: BuildMetrics) { myBuildAttributes.addAll(metrics.buildAttributes) myBuildTimes.addAll(metrics.buildTimes) myBuildMetrics.addAll(metrics.buildPerformanceMetrics) + myGcMetrics.addAll(metrics.gcMetrics) } } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index 218f1efca4f..1dd6dc55657 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -18,6 +18,9 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va DAEMON_INCREASED_MEMORY(readableString = "Increase memory usage", type = ValueType.BYTES), DAEMON_MEMORY_USAGE(readableString = "Total memory usage at the end of build", type = ValueType.BYTES), + DAEMON_GC_TIME(readableString = "Time spent in GC", type = ValueType.NANOSECONDS), + DAEMON_GC_COUNT(readableString = "Count of GC", type = ValueType.NUMBER), + COMPILE_ITERATION(parent = null, "Total compiler iteration", type = ValueType.NUMBER), ANALYZED_LINES_NUMBER(parent = COMPILE_ITERATION, "Number of lines analyzed", type = ValueType.NUMBER), CODE_GENERATED_LINES_NUMBER(parent = COMPILE_ITERATION, "Number of lines for code generation", type = ValueType.NUMBER), diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt index 390be0d99e7..f0a322aa1b3 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt @@ -24,6 +24,15 @@ object DoNothingBuildMetricsReporter : BuildMetricsReporter { override fun addAttribute(attribute: BuildAttribute) { } + override fun addGcMetric(metric: String, value: GcMetric) { + } + + override fun startGcMetric(name: String, value: GcMetric) { + } + + override fun endGcMetric(name: String, value: GcMetric) { + } + override fun getMetrics(): BuildMetrics = BuildMetrics( BuildTimes(), diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt new file mode 100644 index 00000000000..c56249a8186 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.io.Serializable +import kotlin.collections.HashMap + +class GcMetrics : Serializable { + + private val myGcMetrics = HashMap() + fun addAll(gcMetrics: GcMetrics) { + myGcMetrics.putAll(gcMetrics.myGcMetrics) + } + + fun add(metric: String, value: GcMetric) { + myGcMetrics[metric] = value + } + + fun asGcCountMap(): Map = myGcMetrics.mapValues { it.value.count } + fun asGcTimeMap(): Map = myGcMetrics.mapValues { it.value.time } + fun asMap(): Map = myGcMetrics +} +data class GcMetric( + val time: Long, + val count: Long +): Serializable { + operator fun minus(increment: GcMetric): GcMetric { + return GcMetric(time - increment.time, count - increment.count) + } +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index d3e7c1ae7e1..9be0166a1cf 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -23,6 +23,8 @@ import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS import org.jetbrains.kotlin.build.report.RemoteBuildReporter import org.jetbrains.kotlin.build.report.info +import org.jetbrains.kotlin.build.report.metrics.endMeasureGc +import org.jetbrains.kotlin.build.report.metrics.startMeasureGc import org.jetbrains.kotlin.cli.common.CLICompiler import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.cli.common.ExitCode @@ -532,6 +534,7 @@ abstract class CompileServiceImplBase( compilerMessageCollector: MessageCollector, reporter: RemoteBuildReporter ): ExitCode { + reporter.startMeasureGc() val allKotlinFiles = arrayListOf() val freeArgsWithoutKotlinFiles = arrayListOf() args.freeArgs.forEach { @@ -565,6 +568,7 @@ abstract class CompileServiceImplBase( return try { compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles) } finally { + reporter.endMeasureGc() reporter.flush() } } @@ -575,6 +579,7 @@ abstract class CompileServiceImplBase( compilerMessageCollector: MessageCollector, reporter: RemoteBuildReporter ): ExitCode { + reporter.startMeasureGc() val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS + (incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct() val dotExtensions = allKotlinExtensions.map { ".$it" } @@ -630,6 +635,7 @@ abstract class CompileServiceImplBase( return try { compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, projectRoot) } finally { + reporter.endMeasureGc() reporter.flush() } } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt index 59bd766fd09..4211f4c1043 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt @@ -73,6 +73,9 @@ class BuildReportsIT : KGPBaseTest() { //for non-incremental builds assertTrue { report.contains("Build attributes:") } assertTrue { report.contains("REBUILD_REASON:") } + //gc metrics + assertTrue {report.contains("GC count:")} + assertTrue {report.contains("GC time:")} } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt index 53fdf639460..5bb11a2ffb0 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt @@ -32,6 +32,8 @@ data class CompileStatisticsData( //TODO think about it,time in milliseconds val buildTimesMetrics: Map, val performanceMetrics: Map, + val gcTimeMetrics: Map?, + val gcCountMetrics: Map?, val type: String = BuildDataType.TASK_DATA.name ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt index eead1d2e204..a1ada3401c1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt @@ -171,7 +171,7 @@ internal class TaskRecord( override val buildMetrics: BuildMetrics, override val didWork: Boolean, override val skipMessage: String?, - override val icLogLines: List + override val icLogLines: List, ) : BuildOperationRecord { override val isFromKotlinPlugin: Boolean = classFqName.startsWith("org.jetbrains.kotlin") } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt index 5d7cf13199e..42fb9c7821c 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt @@ -85,6 +85,9 @@ internal class PlainTextBuildReportWriter( if (newLineBetweenSections) p.println() printBuildAttributes(buildMetrics.buildAttributes) + if (newLineBetweenSections) p.println() + + printGcMetrics(buildMetrics.gcMetrics) } private fun printBuildTimes(buildTimes: BuildTimes) { @@ -162,6 +165,20 @@ internal class PlainTextBuildReportWriter( } } + private fun printGcMetrics(gcMetrics: GcMetrics) { + val allGcMetrics = gcMetrics.asMap() + if (allGcMetrics.isEmpty()) return + p.withIndent("GC metrics:") { + for (gcMetric in allGcMetrics) { + p.println("${gcMetric.key}:") + p.withIndent { + p.println("GC count: ${gcMetric.value.count}") + p.println("GC time: ${gcMetric.value.time}") + } + } + } + } + private fun printTaskOverview(build: BuildExecutionData) { var allTasksTimeMs = 0L var kotlinTotalTimeMs = 0L diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt index 2a8a9a5a973..25281ec0a9e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt @@ -81,7 +81,9 @@ internal fun prepareData( kotlinVersion = kotlinVersion, buildUuid = uuid, finishTime = System.currentTimeMillis(), - compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList() + compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList(), + gcCountMetrics = buildMetrics?.gcMetrics?.asGcCountMap(), + gcTimeMetrics = buildMetrics?.gcMetrics?.asGcTimeMap() ) }