KT-56352 Add metric filter for build scan

#KT-56352 Fixed
2022-11-25 12:51:55 +01:00
parent e8570826e6
commit 8cfe234140
8 changed files with 225 additions and 15 deletions
@@ -223,4 +223,16 @@ class BuildReportsIT : KGPBaseTest() {
        }
    }

+    @DisplayName("build scan metrics validation")
+    @GradleTest
+    fun testBuildScanMetricsValidation(gradleVersion: GradleVersion) {
+        project("simpleProject", gradleVersion) {
+            buildAndFail(
+                "compileKotlin", "-Pkotlin.build.report.output=BUILD_SCAN", "-Pkotlin.build.report.build_scan.metrics=unknown_prop"
+            ) {
+                assertOutputContains("Unknown metric: 'unknown_prop', list of available metrics")
+            }
+        }
+    }
+
 }
@@ -118,6 +118,9 @@ internal class PropertiesProvider private constructor(private val project: Proje
    val buildReportBuildScanCustomValuesLimit: Int
        get() = property("kotlin.build.report.build_scan.custom_values_limit")?.toInt() ?: 1000

+    val buildReportBuildScanMetrics: String?
+        get() = property("kotlin.build.report.build_scan.metrics")
+
    val buildReportMetrics: Boolean
        get() = booleanProperty("kotlin.build.report.metrics") ?: false

@@ -163,7 +163,7 @@ abstract class BuildMetricsService : BuildService<BuildServiceParameters.None>,

 }

-private class TaskRecord(
+internal class TaskRecord(
    override val path: String,
    override val classFqName: String,
    override val startTimeMs: Long,
@@ -187,6 +187,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
                        event, parameters.projectName.get(), buildUuid, parameters.label.orNull,
                        parameters.kotlinVersion.get(),
                        parameters.buildMetricsService.get().buildOperationRecords,
+                        metricsToShow = buildScanSettings.metrics
                    )
                }
                log.debug("Collect data takes $collectDataDuration: $compileStatData")
@@ -46,6 +46,7 @@ data class HttpReportSettings(

 data class BuildScanSettings(
    val customValueLimit: Int,
+    val metrics: Set<String>?
 ): Serializable {
    companion object {
        const val serialVersionUID: Long = 0
@@ -6,12 +6,16 @@
 package org.jetbrains.kotlin.gradle.report

 import org.gradle.api.Project
+import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_SINGLE_FILE
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_HTTP_URL
 import org.jetbrains.kotlin.gradle.utils.isProjectIsolationEnabled
 import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
-import java.io.File
+
+
+private val availableMetrics = BuildTime.values().map { it.name } + BuildPerformanceMetric.values().map { it.name }

 internal fun reportingSettings(project: Project): ReportingSettings {
    val properties = PropertiesProvider(project)
@@ -49,7 +53,13 @@ internal fun reportingSettings(project: Project): ReportingSettings {
    }

    val buildScanSettings = if (buildReportOutputTypes.contains(BuildReportType.BUILD_SCAN)) {
-        BuildScanSettings(properties.buildReportBuildScanCustomValuesLimit)
+        val metrics = properties.buildReportBuildScanMetrics?.split(",")?.toSet()
+        metrics?.forEach {
+            if (!availableMetrics.contains(it.trim().toUpperCase())) {
+                throw IllegalStateException("Unknown metric: '$it', list of available metrics: $availableMetrics")
+            }
+        }
+        BuildScanSettings(properties.buildReportBuildScanCustomValuesLimit, metrics)
    } else {
        null
    }
@@ -32,7 +32,8 @@ internal fun prepareData(
    label: String?,
    kotlinVersion: String,
    buildOperationRecords: Collection<BuildOperationRecord>,
-    additionalTags: List<StatTag> = emptyList()
+    additionalTags: List<StatTag> = emptyList(),
+    metricsToShow: Set<String>? = null
 ): CompileStatisticsData? {
    val result = event.result
    val taskPath = event.descriptor.taskPath
@@ -56,9 +57,11 @@ internal fun prepareData(
    val buildMetrics = buildOperationRecords.firstOrNull { it.path == taskPath }?.buildMetrics

    val performanceMetrics = collectBuildPerformanceMetrics(taskExecutionResult, buildMetrics)
-    val buildTimesMetrics = collectBuildMetrics(taskExecutionResult, buildMetrics, performanceMetrics, result.startTime, System.currentTimeMillis())
+    val buildTimesMetrics = collectBuildMetrics(
+        taskExecutionResult, buildMetrics, performanceMetrics, result.startTime,
+        System.currentTimeMillis()
+    )
    val buildAttributes = collectBuildAttributes(taskExecutionResult, buildMetrics)
-
    val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
        is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
        else -> emptyList<String>()
@@ -67,13 +70,13 @@ internal fun prepareData(
        durationMs = durationMs,
        taskResult = taskResult.name,
        label = label,
-        buildTimesMetrics = buildTimesMetrics,
-        performanceMetrics = performanceMetrics,
+        buildTimesMetrics = filterMetrics(metricsToShow, buildTimesMetrics),
+        performanceMetrics = filterMetrics(metricsToShow, performanceMetrics),
        projectName = projectName,
        taskName = taskPath,
        changes = changes,
        tags = collectTags(taskExecutionResult, buildMetrics, additionalTags).map { it.name },
-        nonIncrementalAttributes = collectBuildAttributes(taskExecutionResult, buildMetrics),
+        nonIncrementalAttributes = buildAttributes,
        hostName = BuildReportsService.hostName,
        kotlinVersion = kotlinVersion,
        buildUuid = uuid,
@@ -82,6 +85,11 @@ internal fun prepareData(
    )
 }

+private fun <E : Enum<E>> filterMetrics(
+    expectedMetrics: Set<String>?,
+    buildTimesMetrics: Map<E, Long>
+): Map<E, Long> = expectedMetrics?.let { buildTimesMetrics.filterKeys { metric -> it.contains(metric.name) } } ?: buildTimesMetrics
+
 private fun collectBuildAttributes(taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics?): Set<BuildAttribute> {
    val attributes = HashSet<BuildAttribute>()
    buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys?.also { attributes.addAll(it) }
@@ -92,13 +100,12 @@ private fun collectBuildAttributes(taskExecutionResult: TaskExecutionResult?, bu

 private fun collectBuildPerformanceMetrics(
    taskExecutionResult: TaskExecutionResult?,
-    buildMetrics: BuildMetrics?,
+    buildMetrics: BuildMetrics?
 ): Map<BuildPerformanceMetric, Long> {
    val taskBuildPerformanceMetrics = HashMap<BuildPerformanceMetric, Long>()
    taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.let { taskBuildPerformanceMetrics.putAll(it) }
    buildMetrics?.buildPerformanceMetrics?.asMap()?.let { taskBuildPerformanceMetrics.putAll(it) }
-    taskBuildPerformanceMetrics.filterValues { value -> value != 0L }
-    return taskBuildPerformanceMetrics
+    return taskBuildPerformanceMetrics.filterValues { value -> value != 0L }
 }
 private fun collectBuildMetrics(
    taskExecutionResult: TaskExecutionResult?,
@@ -124,10 +131,9 @@ private fun collectBuildMetrics(
        performanceMetrics[BuildPerformanceMetric.START_WORKER_EXECUTION]?.let { startWorkerExecutionTime ->
            taskBuildMetrics.put(BuildTime.RUN_WORKER_DELAY, TimeUnit.NANOSECONDS.toMillis(startWorkerExecutionTime - callWorkerTime))
        }
-
    }
-    taskBuildMetrics.filterValues { value -> value != 0L }
-    return taskBuildMetrics
+    return taskBuildMetrics.filterValues { value -> value != 0L }
+
 }

 private fun collectTags(
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.report
+
+import org.gradle.tooling.events.OperationDescriptor
+import org.gradle.tooling.events.PluginIdentifier
+import org.gradle.tooling.events.task.TaskFinishEvent
+import org.gradle.tooling.events.task.TaskOperationDescriptor
+import org.gradle.tooling.events.task.TaskOperationResult
+import org.jetbrains.kotlin.build.report.metrics.*
+import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
+import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
+import org.junit.Test
+import java.util.concurrent.TimeUnit
+import kotlin.math.sign
+import kotlin.test.assertEquals
+import kotlin.test.assertNotNull
+import kotlin.test.assertTrue
+
+class ReportDataTest {
+    private val kotlinTaskPath = "testKotlin"
+
+    @Test
+    fun testTags() {
+        TaskExecutionResults[kotlinTaskPath] = TaskExecutionResult(
+            buildMetrics = BuildMetrics(buildAttributes = BuildAttributes()),
+            taskInfo = TaskExecutionInfo(),
+            icLogLines = emptyList()
+        )
+        val buildOperationRecord =
+            taskRecord(BuildMetrics(buildAttributes = BuildAttributes().also { it.add(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) }))
+        val statisticData = prepareData(
+            event = taskFinishEvent(),
+            projectName = "test",
+            uuid = "uuid",
+            label = "label",
+            kotlinVersion = "version",
+            buildOperationRecords = listOf(buildOperationRecord),
+            additionalTags = listOf(StatTag.KOTLIN_DEBUG)
+        )
+
+        assertNotNull(statisticData)
+        assertTrue(statisticData.tags.contains(StatTag.KOTLIN_DEBUG.name))
+        assertTrue(statisticData.tags.contains(StatTag.NON_INCREMENTAL.name))
+    }
+
+    private fun taskRecord(buildMetrics: BuildMetrics) = TaskRecord(
+        path = kotlinTaskPath,
+        classFqName = "class name",
+        startTimeMs = 10,
+        totalTimeMs = 20,
+        buildMetrics = buildMetrics,
+        didWork = true,
+        skipMessage = null,
+        icLogLines = emptyList()
+    )
+
+    @Test
+    fun testMetricFilter() {
+        TaskExecutionResults["testKotlin"] = TaskExecutionResult(
+            buildMetrics = BuildMetrics(
+                buildPerformanceMetrics = BuildPerformanceMetrics().also {
+                    it.add(BuildPerformanceMetric.BUNDLE_SIZE)
+                    it.add(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE)
+                },
+                buildTimes = BuildTimes().also {
+                    it.addTimeMs(BuildTime.RESTORE_OUTPUT_FROM_BACKUP, 10)
+                    it.addTimeMs(BuildTime.IC_ANALYZE_JAR_FILES, 10)
+                }
+            ),
+            taskInfo = TaskExecutionInfo(),
+            icLogLines = emptyList()
+        )
+        val buildOperationRecord = taskRecord(
+            BuildMetrics(
+                buildPerformanceMetrics = BuildPerformanceMetrics().also {
+                    it.add(BuildPerformanceMetric.COMPILE_ITERATION)
+                    it.add(BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT)
+                },
+                buildTimes = BuildTimes().also {
+                    it.addTimeMs(BuildTime.STORE_BUILD_INFO, 20)
+                    it.addTimeMs(BuildTime.GRADLE_TASK_ACTION, 100)
+                }
+            )
+        )
+
+        val statisticData = prepareData(
+            event = taskFinishEvent(),
+            projectName = "test",
+            uuid = "uuid",
+            label = "label",
+            kotlinVersion = "version",
+            buildOperationRecords = listOf(buildOperationRecord),
+            additionalTags = listOf(StatTag.KOTLIN_DEBUG),
+            metricsToShow = setOf(
+                BuildPerformanceMetric.BUNDLE_SIZE.name,// from TaskExecutionResult
+                BuildTime.GRADLE_TASK_ACTION.name,// from buildOperationRecord
+                BuildPerformanceMetric.COMPILE_ITERATION.name, //from buildOperationRecord
+                BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET.name, //not set
+                BuildPerformanceMetric.START_WORKER_EXECUTION.name, //not set
+                BuildTime.RESTORE_OUTPUT_FROM_BACKUP.name, //from TaskExecutionResult
+            )
+        )
+
+        assertNotNull(statisticData)
+        assertEquals(2, statisticData.performanceMetrics.size)
+        assertTrue(statisticData.performanceMetrics.containsKey(BuildPerformanceMetric.BUNDLE_SIZE))
+        assertTrue(statisticData.performanceMetrics.containsKey(BuildPerformanceMetric.COMPILE_ITERATION))
+        assertEquals(2, statisticData.buildTimesMetrics.size)
+        assertTrue(statisticData.buildTimesMetrics.containsKey(BuildTime.GRADLE_TASK_ACTION))
+        assertTrue(statisticData.buildTimesMetrics.containsKey(BuildTime.RESTORE_OUTPUT_FROM_BACKUP))
+    }
+
+    @Test
+    fun testCalculatedMetrics() {
+        val startGradleTask = 20L
+        val startTaskAction = 40L
+        val callWorker = 100L
+        val startWorker = 110L
+        val finishGradleTask = System.nanoTime()
+
+        TaskExecutionResults["testKotlin"] = TaskExecutionResult(
+            buildMetrics = BuildMetrics(
+                buildPerformanceMetrics = BuildPerformanceMetrics().also {
+                    it.add(BuildPerformanceMetric.FINISH_KOTLIN_DAEMON_EXECUTION, System.currentTimeMillis())
+                    it.add(BuildPerformanceMetric.START_WORKER_EXECUTION, TimeUnit.MILLISECONDS.toNanos(startWorker))
+                }
+            ),
+            taskInfo = TaskExecutionInfo(),
+            icLogLines = emptyList()
+        )
+        val buildOperationRecord = taskRecord(
+            BuildMetrics(
+                buildPerformanceMetrics = BuildPerformanceMetrics().also {
+                    it.add(BuildPerformanceMetric.START_TASK_ACTION_EXECUTION, startTaskAction)
+                    it.add(BuildPerformanceMetric.CALL_WORKER, TimeUnit.MILLISECONDS.toNanos(callWorker))
+                }
+            )
+        )
+
+        val statisticData = prepareData(
+            event = taskFinishEvent(startTime = startGradleTask, endTime = finishGradleTask),
+            projectName = "test",
+            uuid = "uuid",
+            label = "label",
+            kotlinVersion = "version",
+            buildOperationRecords = listOf(buildOperationRecord),
+            additionalTags = listOf(StatTag.KOTLIN_DEBUG),
+        )
+        assertNotNull(statisticData)
+        assertEquals(startTaskAction - startGradleTask, statisticData.buildTimesMetrics[BuildTime.GRADLE_TASK_PREPARATION])
+        assertEquals(1, statisticData.buildTimesMetrics[BuildTime.TASK_FINISH_LISTENER_NOTIFICATION]?.sign)
+        assertEquals(startWorker - callWorker, statisticData.buildTimesMetrics[BuildTime.RUN_WORKER_DELAY])
+    }
+
+    private fun taskFinishEvent(startTime: Long = 1L, endTime: Long =10L) = object : TaskFinishEvent {
+        override fun getEventTime(): Long = System.currentTimeMillis()
+        override fun getDisplayName(): String = "some name"
+        override fun getDescriptor(): TaskOperationDescriptor = object : TaskOperationDescriptor {
+            override fun getName(): String = "task_name"
+            override fun getDisplayName(): String = "some name"
+            override fun getParent(): OperationDescriptor? = null
+            override fun getTaskPath(): String = kotlinTaskPath
+            override fun getDependencies(): MutableSet<out OperationDescriptor> = mutableSetOf()
+            override fun getOriginPlugin(): PluginIdentifier? = null
+        }
+
+        override fun getResult(): TaskOperationResult = object : TaskOperationResult {
+            override fun getStartTime(): Long = startTime
+            override fun getEndTime(): Long = endTime
+        }
+    }
+
+}