Update metric collection and tags for build statistics

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt

@@ -72,6 +72,7 @@ data class BuildFinishStatisticsData(
     val finishTime: Long,
     val timestamp: String = formatter.format(finishTime),
     val hostName: String? = "Unset",
+    val tags: List<String>,
 )
 
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.gradle.report
 
-import com.google.gson.Gson
 import org.gradle.api.Project
 import org.gradle.api.invocation.Gradle
 import org.gradle.api.logging.Logging
@@ -16,14 +15,10 @@ import org.gradle.api.services.BuildService
 import org.gradle.api.services.BuildServiceParameters
 import org.gradle.tooling.events.FinishEvent
 import org.gradle.tooling.events.OperationCompletionListener
-import org.gradle.tooling.events.task.TaskFailureResult
 import org.gradle.tooling.events.task.TaskFinishEvent
-import org.gradle.tooling.events.task.TaskSkippedResult
-import org.gradle.tooling.events.task.TaskSuccessResult
 import org.jetbrains.kotlin.build.report.metrics.SizeMetricType
 import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
 import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
-import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
 import org.jetbrains.kotlin.gradle.plugin.stat.BuildFinishStatisticsData
 import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
 import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters
@@ -31,15 +26,9 @@ import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
 import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
 import org.jetbrains.kotlin.gradle.utils.formatSize
 import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
-import org.jetbrains.kotlin.incremental.ChangedFiles
 import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
 import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult
-import java.io.IOException
-import java.lang.management.ManagementFactory
-import java.net.HttpURLConnection
 import java.net.InetAddress
-import java.net.URL
 import java.text.SimpleDateFormat
 import java.util.*
 import java.util.concurrent.ExecutorService
@@ -119,9 +108,10 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
             label = parameters.label.orNull,
             totalTime = (System.nanoTime() - startTime) / 1_000_000,
             finishTime = System.currentTimeMillis(),
-            hostName = hostName
+            hostName = hostName,
+            tags = tags.toList()
         )
-        sendDataViaHttp(buildFinishData)
+        parameters.reportingSettings.get().httpReportSettings?.also { sendDataViaHttp(it, buildFinishData, log) }
     }
 
     private fun GradleBuildStartParameters.includeVerboseEnvironment(verboseEnvironment: Boolean): GradleBuildStartParameters {
@@ -148,76 +138,26 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
                         buildUuid,
                         parameters.label.orNull,
                         parameters.kotlinVersion.get(),
+                        parameters.buildMetricsService.get().buildOperationRecords,
                         parameters.additionalTags.get()
                     )
-                data?.also { executorService.submit { sendDataViaHttp(data) } }
+                data?.also {  parameters.reportingSettings.get().httpReportSettings?.also {  executorService.submit { sendDataViaHttp(it, data, log) } } }
             }
         }
 
     }
 
-    private var invalidUrl = false
-    private var requestPreviousFailed = false
-    private fun sendDataViaHttp(data: Any) {
-        val httpSettings = parameters.reportingSettings.get().httpReportSettings ?: return
-
-        val elapsedTime = measureTimeMillis {
-            if (invalidUrl) {
-                return
-            }
-            val connection = try {
-                URL(httpSettings.url).openConnection() as HttpURLConnection
-            } catch (e: IOException) {
-                log.warn("Unable to open connection to ${httpSettings.url}: ${e.message}")
-                invalidUrl = true
-                return
-            }
-
-            try {
-                if (httpSettings.user != null && httpSettings.password != null) {
-                    val auth = Base64.getEncoder()
-                        .encode("${httpSettings.user}:${httpSettings.password}".toByteArray())
-                        .toString(Charsets.UTF_8)
-                    connection.addRequestProperty("Authorization", "Basic $auth")
-                }
-                connection.addRequestProperty("Content-Type", "application/json")
-                connection.requestMethod = "POST"
-                connection.doOutput = true
-                connection.outputStream.use {
-                    it.write(Gson().toJson(data).toByteArray())
-                }
-                connection.connect()
-                checkResponseAndLog(connection)
-            } catch (e: Exception) {
-                log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}")
-                checkResponseAndLog(connection)
-            } finally {
-                connection.disconnect()
-            }
-        }
-        log.debug("Report statistic by http takes $elapsedTime ms")
-    }
-
-    private fun checkResponseAndLog(connection: HttpURLConnection) {
-        val isResponseBad = connection.responseCode !in 200..299
-        if (isResponseBad) {
-            val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}"
-            if (!requestPreviousFailed) {
-                log.warn(message)
-            } else {
-                log.debug(message)
-            }
-            requestPreviousFailed = true
-        }
-    }
-
 
     private fun addBuildScanReport(event: FinishEvent?, buildScan: BuildScanExtensionHolder) {
         val buildScanSettings = parameters.reportingSettings.orNull?.buildScanReportSettings
         if (buildScanSettings != null && buildScan.buildScan != null) {
             if (event is TaskFinishEvent) {
                 val (collectDataDuration, compileStatData) = measureTimeMillisWithResult {
-                    prepareData(event, parameters.projectName.get(), buildUuid, parameters.label.orNull, parameters.kotlinVersion.get())
+                    prepareData(
+                        event, parameters.projectName.get(), buildUuid, parameters.label.orNull,
+                        parameters.kotlinVersion.get(),
+                        parameters.buildMetricsService.get().buildOperationRecords,
+                    )
                 }
                 log.debug("Collect data takes $collectDataDuration: $compileStatData")
 
@@ -231,12 +171,6 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
     private fun addBuildScanReport(data: CompileStatisticsData, customValuesLimit: Int, buildScan: BuildScanExtensionHolder) {
         val elapsedTime = measureTimeMillis {
             buildScan.buildScan?.also {
-                if (!tags.contains(buildUuid)) {
-                    addBuildScanTag(buildScan, buildUuid)
-                }
-                data.label?.takeIf { !tags.contains(it) }?.also {
-                    addBuildScanTag(buildScan, it)
-                }
                 data.tags
                     .filter { !tags.contains(it) }
                     .forEach {
@@ -323,9 +257,15 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
         return splattedString
     }
 
+    private fun initBuildScanTags(buildScan: BuildScanExtensionHolder) {
+        buildScan.buildScan?.tag(buildUuid)
+        parameters.label.orNull?.also {
+            buildScan.buildScan?.tag(it)
+        }
+    }
+
     companion object {
 
-        private val log = Logging.getLogger(this.javaClass)
         const val lengthLimit = 100_000
 
         fun getStartParameters(project: Project) = project.gradle.startParameter.let {
@@ -371,6 +311,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
 
                 val buildScanExtension = project.rootProject.extensions.findByName("buildScan")
                 if (reportingSettings.buildScanReportSettings != null && buildScanExtension != null) {
+                    it.get().initBuildScanTags(BuildScanExtensionHolder(buildScanExtension))
                     BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(project.provider {
                         OperationCompletionListener { event ->
                             it.get().addBuildScanReport(event, BuildScanExtensionHolder(buildScanExtension))
@@ -391,99 +332,12 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
             return additionalTags
         }
 
-        internal fun prepareData(
-            event: TaskFinishEvent,
-            projectName: String,
-            uuid: String,
-            label: String?,
-            kotlinVersion: String,
-            additionalTags: List<StatTag> = emptyList()
-        ): CompileStatisticsData? {
-            val result = event.result
-            val taskPath = event.descriptor.taskPath
-            val durationMs = result.endTime - result.startTime
-            val taskResult = when (result) {
-                is TaskSuccessResult -> when {
-                    result.isFromCache -> TaskExecutionState.FROM_CACHE
-                    result.isUpToDate -> TaskExecutionState.UP_TO_DATE
-                    else -> TaskExecutionState.SUCCESS
-                }
-
-                is TaskSkippedResult -> TaskExecutionState.SKIPPED
-                is TaskFailureResult -> TaskExecutionState.FAILED
-                else -> TaskExecutionState.UNKNOWN
-            }
-
-            if (!availableForStat(taskPath)) {
-                return null
-            }
-
-            val taskExecutionResult = TaskExecutionResults[taskPath]
-            val buildTimesMs = taskExecutionResult?.buildMetrics?.buildTimes?.asMapMs()?.filterValues { value -> value != 0L } ?: emptyMap()
-            val perfData =
-                taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
-            val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
-                is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
-                else -> emptyList<String>()
-            }
-            return CompileStatisticsData(
-                durationMs = durationMs,
-                taskResult = taskResult.name,
-                label = label,
-                buildTimesMetrics = buildTimesMs,
-                performanceMetrics = perfData,
-                projectName = projectName,
-                taskName = taskPath,
-                changes = changes,
-                tags = parseTags(taskExecutionResult, additionalTags).map { it.name },
-                nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys
-                    ?: emptySet(),
-                hostName = hostName,
-                kotlinVersion = kotlinVersion,
-                buildUuid = uuid,
-                finishTime = System.currentTimeMillis(),
-                compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList()
-            )
-        }
-
         val hostName: String? = try {
             InetAddress.getLocalHost().hostName
         } catch (_: Exception) {
             //do nothing
             null
         }
-
-        private fun availableForStat(taskPath: String): Boolean {
-            return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
-        }
-
-        private fun parseTags(taskExecutionResult: TaskExecutionResult?, additionalTags: List<StatTag>): List<StatTag> {
-            val tags = ArrayList(additionalTags)
-
-            val nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap()
-
-            if (nonIncrementalAttributes.isEmpty()) {
-                tags.add(StatTag.INCREMENTAL)
-            } else {
-                tags.add(StatTag.NON_INCREMENTAL)
-            }
-
-            val taskInfo = taskExecutionResult?.taskInfo
-
-            taskInfo?.withAbiSnapshot?.ifTrue {
-                tags.add(StatTag.ABI_SNAPSHOT)
-            }
-            taskInfo?.withArtifactTransform?.ifTrue {
-                tags.add(StatTag.ARTIFACT_TRANSFORM)
-            }
-
-            val debugConfiguration = "-agentlib:"
-            if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) {
-                tags.add(StatTag.GRADLE_DEBUG)
-            }
-            return tags
-        }
-
     }
 
 }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/httpReport.kt

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.report
+
+import com.google.gson.Gson
+import org.gradle.api.logging.Logger
+import java.io.IOException
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.*
+import kotlin.system.measureTimeMillis
+
+private var invalidUrl = false
+private var requestPreviousFailed = false
+
+internal fun sendDataViaHttp(httpSettings: HttpReportSettings, data: Any, log: Logger) {
+    val elapsedTime = measureTimeMillis {
+        if (invalidUrl) {
+            return
+        }
+        val connection = try {
+            URL(httpSettings.url).openConnection() as HttpURLConnection
+        } catch (e: IOException) {
+            log.warn("Unable to open connection to ${httpSettings.url}: ${e.message}")
+            invalidUrl = true
+            return
+        }
+
+        try {
+            if (httpSettings.user != null && httpSettings.password != null) {
+                val auth = Base64.getEncoder()
+                    .encode("${httpSettings.user}:${httpSettings.password}".toByteArray())
+                    .toString(Charsets.UTF_8)
+                connection.addRequestProperty("Authorization", "Basic $auth")
+            }
+            connection.addRequestProperty("Content-Type", "application/json")
+            connection.requestMethod = "POST"
+            connection.doOutput = true
+            connection.outputStream.use {
+                it.write(Gson().toJson(data).toByteArray())
+            }
+            connection.connect()
+            checkResponseAndLog(connection, log)
+        } catch (e: Exception) {
+            log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}")
+            checkResponseAndLog(connection, log)
+        } finally {
+            connection.disconnect()
+        }
+    }
+    log.debug("Report statistic by http takes $elapsedTime ms")
+}
+
+private fun checkResponseAndLog(connection: HttpURLConnection, log: Logger) {
+    val isResponseBad = connection.responseCode !in 200..299
+    if (isResponseBad) {
+        val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}"
+        if (!requestPreviousFailed) {
+            log.warn(message)
+        } else {
+            log.debug(message)
+        }
+        requestPreviousFailed = true
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.report
+
+import org.gradle.tooling.events.task.TaskFailureResult
+import org.gradle.tooling.events.task.TaskFinishEvent
+import org.gradle.tooling.events.task.TaskSkippedResult
+import org.gradle.tooling.events.task.TaskSuccessResult
+import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
+import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
+import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
+import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
+import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
+import org.jetbrains.kotlin.incremental.ChangedFiles
+import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
+import java.lang.management.ManagementFactory
+import java.util.ArrayList
+
+
+private fun availableForStat(taskPath: String): Boolean {
+    return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
+}
+
+internal fun prepareData(
+    event: TaskFinishEvent,
+    projectName: String,
+    uuid: String,
+    label: String?,
+    kotlinVersion: String,
+    buildOperationRecords: Collection<BuildOperationRecord>,
+    additionalTags: List<StatTag> = emptyList()
+): CompileStatisticsData? {
+    val result = event.result
+    val taskPath = event.descriptor.taskPath
+    val durationMs = result.endTime - result.startTime
+    val taskResult = when (result) {
+        is TaskSuccessResult -> when {
+            result.isFromCache -> TaskExecutionState.FROM_CACHE
+            result.isUpToDate -> TaskExecutionState.UP_TO_DATE
+            else -> TaskExecutionState.SUCCESS
+        }
+
+        is TaskSkippedResult -> TaskExecutionState.SKIPPED
+        is TaskFailureResult -> TaskExecutionState.FAILED
+        else -> TaskExecutionState.UNKNOWN
+    }
+
+    if (!availableForStat(taskPath)) {
+        return null
+    }
+    val buildMetrics = buildOperationRecords.firstOrNull { it.path == taskPath }?.buildMetrics
+    val taskExecutionResult = TaskExecutionResults[taskPath]
+    val buildTimesMs = buildMetrics?.buildTimes?.asMapMs()?.filterValues { value -> value != 0L } ?: emptyMap()
+    val perfData = buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
+    val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
+        is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
+        else -> emptyList<String>()
+    }
+    return CompileStatisticsData(
+        durationMs = durationMs,
+        taskResult = taskResult.name,
+        label = label,
+        buildTimesMetrics = buildTimesMs,
+        performanceMetrics = perfData,
+        projectName = projectName,
+        taskName = taskPath,
+        changes = changes,
+        tags = parseTags(taskExecutionResult, buildMetrics, additionalTags).map { it.name },
+        nonIncrementalAttributes = buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys
+            ?: emptySet(),
+        hostName = BuildReportsService.hostName,
+        kotlinVersion = kotlinVersion,
+        buildUuid = uuid,
+        finishTime = System.currentTimeMillis(),
+        compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList()
+    )
+}
+
+private fun parseTags(taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics?, additionalTags: List<StatTag>): List<StatTag> {
+    val tags = parseTags(taskExecutionResult, additionalTags)
+    val nonIncrementalAttributes = buildMetrics?.buildAttributes?.asMap() ?: emptyMap()
+    if (nonIncrementalAttributes.isEmpty()) {
+        tags.add(StatTag.INCREMENTAL)
+    } else {
+        tags.add(StatTag.NON_INCREMENTAL)
+    }
+    return tags
+}
+
+private fun parseTags(
+    taskExecutionResult: TaskExecutionResult?,
+    additionalTags: List<StatTag>,
+): MutableList<StatTag>{
+    val tags = ArrayList(additionalTags)
+    val taskInfo = taskExecutionResult?.taskInfo
+
+    taskInfo?.withAbiSnapshot?.ifTrue {
+        tags.add(StatTag.ABI_SNAPSHOT)
+    }
+    taskInfo?.withArtifactTransform?.ifTrue {
+        tags.add(StatTag.ARTIFACT_TRANSFORM)
+    }
+
+    val debugConfiguration = "-agentlib:"
+    if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) {
+        tags.add(StatTag.GRADLE_DEBUG)
+    }
+    return tags
+}