From 5fa3b85581ff79b597b0737d0bf09f3d648a59cf Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Fri, 28 Oct 2022 12:10:35 +0200 Subject: [PATCH] Update metric collection and tags for build statistics --- .../statistics/CompileStatisticsData.kt | 1 + .../gradle/report/BuildReportsService.kt | 182 ++---------------- .../kotlin/gradle/report/httpReport.kt | 68 +++++++ .../kotlin/gradle/report/reportDataUtil.kt | 112 +++++++++++ 4 files changed, 199 insertions(+), 164 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/httpReport.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt index 192e1fe7145..b00e43ef310 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt @@ -72,6 +72,7 @@ data class BuildFinishStatisticsData( val finishTime: Long, val timestamp: String = formatter.format(finishTime), val hostName: String? = "Unset", + val tags: List, ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt index ece7fb7d2ac..8929d1c35a6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.gradle.report -import com.google.gson.Gson import org.gradle.api.Project import org.gradle.api.invocation.Gradle import org.gradle.api.logging.Logging @@ -16,14 +15,10 @@ import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.OperationCompletionListener -import org.gradle.tooling.events.task.TaskFailureResult import org.gradle.tooling.events.task.TaskFinishEvent -import org.gradle.tooling.events.task.TaskSkippedResult -import org.gradle.tooling.events.task.TaskSuccessResult import org.jetbrains.kotlin.build.report.metrics.SizeMetricType import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion -import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.stat.BuildFinishStatisticsData import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters @@ -31,15 +26,9 @@ import org.jetbrains.kotlin.gradle.plugin.stat.StatTag import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData import org.jetbrains.kotlin.gradle.utils.formatSize import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable -import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.konan.file.File -import org.jetbrains.kotlin.utils.addToStdlib.ifTrue import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult -import java.io.IOException -import java.lang.management.ManagementFactory -import java.net.HttpURLConnection import java.net.InetAddress -import java.net.URL import java.text.SimpleDateFormat import java.util.* import java.util.concurrent.ExecutorService @@ -119,9 +108,10 @@ abstract class BuildReportsService : BuildService it.get().addBuildScanReport(event, BuildScanExtensionHolder(buildScanExtension)) @@ -391,99 +332,12 @@ abstract class BuildReportsService : BuildService = emptyList() - ): CompileStatisticsData? { - val result = event.result - val taskPath = event.descriptor.taskPath - val durationMs = result.endTime - result.startTime - val taskResult = when (result) { - is TaskSuccessResult -> when { - result.isFromCache -> TaskExecutionState.FROM_CACHE - result.isUpToDate -> TaskExecutionState.UP_TO_DATE - else -> TaskExecutionState.SUCCESS - } - - is TaskSkippedResult -> TaskExecutionState.SKIPPED - is TaskFailureResult -> TaskExecutionState.FAILED - else -> TaskExecutionState.UNKNOWN - } - - if (!availableForStat(taskPath)) { - return null - } - - val taskExecutionResult = TaskExecutionResults[taskPath] - val buildTimesMs = taskExecutionResult?.buildMetrics?.buildTimes?.asMapMs()?.filterValues { value -> value != 0L } ?: emptyMap() - val perfData = - taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap() - val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) { - is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } - else -> emptyList() - } - return CompileStatisticsData( - durationMs = durationMs, - taskResult = taskResult.name, - label = label, - buildTimesMetrics = buildTimesMs, - performanceMetrics = perfData, - projectName = projectName, - taskName = taskPath, - changes = changes, - tags = parseTags(taskExecutionResult, additionalTags).map { it.name }, - nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys - ?: emptySet(), - hostName = hostName, - kotlinVersion = kotlinVersion, - buildUuid = uuid, - finishTime = System.currentTimeMillis(), - compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList() - ) - } - val hostName: String? = try { InetAddress.getLocalHost().hostName } catch (_: Exception) { //do nothing null } - - private fun availableForStat(taskPath: String): Boolean { - return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null) - } - - private fun parseTags(taskExecutionResult: TaskExecutionResult?, additionalTags: List): List { - val tags = ArrayList(additionalTags) - - val nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap() - - if (nonIncrementalAttributes.isEmpty()) { - tags.add(StatTag.INCREMENTAL) - } else { - tags.add(StatTag.NON_INCREMENTAL) - } - - val taskInfo = taskExecutionResult?.taskInfo - - taskInfo?.withAbiSnapshot?.ifTrue { - tags.add(StatTag.ABI_SNAPSHOT) - } - taskInfo?.withArtifactTransform?.ifTrue { - tags.add(StatTag.ARTIFACT_TRANSFORM) - } - - val debugConfiguration = "-agentlib:" - if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) { - tags.add(StatTag.GRADLE_DEBUG) - } - return tags - } - } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/httpReport.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/httpReport.kt new file mode 100644 index 00000000000..7e999bb46d9 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/httpReport.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import com.google.gson.Gson +import org.gradle.api.logging.Logger +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.util.* +import kotlin.system.measureTimeMillis + +private var invalidUrl = false +private var requestPreviousFailed = false + +internal fun sendDataViaHttp(httpSettings: HttpReportSettings, data: Any, log: Logger) { + val elapsedTime = measureTimeMillis { + if (invalidUrl) { + return + } + val connection = try { + URL(httpSettings.url).openConnection() as HttpURLConnection + } catch (e: IOException) { + log.warn("Unable to open connection to ${httpSettings.url}: ${e.message}") + invalidUrl = true + return + } + + try { + if (httpSettings.user != null && httpSettings.password != null) { + val auth = Base64.getEncoder() + .encode("${httpSettings.user}:${httpSettings.password}".toByteArray()) + .toString(Charsets.UTF_8) + connection.addRequestProperty("Authorization", "Basic $auth") + } + connection.addRequestProperty("Content-Type", "application/json") + connection.requestMethod = "POST" + connection.doOutput = true + connection.outputStream.use { + it.write(Gson().toJson(data).toByteArray()) + } + connection.connect() + checkResponseAndLog(connection, log) + } catch (e: Exception) { + log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}") + checkResponseAndLog(connection, log) + } finally { + connection.disconnect() + } + } + log.debug("Report statistic by http takes $elapsedTime ms") +} + +private fun checkResponseAndLog(connection: HttpURLConnection, log: Logger) { + val isResponseBad = connection.responseCode !in 200..299 + if (isResponseBad) { + val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}" + if (!requestPreviousFailed) { + log.warn(message) + } else { + log.debug(message) + } + requestPreviousFailed = true + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt new file mode 100644 index 00000000000..193d5874b56 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt @@ -0,0 +1,112 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import org.gradle.tooling.events.task.TaskFailureResult +import org.gradle.tooling.events.task.TaskFinishEvent +import org.gradle.tooling.events.task.TaskSkippedResult +import org.gradle.tooling.events.task.TaskSuccessResult +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics +import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults +import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData +import org.jetbrains.kotlin.gradle.plugin.stat.StatTag +import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord +import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.utils.addToStdlib.ifTrue +import java.lang.management.ManagementFactory +import java.util.ArrayList + + +private fun availableForStat(taskPath: String): Boolean { + return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null) +} + +internal fun prepareData( + event: TaskFinishEvent, + projectName: String, + uuid: String, + label: String?, + kotlinVersion: String, + buildOperationRecords: Collection, + additionalTags: List = emptyList() +): CompileStatisticsData? { + val result = event.result + val taskPath = event.descriptor.taskPath + val durationMs = result.endTime - result.startTime + val taskResult = when (result) { + is TaskSuccessResult -> when { + result.isFromCache -> TaskExecutionState.FROM_CACHE + result.isUpToDate -> TaskExecutionState.UP_TO_DATE + else -> TaskExecutionState.SUCCESS + } + + is TaskSkippedResult -> TaskExecutionState.SKIPPED + is TaskFailureResult -> TaskExecutionState.FAILED + else -> TaskExecutionState.UNKNOWN + } + + if (!availableForStat(taskPath)) { + return null + } + val buildMetrics = buildOperationRecords.firstOrNull { it.path == taskPath }?.buildMetrics + val taskExecutionResult = TaskExecutionResults[taskPath] + val buildTimesMs = buildMetrics?.buildTimes?.asMapMs()?.filterValues { value -> value != 0L } ?: emptyMap() + val perfData = buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap() + val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) { + is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } + else -> emptyList() + } + return CompileStatisticsData( + durationMs = durationMs, + taskResult = taskResult.name, + label = label, + buildTimesMetrics = buildTimesMs, + performanceMetrics = perfData, + projectName = projectName, + taskName = taskPath, + changes = changes, + tags = parseTags(taskExecutionResult, buildMetrics, additionalTags).map { it.name }, + nonIncrementalAttributes = buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys + ?: emptySet(), + hostName = BuildReportsService.hostName, + kotlinVersion = kotlinVersion, + buildUuid = uuid, + finishTime = System.currentTimeMillis(), + compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList() + ) +} + +private fun parseTags(taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics?, additionalTags: List): List { + val tags = parseTags(taskExecutionResult, additionalTags) + val nonIncrementalAttributes = buildMetrics?.buildAttributes?.asMap() ?: emptyMap() + if (nonIncrementalAttributes.isEmpty()) { + tags.add(StatTag.INCREMENTAL) + } else { + tags.add(StatTag.NON_INCREMENTAL) + } + return tags +} + +private fun parseTags( + taskExecutionResult: TaskExecutionResult?, + additionalTags: List, +): MutableList{ + val tags = ArrayList(additionalTags) + val taskInfo = taskExecutionResult?.taskInfo + + taskInfo?.withAbiSnapshot?.ifTrue { + tags.add(StatTag.ABI_SNAPSHOT) + } + taskInfo?.withArtifactTransform?.ifTrue { + tags.add(StatTag.ARTIFACT_TRANSFORM) + } + + val debugConfiguration = "-agentlib:" + if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) { + tags.add(StatTag.GRADLE_DEBUG) + } + return tags +} \ No newline at end of file