From 038261012050d35e3e4f31a3d7f183ffcdf4b6b1 Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Fri, 21 Jan 2022 13:27:06 +0300 Subject: [PATCH] Report build reports for http in separate executor --- .../plugin/KotlinGradleBuildServices.kt | 10 +- .../gradle/plugin/KotlinPluginWrapper.kt | 4 + .../statistics/KotlinBuildStatListener.kt | 90 +++++++-------- .../statistics/ReportStatisticsByHttp.kt | 62 ---------- .../statistics/ReportStatisticsToBuildScan.kt | 3 +- .../kotlin/gradle/report/HttpReportService.kt | 108 ++++++++++++++++++ .../kotlin/gradle/report/ReportingSettings.kt | 1 + .../kotlin/gradle/report/configureReporing.kt | 1 + 8 files changed, 159 insertions(+), 120 deletions(-) delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt index 5beada285d4..de388830abd 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt @@ -17,7 +17,6 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToBuildScan -import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsByHttp import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.report.ReportingSettings import org.jetbrains.kotlin.gradle.report.reportingSettings @@ -50,7 +49,6 @@ internal abstract class KotlinGradleBuildServices : BuildService = project.gradle.sharedServices.registerIfAbsent( "kotlin-build-service-${KotlinGradleBuildServices::class.java.canonicalName}_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}", KotlinGradleBuildServices::class.java @@ -66,12 +64,6 @@ internal abstract class KotlinGradleBuildServices : BuildService()) - reportingSettings.httpReportSettings?.let { - listeners.add( - ReportStatisticsByHttp(reportingSettings.httpReportSettings) - ) - } - project.rootProject.extensions.findByName("buildScan") ?.also { if (reportingSettings.buildReportOutputs.contains(BuildReportType.BUILD_SCAN)) { @@ -81,7 +73,7 @@ internal abstract class KotlinGradleBuildServices : BuildService { buildMetricReporter?.also { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(it) } + HttpReportService.registerIfAbsent(project) + ?.also { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(it) } + project.tasks.withType(AbstractKotlinCompile::class.java).configureEach { if (buildMetricReporter != null) { it.buildMetricsReporterService.set(buildMetricReporter) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt index 6094af5dfd1..b1e1799f4f6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt @@ -12,7 +12,6 @@ import org.gradle.tooling.events.task.TaskFailureResult import org.gradle.tooling.events.task.TaskFinishEvent import org.gradle.tooling.events.task.TaskSkippedResult import org.gradle.tooling.events.task.TaskSuccessResult -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics @@ -32,58 +31,45 @@ enum class TaskExecutionState { ; } -class KotlinBuildStatListener(val projectName: String, val reportStatistics: List) : - OperationCompletionListener, AutoCloseable { - - companion object { - /* - All listeners process events in single thread pool. After build finished it has only 60 seconds to finish processing. - Our listeners should not spend significant amount of time during event processing. - */ - const val LIMIT_DURATION_MS = 5 * 1000 - } +class KotlinBuildStatListener( + val projectName: String, + val label: String?, + val reportStatistics: List +) : OperationCompletionListener, AutoCloseable { private val log = Logging.getLogger(this.javaClass) val buildUuid: String = UUID.randomUUID().toString() - val label by lazy { CompilerSystemProperties.KOTLIN_STAT_LABEl_PROPERTY.value } - val hostName: String? = try { - InetAddress.getLocalHost().hostName - } catch (_: Exception) { - //do nothing - null - } + companion object { + val hostName: String? = try { + InetAddress.getLocalHost().hostName + } catch (_: Exception) { + //do nothing + null + } - override fun onFinish(event: FinishEvent?) { - val measuredTimeMs = measureTimeMillis { - if (event is TaskFinishEvent) { - val result = event.result - val taskPath = event.descriptor.taskPath - val durationMs = result.endTime - result.startTime - val taskResult = when (result) { - is TaskSuccessResult -> when { - result.isFromCache -> TaskExecutionState.FROM_CACHE - result.isUpToDate -> TaskExecutionState.UP_TO_DATE - else -> TaskExecutionState.SUCCESS - } + private fun availableForStat(taskPath: String): Boolean { + return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null) + } + + internal fun prepareData(event: TaskFinishEvent, projectName:String, uuid: String, label: String?): CompileStatData? { + val result = event.result + val taskPath = event.descriptor.taskPath + val durationMs = result.endTime - result.startTime + val taskResult = when (result) { + is TaskSuccessResult -> when { + result.isFromCache -> TaskExecutionState.FROM_CACHE + result.isUpToDate -> TaskExecutionState.UP_TO_DATE + else -> TaskExecutionState.SUCCESS + } is TaskSkippedResult -> TaskExecutionState.SKIPPED is TaskFailureResult -> TaskExecutionState.FAILED else -> TaskExecutionState.UNKNOWN } - reportData(taskPath, durationMs, taskResult) - } - } - if (measuredTimeMs > LIMIT_DURATION_MS) { - log.warn("Exceed time limit for $event. Takes ${measuredTimeMs}ms ") - } - } - - private fun reportData(taskPath: String, durationMs: Long, taskResult: TaskExecutionState) { - val (reportDataDuration, compileStatData) = measureTimeMillisWithResult { if (!availableForStat(taskPath)) { - return + return null } val taskExecutionResult = TaskExecutionResults[taskPath] @@ -96,21 +82,29 @@ class KotlinBuildStatListener(val projectName: String, val reportStatistics: Lis else -> emptyList() } - val compileStatData = CompileStatData( + return CompileStatData( durationMs = durationMs, taskResult = taskResult.name, label = label, buildTimesMs = buildTimesMs, perfData = perfData, projectName = projectName, taskName = taskPath, changes = changes, tags = taskExecutionResult?.taskInfo?.properties?.map { it.name } ?: emptyList(), nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap(), - hostName = hostName, kotlinVersion = "1.6", buildUuid = buildUuid, timeInMillis = System.currentTimeMillis() + hostName = hostName, kotlinVersion = "1.6", buildUuid = uuid, timeInMillis = System.currentTimeMillis() ) - reportStatistics.forEach { it.report(compileStatData) } - compileStatData } - log.debug("Report data takes $reportDataDuration: $compileStatData") + } - private fun availableForStat(taskPath: String): Boolean { - return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null) + override fun onFinish(event: FinishEvent?) { + if (event is TaskFinishEvent) { + val (collectDataDuration, compileStatData) = measureTimeMillisWithResult { + prepareData(event, projectName, buildUuid, label) + } + log.debug("Collect data takes $collectDataDuration: $compileStatData") + + val reportDataDuration = measureTimeMillis { + compileStatData?.also { data -> reportStatistics.forEach { it.report(data) }} + } + log.debug("Report data takes $reportDataDuration: $compileStatData") + } } override fun close() { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt deleted file mode 100644 index 89f6c16de09..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.statistics - -import com.google.gson.Gson -import org.gradle.api.logging.Logging -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties -import org.jetbrains.kotlin.cli.common.toBooleanLenient -import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics -import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData -import org.jetbrains.kotlin.gradle.report.HttpReportSettings -import java.net.HttpURLConnection -import java.net.URL -import java.util.* -import kotlin.system.measureTimeMillis - -class ReportStatisticsByHttp( - private val httpProperties: HttpReportSettings -) : ReportStatistics { - - private val log = Logging.getLogger(this.javaClass) - - override fun report(data: CompileStatData) { - val elapsedTime = measureTimeMillis { - val connection = URL(httpProperties.url).openConnection() as HttpURLConnection - - try { - if (httpProperties.user != null && httpProperties.password != null) { - val auth = Base64.getEncoder() - .encode("${httpProperties.user}:${httpProperties.password}".toByteArray()).toString(Charsets.UTF_8) - connection.addRequestProperty("Authorization", "Basic $auth") - } - connection.addRequestProperty("Content-Type", "application/json") - connection.requestMethod = "POST" - connection.doOutput = true - connection.outputStream.use { - it.write(Gson().toJson(data).toByteArray()) - } - connection.connect() - checkResponseAndLog(connection) - connection.inputStream.use { it.reader().use { reader -> reader.readText() } } - } catch (e: Exception) { - checkResponseAndLog(connection) - } finally { - connection.disconnect() - } - } - log.debug("Report statistic to elastic search takes $elapsedTime ms") - } - - private fun checkResponseAndLog(connection: HttpURLConnection) { - val isResponseBad = connection.responseCode !in 200..299 - if (isResponseBad) { - throw Exception( - "Failed to send statistic to ${connection.url}: ${connection.responseMessage}" - ) - } - } -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt index 8f7f2f395b7..60765db1995 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt @@ -10,6 +10,7 @@ import org.gradle.api.logging.Logging import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics import org.jetbrains.kotlin.konan.file.File +import java.util.function.Consumer import kotlin.system.measureTimeMillis class ReportStatisticsToBuildScan( @@ -19,7 +20,7 @@ class ReportStatisticsToBuildScan( const val kbSize = 1024 const val mbSize = kbSize * kbSize const val gbSize = kbSize * mbSize - const val lengthLimit = 20 + const val lengthLimit = 100_000 } private val tags = LinkedHashSet() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt new file mode 100644 index 00000000000..4144c967f2f --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt @@ -0,0 +1,108 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import com.google.gson.Gson +import org.gradle.api.Project +import org.gradle.api.logging.Logging +import org.gradle.api.provider.Provider +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters +import org.gradle.tooling.events.FinishEvent +import org.gradle.tooling.events.OperationCompletionListener +import org.gradle.tooling.events.task.TaskFinishEvent +import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData +import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener.Companion.prepareData +import java.net.HttpURLConnection +import java.net.URL +import java.util.* +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import kotlin.system.measureTimeMillis + +abstract class HttpReportService : BuildService, + OperationCompletionListener, AutoCloseable { + + var executorService: ExecutorService = Executors.newSingleThreadExecutor() + + interface Parameters : BuildServiceParameters { + var label: String? + var uuid: String + var projectName: String + var httpSettings: HttpReportSettings + } + + private val log = Logging.getLogger(this.javaClass) + + override fun onFinish(event: FinishEvent?) { + if (event is TaskFinishEvent) { + val data = prepareData(event, parameters.projectName, parameters.uuid, parameters.label) + data?.also { executorService.submit { report(data) } } + } + } + + override fun close() { + executorService.shutdown() + } + + companion object { + + fun registerIfAbsent(project: Project): Provider? { + val rootProject = project.gradle.rootProject + val reportingSettings = reportingSettings(rootProject) + + return reportingSettings.httpReportSettings?.let { httpSettings -> + project.gradle.sharedServices.registerIfAbsent( + "build_http_metric_service_${HttpReportService::class.java.classLoader.hashCode()}", + HttpReportService::class.java + ) { + it.parameters.label = reportingSettings.buildReportLabel + it.parameters.projectName = rootProject.name + it.parameters.uuid = UUID.randomUUID().toString() + it.parameters.httpSettings = httpSettings + }!! + } + + } + + } + + fun report(data: CompileStatData) { + val elapsedTime = measureTimeMillis { + val connection = URL(parameters.httpSettings.url).openConnection() as HttpURLConnection + + try { + if (parameters.httpSettings.user != null && parameters.httpSettings.password != null) { + val auth = Base64.getEncoder() + .encode("${parameters.httpSettings.user}:${parameters.httpSettings.password}".toByteArray()).toString(Charsets.UTF_8) + connection.addRequestProperty("Authorization", "Basic $auth") + } + connection.addRequestProperty("Content-Type", "application/json") + connection.requestMethod = "POST" + connection.doOutput = true + connection.outputStream.use { + it.write(Gson().toJson(data).toByteArray()) + } + connection.connect() + checkResponseAndLog(connection) + connection.inputStream.use { it.reader().use { reader -> reader.readText() } } + } catch (e: Exception) { + checkResponseAndLog(connection) + } finally { + connection.disconnect() + } + } + log.debug("Report statistic to elastic search takes $elapsedTime ms") + } + + private fun checkResponseAndLog(connection: HttpURLConnection) { + val isResponseBad = connection.responseCode !in 200..299 + if (isResponseBad) { + log.warn("Failed to send statistic to ${connection.url}: ${connection.responseMessage}") + } + } + +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt index 7308b68d4cd..e2521ecf443 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt @@ -12,6 +12,7 @@ data class ReportingSettings( val metricsOutputFile: File? = null, val buildReportOutputs: List = emptyList(), val buildReportMode: BuildReportMode = BuildReportMode.NONE, + val buildReportLabel: String? = null, val fileReportSettings: FileReportSettings? = null, val httpReportSettings: HttpReportSettings? = null ) : Serializable { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt index 221aee048d6..b5e9a83cb51 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt @@ -40,6 +40,7 @@ internal fun reportingSettings(rootProject: Project): ReportingSettings { return ReportingSettings( metricsOutputFile = metricsOutputFile, buildReportMode = buildReportMode, + buildReportLabel = properties.buildReportLabel, fileReportSettings = fileReportSettings, httpReportSettings = httpReportSettings, buildReportOutputs = buildReportOutputTypes