Report build reports for http in separate executor

2022-01-21 13:27:06 +03:00
parent 7e1a9e2b9a
commit 0382610120
8 changed files with 159 additions and 120 deletions
@@ -17,7 +17,6 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
 import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
 import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener
 import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToBuildScan
-import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsByHttp
 import org.jetbrains.kotlin.gradle.report.BuildReportType
 import org.jetbrains.kotlin.gradle.report.ReportingSettings
 import org.jetbrains.kotlin.gradle.report.reportingSettings
@@ -50,7 +49,6 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
    }

    companion object {
-
        fun registerIfAbsent(project: Project): Provider<KotlinGradleBuildServices> = project.gradle.sharedServices.registerIfAbsent(
            "kotlin-build-service-${KotlinGradleBuildServices::class.java.canonicalName}_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}",
            KotlinGradleBuildServices::class.java
@@ -66,12 +64,6 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
            val listeners = project.rootProject.objects.listProperty(ReportStatistics::class.java)
                .value(listOf<ReportStatistics>())

-            reportingSettings.httpReportSettings?.let {
-                listeners.add(
-                    ReportStatisticsByHttp(reportingSettings.httpReportSettings)
-                )
-            }
-
            project.rootProject.extensions.findByName("buildScan")
                ?.also {
                    if (reportingSettings.buildReportOutputs.contains(BuildReportType.BUILD_SCAN)) {
@@ -81,7 +73,7 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui

            if (listeners.get().isNotEmpty()) {
                val listenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project)
-                val statListener = KotlinBuildStatListener(project.rootProject.name, listeners.get())
+                val statListener = KotlinBuildStatListener(project.rootProject.name, reportingSettings.buildReportLabel, listeners.get())
                listenerRegistryHolder.listenerRegistry.onTaskCompletion(project.provider { statListener })
            }
        }
@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinPm20ProjectExtension
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory
 import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
 import org.jetbrains.kotlin.gradle.report.BuildMetricsReporterService
+import org.jetbrains.kotlin.gradle.report.HttpReportService
 import org.jetbrains.kotlin.gradle.targets.js.KotlinJsCompilerAttribute
 import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin
 import org.jetbrains.kotlin.gradle.targets.js.npm.addNpmDependencyExtension
@@ -97,6 +98,9 @@ abstract class KotlinBasePluginWrapper : Plugin<Project> {

        buildMetricReporter?.also { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(it) }

+        HttpReportService.registerIfAbsent(project)
+            ?.also { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(it) }
+
        project.tasks.withType(AbstractKotlinCompile::class.java).configureEach {
            if (buildMetricReporter != null) {
                it.buildMetricsReporterService.set(buildMetricReporter)
@@ -12,7 +12,6 @@ import org.gradle.tooling.events.task.TaskFailureResult
 import org.gradle.tooling.events.task.TaskFinishEvent
 import org.gradle.tooling.events.task.TaskSkippedResult
 import org.gradle.tooling.events.task.TaskSuccessResult
-import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
 import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
 import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
 import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
@@ -32,58 +31,45 @@ enum class TaskExecutionState {
    ;
 }

-class KotlinBuildStatListener(val projectName: String, val reportStatistics: List<ReportStatistics>) :
-    OperationCompletionListener, AutoCloseable {
-
-    companion object {
-        /*
-            All listeners process events in single thread pool. After build finished it has only 60 seconds to finish processing.
-            Our listeners should not spend significant amount of time during event processing.
-        */
-        const val LIMIT_DURATION_MS = 5 * 1000
-    }
+class KotlinBuildStatListener(
+    val projectName: String,
+    val label: String?,
+    val reportStatistics: List<ReportStatistics>
+) : OperationCompletionListener, AutoCloseable {

    private val log = Logging.getLogger(this.javaClass)
    val buildUuid: String = UUID.randomUUID().toString()

-    val label by lazy { CompilerSystemProperties.KOTLIN_STAT_LABEl_PROPERTY.value }
-    val hostName: String? = try {
-        InetAddress.getLocalHost().hostName
-    } catch (_: Exception) {
-        //do nothing
-        null
-    }
+    companion object {
+        val hostName: String? = try {
+            InetAddress.getLocalHost().hostName
+        } catch (_: Exception) {
+            //do nothing
+            null
+        }

-    override fun onFinish(event: FinishEvent?) {
-        val measuredTimeMs = measureTimeMillis {
-            if (event is TaskFinishEvent) {
-                val result = event.result
-                val taskPath = event.descriptor.taskPath
-                val durationMs = result.endTime - result.startTime
-                val taskResult = when (result) {
-                    is TaskSuccessResult -> when {
-                        result.isFromCache -> TaskExecutionState.FROM_CACHE
-                        result.isUpToDate -> TaskExecutionState.UP_TO_DATE
-                        else -> TaskExecutionState.SUCCESS
-                    }
+        private fun availableForStat(taskPath: String): Boolean {
+            return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
+        }
+
+        internal fun prepareData(event: TaskFinishEvent, projectName:String, uuid: String, label: String?): CompileStatData? {
+            val result = event.result
+            val taskPath = event.descriptor.taskPath
+            val durationMs = result.endTime - result.startTime
+            val taskResult = when (result) {
+                is TaskSuccessResult -> when {
+                    result.isFromCache -> TaskExecutionState.FROM_CACHE
+                    result.isUpToDate -> TaskExecutionState.UP_TO_DATE
+                    else -> TaskExecutionState.SUCCESS
+                }

                    is TaskSkippedResult -> TaskExecutionState.SKIPPED
                    is TaskFailureResult -> TaskExecutionState.FAILED
                    else -> TaskExecutionState.UNKNOWN
                }

-                reportData(taskPath, durationMs, taskResult)
-            }
-        }
-        if (measuredTimeMs > LIMIT_DURATION_MS) {
-            log.warn("Exceed time limit for $event. Takes ${measuredTimeMs}ms ")
-        }
-    }
-
-    private fun reportData(taskPath: String, durationMs: Long, taskResult: TaskExecutionState) {
-        val (reportDataDuration, compileStatData) = measureTimeMillisWithResult {
            if (!availableForStat(taskPath)) {
-                return
+                return null
            }

            val taskExecutionResult = TaskExecutionResults[taskPath]
@@ -96,21 +82,29 @@ class KotlinBuildStatListener(val projectName: String, val reportStatistics: Lis
                else -> emptyList<String>()

            }
-            val compileStatData = CompileStatData(
+            return CompileStatData(
                durationMs = durationMs, taskResult = taskResult.name, label = label,
                buildTimesMs = buildTimesMs, perfData = perfData, projectName = projectName, taskName = taskPath, changes = changes,
                tags = taskExecutionResult?.taskInfo?.properties?.map { it.name } ?: emptyList(),
                nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap(),
-                hostName = hostName, kotlinVersion = "1.6", buildUuid = buildUuid, timeInMillis = System.currentTimeMillis()
+                hostName = hostName, kotlinVersion = "1.6", buildUuid = uuid, timeInMillis = System.currentTimeMillis()
            )
-            reportStatistics.forEach { it.report(compileStatData) }
-            compileStatData
        }
-        log.debug("Report data takes $reportDataDuration: $compileStatData")
+
    }

-    private fun availableForStat(taskPath: String): Boolean {
-        return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
+    override fun onFinish(event: FinishEvent?) {
+        if (event is TaskFinishEvent) {
+            val (collectDataDuration, compileStatData) = measureTimeMillisWithResult {
+                prepareData(event, projectName, buildUuid, label)
+            }
+            log.debug("Collect data takes $collectDataDuration: $compileStatData")
+
+            val reportDataDuration = measureTimeMillis {
+                compileStatData?.also { data -> reportStatistics.forEach { it.report(data) }}
+            }
+            log.debug("Report data takes $reportDataDuration: $compileStatData")
+        }
    }

    override fun close() {
@@ -1,62 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.plugin.statistics
-
-import com.google.gson.Gson
-import org.gradle.api.logging.Logging
-import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
-import org.jetbrains.kotlin.cli.common.toBooleanLenient
-import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
-import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
-import org.jetbrains.kotlin.gradle.report.HttpReportSettings
-import java.net.HttpURLConnection
-import java.net.URL
-import java.util.*
-import kotlin.system.measureTimeMillis
-
-class ReportStatisticsByHttp(
-    private val httpProperties: HttpReportSettings
-) : ReportStatistics {
-
-    private val log = Logging.getLogger(this.javaClass)
-
-    override fun report(data: CompileStatData) {
-        val elapsedTime = measureTimeMillis {
-            val connection = URL(httpProperties.url).openConnection() as HttpURLConnection
-
-            try {
-                if (httpProperties.user != null && httpProperties.password != null) {
-                    val auth = Base64.getEncoder()
-                        .encode("${httpProperties.user}:${httpProperties.password}".toByteArray()).toString(Charsets.UTF_8)
-                    connection.addRequestProperty("Authorization", "Basic $auth")
-                }
-                connection.addRequestProperty("Content-Type", "application/json")
-                connection.requestMethod = "POST"
-                connection.doOutput = true
-                connection.outputStream.use {
-                    it.write(Gson().toJson(data).toByteArray())
-                }
-                connection.connect()
-                checkResponseAndLog(connection)
-                connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
-            } catch (e: Exception) {
-                checkResponseAndLog(connection)
-            } finally {
-                connection.disconnect()
-            }
-        }
-        log.debug("Report statistic to elastic search takes $elapsedTime ms")
-    }
-
-    private fun checkResponseAndLog(connection: HttpURLConnection) {
-        val isResponseBad = connection.responseCode !in 200..299
-        if (isResponseBad) {
-            throw Exception(
-                "Failed to send statistic to ${connection.url}: ${connection.responseMessage}"
-            )
-        }
-    }
-}
@@ -10,6 +10,7 @@ import org.gradle.api.logging.Logging
 import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
 import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
 import org.jetbrains.kotlin.konan.file.File
+import java.util.function.Consumer
 import kotlin.system.measureTimeMillis

 class ReportStatisticsToBuildScan(
@@ -19,7 +20,7 @@ class ReportStatisticsToBuildScan(
        const val kbSize = 1024
        const val mbSize = kbSize * kbSize
        const val gbSize = kbSize * mbSize
-        const val lengthLimit = 20
+        const val lengthLimit = 100_000
    }

    private val tags = LinkedHashSet<String>()
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.report
+
+import com.google.gson.Gson
+import org.gradle.api.Project
+import org.gradle.api.logging.Logging
+import org.gradle.api.provider.Provider
+import org.gradle.api.services.BuildService
+import org.gradle.api.services.BuildServiceParameters
+import org.gradle.tooling.events.FinishEvent
+import org.gradle.tooling.events.OperationCompletionListener
+import org.gradle.tooling.events.task.TaskFinishEvent
+import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
+import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener.Companion.prepareData
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.*
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import kotlin.system.measureTimeMillis
+
+abstract class HttpReportService : BuildService<HttpReportService.Parameters>,
+    OperationCompletionListener, AutoCloseable {
+
+    var executorService: ExecutorService = Executors.newSingleThreadExecutor()
+
+    interface Parameters : BuildServiceParameters {
+        var label: String?
+        var uuid: String
+        var projectName: String
+        var httpSettings: HttpReportSettings
+    }
+
+    private val log = Logging.getLogger(this.javaClass)
+
+    override fun onFinish(event: FinishEvent?) {
+        if (event is TaskFinishEvent) {
+            val data = prepareData(event, parameters.projectName, parameters.uuid, parameters.label)
+            data?.also { executorService.submit { report(data) } }
+        }
+    }
+
+    override fun close() {
+        executorService.shutdown()
+    }
+
+    companion object {
+
+        fun registerIfAbsent(project: Project): Provider<HttpReportService>? {
+            val rootProject = project.gradle.rootProject
+            val reportingSettings = reportingSettings(rootProject)
+
+            return reportingSettings.httpReportSettings?.let { httpSettings ->
+                project.gradle.sharedServices.registerIfAbsent(
+                    "build_http_metric_service_${HttpReportService::class.java.classLoader.hashCode()}",
+                    HttpReportService::class.java
+                ) {
+                    it.parameters.label = reportingSettings.buildReportLabel
+                    it.parameters.projectName = rootProject.name
+                    it.parameters.uuid = UUID.randomUUID().toString()
+                    it.parameters.httpSettings = httpSettings
+                }!!
+            }
+
+        }
+
+    }
+
+    fun report(data: CompileStatData) {
+        val elapsedTime = measureTimeMillis {
+            val connection = URL(parameters.httpSettings.url).openConnection() as HttpURLConnection
+
+            try {
+                if (parameters.httpSettings.user != null && parameters.httpSettings.password != null) {
+                    val auth = Base64.getEncoder()
+                        .encode("${parameters.httpSettings.user}:${parameters.httpSettings.password}".toByteArray()).toString(Charsets.UTF_8)
+                    connection.addRequestProperty("Authorization", "Basic $auth")
+                }
+                connection.addRequestProperty("Content-Type", "application/json")
+                connection.requestMethod = "POST"
+                connection.doOutput = true
+                connection.outputStream.use {
+                    it.write(Gson().toJson(data).toByteArray())
+                }
+                connection.connect()
+                checkResponseAndLog(connection)
+                connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
+            } catch (e: Exception) {
+                checkResponseAndLog(connection)
+            } finally {
+                connection.disconnect()
+            }
+        }
+        log.debug("Report statistic to elastic search takes $elapsedTime ms")
+    }
+
+    private fun checkResponseAndLog(connection: HttpURLConnection) {
+        val isResponseBad = connection.responseCode !in 200..299
+        if (isResponseBad) {
+            log.warn("Failed to send statistic to ${connection.url}: ${connection.responseMessage}")
+        }
+    }
+
+}
@@ -12,6 +12,7 @@ data class ReportingSettings(
    val metricsOutputFile: File? = null,
    val buildReportOutputs: List<BuildReportType> = emptyList(),
    val buildReportMode: BuildReportMode = BuildReportMode.NONE,
+    val buildReportLabel: String? = null,
    val fileReportSettings: FileReportSettings? = null,
    val httpReportSettings: HttpReportSettings? = null
 ) : Serializable {
@@ -40,6 +40,7 @@ internal fun reportingSettings(rootProject: Project): ReportingSettings {
    return ReportingSettings(
        metricsOutputFile = metricsOutputFile,
        buildReportMode = buildReportMode,
+        buildReportLabel = properties.buildReportLabel,
        fileReportSettings = fileReportSettings,
        httpReportSettings = httpReportSettings,
        buildReportOutputs = buildReportOutputTypes