Add listener's metrics

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt

@@ -15,7 +15,7 @@ import org.jetbrains.kotlin.gradle.logging.kotlinDebug
 import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
 import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
 import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
-import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildEsStatListener
+import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener
 import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToBuildScan
 import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToElasticSearch
 import java.io.File
@@ -63,10 +63,10 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
                     val listeners = project.rootProject.objects.listProperty(ReportStatistics::class.java)
                         .value(listOf<ReportStatistics>(ReportStatisticsToElasticSearch))
                     listeners.add(ReportStatisticsToBuildScan(it as BuildScanExtension))
-                    val esStatListener = KotlinBuildEsStatListener(project.rootProject.name, listeners.get())
+                    val statListener = KotlinBuildStatListener(project.rootProject.name, listeners.get())
                     val listenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project)
 
-                    listenerRegistryHolder.listenerRegistry.onTaskCompletion(project.provider { esStatListener })
+                    listenerRegistryHolder.listenerRegistry.onTaskCompletion(project.provider { statListener })
                 }
         }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildEsStatListener.kt

@@ -1,116 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.plugin.statistics
-
-import org.gradle.api.Task
-import org.gradle.api.execution.TaskExecutionListener
-import org.gradle.api.tasks.TaskState
-import org.gradle.tooling.events.FinishEvent
-import org.gradle.tooling.events.OperationCompletionListener
-import org.gradle.tooling.events.task.*
-import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
-import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
-import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
-import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
-import org.jetbrains.kotlin.incremental.ChangedFiles
-import java.net.InetAddress
-import java.util.*
-
-enum class TaskExecutionState {
-    SKIPPED,
-    FAILED,
-    UNKNOWN,
-    SUCCESS,
-    FROM_CACHE,
-    UP_TO_DATE
-    ;
-}
-
-class KotlinBuildEsStatListener(val projectName: String, val reportStatistics: List<ReportStatistics>) :
-    OperationCompletionListener, AutoCloseable, TaskExecutionListener {
-
-    val buildUuid: String = UUID.randomUUID().toString()
-
-    val label by lazy { CompilerSystemProperties.KOTLIN_STAT_LABEl_PROPERTY.value }
-    val hostName: String? by lazy {
-        try {
-            InetAddress.getLocalHost().hostName
-        } catch (_: Exception) {
-            //do nothing
-            null
-        }
-    }
-
-    override fun onFinish(event: FinishEvent?) {
-        if (event is TaskFinishEvent) {
-            val result = event.result
-            val taskPath = event.descriptor.taskPath
-            val duration = event.result.endTime - event.result.startTime
-            val taskResult = when (result) {
-                is TaskSuccessResult -> when {
-                    result.isFromCache -> TaskExecutionState.FROM_CACHE
-                    result.isUpToDate -> TaskExecutionState.UP_TO_DATE
-                    else -> TaskExecutionState.SUCCESS
-                }
-
-                is TaskSkippedResult -> TaskExecutionState.SKIPPED
-                is TaskFailureResult -> TaskExecutionState.FAILED
-                else -> TaskExecutionState.UNKNOWN
-            }
-
-            reportData(taskPath, duration, taskResult)
-        }
-    }
-
-    private fun reportData(taskPath: String, duration: Long, taskResult: TaskExecutionState) {
-        if (!availableForStat(taskPath)) {
-            return;
-        }
-        val taskExecutionResult = TaskExecutionResults[taskPath]
-        val timeData = taskExecutionResult?.buildMetrics?.buildTimes?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
-        val perfData = taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
-        val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
-            is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
-            is ChangedFiles.Dependencies -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
-            else -> emptyList<String>()
-
-        }
-        val compileStatData = CompileStatData(
-            duration = duration, taskResult = taskResult.name, label = label,
-            timeData = timeData, perfData = perfData, projectName = projectName, taskName = taskPath, changes = changes,
-            tags = taskExecutionResult?.taskInfo?.properties?.map { it.name } ?: emptyList(),
-            nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap(),
-            hostName = hostName, kotlinVersion = "1.6", buildUuid = buildUuid, timeInMillis = System.currentTimeMillis()
-        )
-        reportStatistics.forEach { it.report(compileStatData) }
-    }
-
-    private fun availableForStat(taskPath: String): Boolean {
-        return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
-    }
-
-    override fun afterExecute(task: Task, taskState: TaskState) {
-        val taskResult = when {
-            taskState.skipped -> TaskExecutionState.SKIPPED
-            taskState.failure != null -> TaskExecutionState.FAILED
-            taskState.upToDate -> TaskExecutionState.UP_TO_DATE
-            taskState.didWork -> TaskExecutionState.SUCCESS
-            taskState.executed -> TaskExecutionState.FROM_CACHE
-            else -> TaskExecutionState.UNKNOWN
-        }
-        val taskPath = task.path
-        reportData(taskPath, 0L, taskResult)
-    }
-
-    override fun close() {
-    }
-
-    override fun beforeExecute(p0: Task) {
-        //Do nothing
-    }
-
-}
-

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt

@@ -1,66 +1,119 @@
 /*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
 package org.jetbrains.kotlin.gradle.plugin.statistics
 
-import org.gradle.api.invocation.Gradle
+import org.gradle.api.logging.Logging
 import org.gradle.tooling.events.FinishEvent
 import org.gradle.tooling.events.OperationCompletionListener
-import org.gradle.tooling.events.configuration.ProjectConfigurationFinishEvent
+import org.gradle.tooling.events.task.TaskFailureResult
-import org.jetbrains.kotlin.statistics.BuildSessionLogger
+import org.gradle.tooling.events.task.TaskFinishEvent
-import java.lang.management.ManagementFactory
+import org.gradle.tooling.events.task.TaskSkippedResult
-import javax.management.MBeanServer
+import org.gradle.tooling.events.task.TaskSuccessResult
-import javax.management.ObjectName
+import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
+import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
+import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
+import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
+import org.jetbrains.kotlin.incremental.ChangedFiles
+import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult
+import java.net.InetAddress
+import java.util.*
+import kotlin.system.measureTimeMillis
 
-open class KotlinBuildStatListener(val beanName: ObjectName/*, val gradle: Gradle*/) : OperationCompletionListener, AutoCloseable {
+enum class TaskExecutionState {
+    SKIPPED,
+    FAILED,
+    UNKNOWN,
+    SUCCESS,
+    FROM_CACHE,
+    UP_TO_DATE
+    ;
+}
 
-    private var projectEvaluatedTime: Long? = null
+class KotlinBuildStatListener(val projectName: String, val reportStatistics: List<ReportStatistics>) :
+    OperationCompletionListener, AutoCloseable {
+
+    companion object {
+        /*
+            All listeners process events in single thread pool. After build finished it has only 60 seconds to finish processing.
+            Our listeners should not spend significant amount of time during event processing.
+        */
+        const val LIMIT_DURATION_MS = 5 * 1000
+    }
+
+    private val log = Logging.getLogger(this.javaClass)
+    val buildUuid: String = UUID.randomUUID().toString()
+
+    val label by lazy { CompilerSystemProperties.KOTLIN_STAT_LABEl_PROPERTY.value }
+    val hostName: String? = try {
+        InetAddress.getLocalHost().hostName
+    } catch (_: Exception) {
+        //do nothing
+        null
+    }
 
     override fun onFinish(event: FinishEvent?) {
-        if (event is ProjectConfigurationFinishEvent) {
+        val measuredTimeMs = measureTimeMillis {
-            projectEvaluatedTime = event.eventTime
+            if (event is TaskFinishEvent) {
-        }
+                val result = event.result
-        //todo is it any chance to get failure exception?
+                val taskPath = event.descriptor.taskPath
-        //todo nothing to do?
+                val duration = result.endTime - result.startTime
-//        KotlinBuildStatHandler.runSafe("${KotlinBuildStatListener::class.java}.onFinish") {
+                val taskResult = when (result) {
-//
+                    is TaskSuccessResult -> when {
-//
+                        result.isFromCache -> TaskExecutionState.FROM_CACHE
-//            try {
+                        result.isUpToDate -> TaskExecutionState.UP_TO_DATE
-//                val finishTime = event?.result?.endTime
+                        else -> TaskExecutionState.SUCCESS
-//                val startTime = event?.result?.startTime
+                    }
-//                    report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
-//                    report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
-//                    report(NumericalMetrics.BUILD_FINISH_TIME, finishTime)
-//            } finally {
-//                val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
-//                if (mbs.isRegistered(beanName)) {
-//                    mbs.unregisterMBean(beanName)
-//                }
-//            }
-//        }
 
+                    is TaskSkippedResult -> TaskExecutionState.SKIPPED
+                    is TaskFailureResult -> TaskExecutionState.FAILED
+                    else -> TaskExecutionState.UNKNOWN
+                }
+
+                reportData(taskPath, duration, taskResult)
+            }
+        }
+        if (measuredTimeMs > LIMIT_DURATION_MS) {
+            log.warn("Exceed time limit for $event. Takes ${measuredTimeMs}ms ")
+        }
+    }
+
+    private fun reportData(taskPath: String, duration: Long, taskResult: TaskExecutionState) {
+        val (reportDataDuration, compileStatData) = measureTimeMillisWithResult {
+            if (!availableForStat(taskPath)) {
+                return
+            }
+
+            val taskExecutionResult = TaskExecutionResults[taskPath]
+            val timeData = taskExecutionResult?.buildMetrics?.buildTimes?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
+            val perfData = taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
+            val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
+                is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
+                is ChangedFiles.Dependencies -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
+                else -> emptyList<String>()
+
+            }
+            val compileStatData = CompileStatData(
+                duration = duration, taskResult = taskResult.name, label = label,
+                timeData = timeData, perfData = perfData, projectName = projectName, taskName = taskPath, changes = changes,
+                tags = taskExecutionResult?.taskInfo?.properties?.map { it.name } ?: emptyList(),
+                nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap(),
+                hostName = hostName, kotlinVersion = "1.6", buildUuid = buildUuid, timeInMillis = System.currentTimeMillis()
+            )
+            reportStatistics.forEach { it.report(compileStatData) }
+            compileStatData
+        }
+        log.debug("Report data takes $reportDataDuration: $compileStatData")
+    }
+
+    private fun availableForStat(taskPath: String): Boolean {
+        return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
     }
 
     override fun close() {
-//        val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir)
-//        KotlinBuildStatHandler.runSafe("${KotlinBuildStatListener::class.java}.close()") {
-//            try {
-//                try {
-//                    KotlinBuildStatHandler().reportGlobalMetrics(gradle, sessionLogger)
-//                } finally {
-////                    report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
-////                    report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
-////                    report(NumericalMetrics.BUILD_FINISH_TIME, finishTime)
-//                }
-//
-//            } finally {
-//                val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
-//                if (mbs.isRegistered(beanName)) {
-//                    mbs.unregisterMBean(beanName)
-//                }
-//            }
-//        }
     }
-}
+
+}
+

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt

@@ -105,7 +105,6 @@ internal abstract class KotlinBuildStatsService internal constructor() : BuildAd
                             )
                             instance = JMXKotlinBuildStatsService(mbs, beanName)
                         } else {
-                            project.provider { KotlinBuildStatListener(beanName) }
                             val newInstance = DefaultKotlinBuildStatsService(gradle, beanName)
 
                             instance = newInstance

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt

@@ -6,9 +6,11 @@
 package org.jetbrains.kotlin.gradle.plugin.statistics
 
 import com.gradle.scan.plugin.BuildScanExtension
+import org.gradle.api.logging.Logging
 import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
 import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
 import org.jetbrains.kotlin.konan.file.File
+import kotlin.system.measureTimeMillis
 
 class ReportStatisticsToBuildScan(
     private val buildScan: BuildScanExtension
@@ -19,9 +21,21 @@ class ReportStatisticsToBuildScan(
         const val gbSize = kbSize * mbSize
     }
 
+    private val tags = LinkedHashSet<String>()
+    private val log = Logging.getLogger(this.javaClass)
+
     override fun report(data: CompileStatData) {
-        buildScan.value(data.taskName, readableString(data))
+        val elapsedTime = measureTimeMillis {
-        data.tags.forEach { buildScan.tag(it) }
+            buildScan.value(data.taskName, readableString(data))
+
+            data.tags
+                .filter { !tags.contains(it) }
+                .forEach {
+                    buildScan.tag(it)
+                    tags.add(it)
+                }
+        }
+        log.debug("Report statistic to build scan takes $elapsedTime ms")
     }
 
     private fun readableString(data: CompileStatData): String {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToElasticSearch.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.gradle.plugin.statistics
 
 import com.google.gson.Gson
+import org.gradle.api.logging.Logging
 import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
 import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics
@@ -13,6 +14,7 @@ import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData
 import java.net.HttpURLConnection
 import java.net.URL
 import java.util.*
+import kotlin.system.measureTimeMillis
 
 object ReportStatisticsToElasticSearch : ReportStatistics {
     val url by lazy { CompilerSystemProperties.KOTLIN_STAT_ENDPOINT_PROPERTY.value }
@@ -21,34 +23,38 @@ object ReportStatisticsToElasticSearch : ReportStatistics {
 
     //TODO Do not store password as string
     val password by lazy { CompilerSystemProperties.KOTLIN_STAT_PASSWORD_PROPERTY.value }
+    private val log = Logging.getLogger(this.javaClass)
 
     override fun report(data: CompileStatData) {
-        if (!enable) {
+        val elapsedTime = measureTimeMillis {
-            return;
+            if (!enable) {
-        }
+                return;
-
-        val connection = URL(url).openConnection() as HttpURLConnection
-
-        try {
-            if (user != null && password != null) {
-                val auth = Base64.getEncoder()
-                    .encode("$user:$password".toByteArray()).toString(Charsets.UTF_8)
-                connection.addRequestProperty("Authorization", "Basic $auth")
             }
-            connection.addRequestProperty("Content-Type", "application/json")
+
-            connection.requestMethod = "POST"
+            val connection = URL(url).openConnection() as HttpURLConnection
-            connection.doOutput = true
+
-            connection.outputStream.use {
+            try {
-                it.write(Gson().toJson(data).toByteArray())
+                if (user != null && password != null) {
+                    val auth = Base64.getEncoder()
+                        .encode("$user:$password".toByteArray()).toString(Charsets.UTF_8)
+                    connection.addRequestProperty("Authorization", "Basic $auth")
+                }
+                connection.addRequestProperty("Content-Type", "application/json")
+                connection.requestMethod = "POST"
+                connection.doOutput = true
+                connection.outputStream.use {
+                    it.write(Gson().toJson(data).toByteArray())
+                }
+                connection.connect()
+                checkResponseAndLog(connection)
+                connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
+            } catch (e: Exception) {
+                checkResponseAndLog(connection)
+            } finally {
+                connection.disconnect()
             }
-            connection.connect()
-            checkResponseAndLog(connection)
-            connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
-        } catch (e: Exception) {
-            checkResponseAndLog(connection)
-        } finally {
-            connection.disconnect()
         }
+        log.debug("Report statistic to elastic search takes $elapsedTime ms")
     }
 
     private fun checkResponseAndLog(connection: HttpURLConnection) {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -249,7 +249,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
     val startParameters = BuildMetricsReporterService.getStartParameters(project)
 
     @get:Internal
-    abstract val buildMetricsReporterService: Property<BuildMetricsReporterService?>
+    internal abstract val buildMetricsReporterService: Property<BuildMetricsReporterService?>
 
     internal fun reportingSettings() = buildMetricsReporterService.orNull?.parameters?.reportingSettings ?: ReportingSettings()