From cf3672314a40dbc882f859c081bd110c995c4091 Mon Sep 17 00:00:00 2001 From: "Nataliya.Valtman" Date: Tue, 2 Jan 2024 14:56:57 +0100 Subject: [PATCH] Remove JMX bean invocation for internal build FUS metrics #KT-58768 In Progress --- .../kotlin/gradle/BuildFusStatisticsIT.kt | 65 +++- .../kotlin/gradle/FusStatisticsIT.kt | 51 ++- .../kotlin/gradle/testbase/TestVersions.kt | 1 + .../plugin/StatisticsBuildFlowManager.kt | 7 +- .../plugin/statistics/BuildFusService.kt | 64 ++-- .../statistics/KotlinBuildStatHandler.kt | 316 ------------------ .../statistics/KotlinBuildStatsBeanService.kt | 189 +++++++++++ .../KotlinBuildStatsLoggerService.kt | 66 ++++ .../statistics/KotlinBuildStatsService.kt | 308 ----------------- .../KotlinBuildStatsServicesRegistry.kt | 55 ++- .../statistics/kotlinBuildStatisticsUtils.kt | 266 +++++++++++++++ ... Pre232IdeaKotlinBuildStatsBeanService.kt} | 13 +- .../kotlin/gradle/utils/reportUtils.kt | 27 +- .../plugin/statistics/BuildStatServiceTest.kt | 69 ---- .../statistics/BuildSessionLoggerTest.kt | 36 +- .../kotlin/statistics/BuildSession.kt | 4 +- .../kotlin/statistics/BuildSessionLogger.kt | 35 +- .../fileloggers/MetricsContainer.kt | 1 + 18 files changed, 715 insertions(+), 858 deletions(-) delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatHandler.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsBeanService.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsLoggerService.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt rename libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/{Pre232IdeaKotlinBuildStatsService.kt => Pre232IdeaKotlinBuildStatsBeanService.kt} (80%) delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildStatServiceTest.kt diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildFusStatisticsIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildFusStatisticsIT.kt index efcb373e6cd..f7540924b1a 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildFusStatisticsIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildFusStatisticsIT.kt @@ -15,6 +15,10 @@ import kotlin.io.path.pathString class BuildFusStatisticsIT : KGPDaemonsBaseTest() { @DisplayName("works for project with buildSrc and kotlinDsl plugin") @GradleTest + @GradleTestVersions( + additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0, TestVersions.Gradle.G_8_2, TestVersions.Gradle.G_8_3], + maxVersion = TestVersions.Gradle.G_8_5 + ) fun testCompatibilityBuildSrcWithKotlinDsl(gradleVersion: GradleVersion) { project( "buildSrcUsingKotlinCompilationAndKotlinPlugin", @@ -23,22 +27,59 @@ class BuildFusStatisticsIT : KGPDaemonsBaseTest() { ) { build("assemble") { //register build service for buildSrc. - val legacyBuildServiceMessagesCount = if (gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_0)) { + when { // until 8.0, Gradle was embedding the Kotlin version that used a slightly different approach to detect build finish, // so the service was unregistered after the finish of the buildSrc build // and then registered again in the root build - 2 - } else { - 1 + gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_0) -> { + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService: new instance", // the service for buildSrc + 1 + ) + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsBeanService: new instance", // the legacy service for compatibility + 1 + ) + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsBeanService_v2: new instance", // the current default version of the service + 1 + ) + } + gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_3) -> { + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService: new instance", // the legacy service for compatibility + 1 + ) + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsBeanService_v2: new instance", // the current default version of the service + 1 + ) + } + //for gradle 8.3 kotlin 1.9.0 is used, log message is changed + gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_5) -> { + assertOutputContainsExactTimes( + "Register JMX service for backward compatibility", // the legacy service for compatibility + 1 + ) + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService_v2: new instance", // the current default version of the service + 1 + ) + } + //for other versions KGP from buildSrc registered both services + else -> { + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService: new instance", // the legacy service for compatibility + 1 + ) + assertOutputContainsExactTimes( + "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService_v2: new instance", // the current default version of the service + 1 + ) + + } } - assertOutputContainsExactTimes( - "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService: new instance", // the legacy service for compatibility - legacyBuildServiceMessagesCount - ) - assertOutputContainsExactTimes( - "Instantiated class org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService_v2: new instance", // the current default version of the service - 1 - ) + assertOutputDoesNotContain("[org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatHandler] Could not execute") } } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/FusStatisticsIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/FusStatisticsIT.kt index dd76a5c91d2..9b94401dffd 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/FusStatisticsIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/FusStatisticsIT.kt @@ -12,7 +12,9 @@ import org.jetbrains.kotlin.gradle.util.replaceText import org.junit.jupiter.api.DisplayName import java.nio.file.Files import java.nio.file.Path +import kotlin.io.path.deleteIfExists import kotlin.streams.toList +import kotlin.test.assertTrue @DisplayName("FUS statistic") //Tests for FUS statistics have to create new instance of KotlinBuildStatsService @@ -95,9 +97,10 @@ class FusStatisticsIT : KGPDaemonsBaseTest() { @DisplayName("for project with buildSrc") @GradleTest @GradleTestVersions( - maxVersion = TestVersions.Gradle.G_7_6 + additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0] ) fun testProjectWithBuildSrcForGradleVersion7(gradleVersion: GradleVersion) { + //KT-64022 there are a different build instances in buildSrc and rest project: project( "instantExecutionWithBuildSrc", gradleVersion, @@ -112,26 +115,6 @@ class FusStatisticsIT : KGPDaemonsBaseTest() { } } - @DisplayName("for project with buildSrc") - @GradleTest - @GradleTestVersions( - minVersion = TestVersions.Gradle.G_8_0 - ) - fun testProjectWithBuildSrc(gradleVersion: GradleVersion) { - project( - "instantExecutionWithBuildSrc", - gradleVersion, - ) { - build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") { - assertFileContains( - fusStatisticsPath, - *expectedMetrics, - "BUILD_SRC_EXISTS=true" - ) - } - } - } - @DisplayName("for project with included build") @GradleTest @GradleTestVersions( @@ -139,24 +122,27 @@ class FusStatisticsIT : KGPDaemonsBaseTest() { maxVersion = TestVersions.Gradle.G_8_0 ) fun testProjectWithIncludedBuild(gradleVersion: GradleVersion) { + //KT-64022 + //there are a different build instances in buildSrc and rest project + project( "instantExecutionWithIncludedBuildPlugin", gradleVersion, buildOptions = defaultBuildOptions.copy(configurationCache = true) ) { build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") { - val fusStatisticsPath = fusStatisticsPath - assertFileContains( - fusStatisticsPath, - *expectedMetrics, - ) + Files.list(projectPath.resolve("kotlin-profile")).forEach { + assertFileContains(it, *expectedMetrics) + } } + Files.list(projectPath.resolve("kotlin-profile")).forEach { + assertTrue(it.deleteIfExists()) + } + build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") { - val fusStatisticsPath = fusStatisticsPath - assertFileContains( - fusStatisticsPath, - *expectedMetrics, - ) + Files.list(projectPath.resolve("kotlin-profile")).forEach { + assertFileContains(it, *expectedMetrics) + } } } } @@ -256,6 +242,9 @@ class FusStatisticsIT : KGPDaemonsBaseTest() { ) } + assertTrue(fusStatisticsPath.deleteIfExists()) + build("clean", buildOptions = buildOptions) + build( "compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath", diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/TestVersions.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/TestVersions.kt index 4020c553fec..2220e97c588 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/TestVersions.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/TestVersions.kt @@ -28,6 +28,7 @@ interface TestVersions { const val G_8_2 = "8.2.1" const val G_8_3 = "8.3" const val G_8_4 = "8.4" + const val G_8_5 = "8.5" const val MIN_SUPPORTED = minSupportedGradleVersion const val MIN_SUPPORTED_KPM = G_7_0 diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/StatisticsBuildFlowManager.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/StatisticsBuildFlowManager.kt index ec84bdccba8..022bdd16bc1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/StatisticsBuildFlowManager.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/StatisticsBuildFlowManager.kt @@ -60,17 +60,12 @@ internal class BuildFinishFlowAction : FlowAction - @get:Input - val action: Property - @get:Input val buildFailed: Property } override fun execute(parameters: Parameters) { - parameters.buildFusServiceProperty.orNull?.recordBuildFinished( - parameters.action.orNull, parameters.buildFailed.get() - ) + parameters.buildFusServiceProperty.orNull?.recordBuildFinished(parameters.buildFailed.get()) } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildFusService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildFusService.kt index 863d47a1474..217a9cb56fe 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildFusService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildFusService.kt @@ -49,14 +49,17 @@ internal interface UsesBuildFusService : Task { abstract class BuildFusService : BuildService, AutoCloseable, OperationCompletionListener { private var buildFailed: Boolean = false private val log = Logging.getLogger(this.javaClass) + private val buildId = randomUUID().toString() init { log.kotlinDebug("Initialize ${this.javaClass.simpleName}") + KotlinBuildStatsBeanService.recordBuildStart(buildId) } interface Parameters : BuildServiceParameters { val configurationMetrics: ListProperty val useBuildFinishFlowAction: Property + val buildStatisticsConfiguration: Property } private val fusMetricsConsumer = NonSynchronizedMetricsContainer() @@ -67,10 +70,11 @@ abstract class BuildFusService : BuildService, AutoC reportAction(fusMetricsConsumer) } - private val buildId = randomUUID().toString() + private val projectEvaluatedTime: Long = System.currentTimeMillis() companion object { internal val serviceName = "${BuildFusService::class.simpleName}_${BuildFusService::class.java.classLoader.hashCode()}" + private var buildStartTime: Long = System.currentTimeMillis() fun registerIfAbsent(project: Project, pluginVersion: String) = registerIfAbsentImpl(project, pluginVersion).also { serviceProvider -> @@ -90,20 +94,18 @@ abstract class BuildFusService : BuildService, AutoC val isProjectIsolationEnabled = project.isProjectIsolationEnabled project.gradle.sharedServices.registrations.findByName(serviceName)?.let { - @Suppress("UNCHECKED_CAST") - return (it.service as Provider).also { buildServiceProvider -> - project.afterEvaluate { - buildServiceProvider.get().parameters.configurationMetrics.add( - project.provider { KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project) } - ) + (it.parameters as Parameters).configurationMetrics.add( + project.provider { + collectProjectConfigurationTimeMetrics(project) } - } + + ) + @Suppress("UNCHECKED_CAST") + return (it.service as Provider) } //init buildStatsService - KotlinBuildStatsService.getOrCreateInstance(project)?.also { - it.recordProjectsEvaluated(project) - } + KotlinBuildStatsBeanService.initStatsService(project) val buildReportOutputs = reportingSettings(project).buildReportOutputs val useClasspathSnapshot = PropertiesProvider(project).useClasspathSnapshot @@ -113,9 +115,8 @@ abstract class BuildFusService : BuildService, AutoC // when this OperationCompletionListener is called services can be already closed for Gradle 8, // so there is a change that no VariantImplementationFactory will be found return gradle.sharedServices.registerIfAbsent(serviceName, BuildFusService::class.java) { spec -> - spec.parameters.configurationMetrics.add(project.provider { - KotlinBuildStatHandler.collectGeneralConfigurationTimeMetrics( + collectGeneralConfigurationTimeMetrics( gradle, buildReportOutputs, useClasspathSnapshot, @@ -123,8 +124,18 @@ abstract class BuildFusService : BuildService, AutoC isProjectIsolationEnabled ) }) + spec.parameters.configurationMetrics.add( + project.provider { + collectProjectConfigurationTimeMetrics(project) + } + ) spec.parameters.useBuildFinishFlowAction.set(GradleVersion.current().baseVersion >= GradleVersion.version("8.1")) + spec.parameters.buildStatisticsConfiguration.set(KotlinBuildStatsConfiguration(project)) }.also { buildService -> + //DO NOT call buildService.get() before all parameters.configurationMetrics are set. + // buildService.get() call will cause parameters calculation and configuration cache storage. + + //Gradle throws an exception when Gradle version less than 7.4 with configuration cache enabled and buildSrc, @Suppress("DEPRECATION") if (GradleVersion.current().baseVersion >= GradleVersion.version("7.4") || !project.isConfigurationCacheRequested @@ -132,19 +143,15 @@ abstract class BuildFusService : BuildService, AutoC ) { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildService) } + if (GradleVersion.current().baseVersion >= GradleVersion.version("8.1")) { StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult() } - project.afterEvaluate { - buildService.get().parameters.configurationMetrics.add(project.provider { - KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project) - }) - } - } } } + @Synchronized //access fusMetricsConsumer requires synchronisation as long as tasks are executed in parallel override fun onFinish(event: FinishEvent?) { if (event is TaskFinishEvent) { if (event.result is TaskFailureResult) { @@ -154,23 +161,30 @@ abstract class BuildFusService : BuildService, AutoC val taskExecutionResult = TaskExecutionResults[event.descriptor.taskPath] taskExecutionResult?.also { reportTaskMetrics(it, event) } } + event?.descriptor?.name?.also { + getMetricToReport(it)?.also { fusMetricsConsumer.report(it, true) } + } } override fun close() { if (!parameters.useBuildFinishFlowAction.get()) { - recordBuildFinished(null, buildFailed) + recordBuildFinished(buildFailed) } log.kotlinDebug("Close ${this.javaClass.simpleName}") } - internal fun recordBuildFinished(action: String?, buildFailed: Boolean) { - KotlinBuildStatHandler.reportGlobalMetrics(fusMetricsConsumer) + internal fun recordBuildFinished(buildFailed: Boolean) { + reportGlobalMetrics(log, fusMetricsConsumer) parameters.configurationMetrics.orElse(emptyList()).get().forEach { it.addToConsumer(fusMetricsConsumer) } - KotlinBuildStatsService.applyIfInitialised { - it.recordBuildFinish(action, buildFailed, fusMetricsConsumer, buildId) + reportBuildFinished(log, buildFailed, buildStartTime, projectEvaluatedTime, fusMetricsConsumer) + parameters.buildStatisticsConfiguration.orNull?.also { + val loggerService = KotlinBuildStatsLoggerService(it) + loggerService.initSessionLogger(buildId) + loggerService.reportBuildFinished(fusMetricsConsumer) } - KotlinBuildStatsService.closeServices() + KotlinBuildStatsBeanService.closeServices() } + private fun reportTaskMetrics(taskExecutionResult: TaskExecutionResult, event: TaskFinishEvent) { val totalTimeMs = event.result.endTime - event.result.startTime val buildMetrics = taskExecutionResult.buildMetrics diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatHandler.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatHandler.kt deleted file mode 100644 index 6783857c426..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatHandler.kt +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.statistics - -import org.gradle.api.Project -import org.gradle.api.artifacts.DependencySet -import org.gradle.api.artifacts.ProjectDependency -import org.gradle.api.invocation.Gradle -import org.gradle.api.logging.Logging -import org.jetbrains.kotlin.gradle.plugin.statistics.plugins.ObservablePlugins -import org.jetbrains.kotlin.gradle.report.BuildReportType -import org.jetbrains.kotlin.gradle.utils.* -import org.jetbrains.kotlin.statistics.BuildSessionLogger -import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics -import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics -import org.jetbrains.kotlin.statistics.metrics.StringMetrics -import org.jetbrains.kotlin.statistics.MetricValueValidationFailed -import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer -import java.lang.management.ManagementFactory -import javax.management.MBeanServer -import javax.management.ObjectName -import kotlin.system.measureTimeMillis - -class KotlinBuildStatHandler { - companion object { - - @JvmStatic - internal fun getLogger() = Logging.getLogger(KotlinBuildStatHandler::class.java) - - internal fun runSafe(methodName: String, action: () -> T?): T? { - return try { - getLogger().debug("Executing [$methodName]") - action.invoke() - } catch (e: MetricValueValidationFailed) { - throw e - } catch (e: Throwable) { - logException("Could not execute [$methodName]", e) - null - } - } - - internal fun logException(description: String, e: Throwable) { - getLogger().info(description) - getLogger().debug(e.message, e) - } - - /** - * Collect general configuration metrics - */ - internal fun collectGeneralConfigurationTimeMetrics( - gradle: Gradle, - buildReportOutputs: List, - useClasspathSnapshot: Boolean, - pluginVersion: String, - isProjectIsolationEnabled: Boolean, - ): MetricContainer { - val configurationTimeMetrics = MetricContainer() - - val statisticOverhead = measureTimeMillis { - configurationTimeMetrics.put(StringMetrics.KOTLIN_COMPILER_VERSION, pluginVersion) - configurationTimeMetrics.put(StringMetrics.USE_CLASSPATH_SNAPSHOT, useClasspathSnapshot.toString()) - buildReportOutputs.forEach { - when (it) { - BuildReportType.BUILD_SCAN -> configurationTimeMetrics.put(BooleanMetrics.BUILD_SCAN_BUILD_REPORT, true) - BuildReportType.FILE -> configurationTimeMetrics.put(BooleanMetrics.FILE_BUILD_REPORT, true) - BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true) - BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true) - BuildReportType.TRY_NEXT_CONSOLE -> {}//ignore - } - } - configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath) - configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion) - - //will be updated with KT-58266 - if (!isProjectIsolationEnabled) { - gradle.taskGraph.whenReady { taskExecutionGraph -> - val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct() - configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install")) - } - } - - } - configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) - - return configurationTimeMetrics - - } - - /** - * Collect project's configuration metrics including applied plugins. It should be called inside afterEvaluate block. - */ - internal fun collectProjectConfigurationTimeMetrics( - project: Project, - ): MetricContainer { - val configurationTimeMetrics = MetricContainer() - - val statisticOverhead = measureTimeMillis { - collectAppliedPluginsStatistics(project, configurationTimeMetrics) - - val configurations = project.configurations.asMap.values - for (configuration in configurations) { - try { - val configurationName = configuration.name - val dependencies = configuration.dependencies - - when (configurationName) { - "KoverEngineConfig" -> { - configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true) - } - "kapt" -> { - configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true) - for (dependency in dependencies) { - when (dependency.group) { - "com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true) - "com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true) - } - } - } - API -> { - configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1) - reportLibrariesVersions(configurationTimeMetrics, dependencies) - } - IMPLEMENTATION -> { - configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1) - reportLibrariesVersions(configurationTimeMetrics, dependencies) - } - COMPILE -> { - configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1) - reportLibrariesVersions(configurationTimeMetrics, dependencies) - } - COMPILE_ONLY -> { - configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_ONLY_COUNT, 1) - reportLibrariesVersions(configurationTimeMetrics, dependencies) - } - RUNTIME -> { - configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1) - reportLibrariesVersions(configurationTimeMetrics, dependencies) - } - RUNTIME_ONLY -> { - configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_ONLY_COUNT, 1) - reportLibrariesVersions(configurationTimeMetrics, dependencies) - } - } - } catch (e: Throwable) { - // log? - } - } - - configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1) - - - configurationTimeMetrics.put( - BooleanMetrics.KOTLIN_KTS_USED, - project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false - ) - - addTaskMetrics(project, configurationTimeMetrics) - - if (project.name == "buildSrc") { - configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1) - configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true) - } - } - configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) - - return configurationTimeMetrics - } - - private fun addTaskMetrics( - project: Project, - configurationTimeMetrics: MetricContainer, - ) { - try { - val taskNames = project.tasks.names.toList() - configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong()) - configurationTimeMetrics.put( - NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS, - taskNames.count { name -> - try { - project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider") - } catch (_: Exception) { - true - } - }.toLong() - ) - } catch (e: Exception) { - //ignore exceptions for KT-62131. - } - } - - private fun collectAppliedPluginsStatistics( - project: Project, - configurationTimeMetrics: MetricContainer, - ) { - for (plugin in ObservablePlugins.values()) { - project.plugins.withId(plugin.title) { - configurationTimeMetrics.put(plugin.metric, true) - } - } - } - - private fun reportLibrariesVersions( - configurationTimeMetrics: MetricContainer, - dependencies: DependencySet?, - ) { - dependencies?.filter { it !is ProjectDependency }?.forEach { dependency -> - when { - dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put( - StringMetrics.LIBRARY_SPRING_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group?.startsWith("com.vaadin") ?: false -> configurationTimeMetrics.put( - StringMetrics.LIBRARY_VAADIN_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group?.startsWith("com.google.gwt") ?: false -> configurationTimeMetrics.put( - StringMetrics.LIBRARY_GWT_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group?.startsWith("org.hibernate") ?: false -> configurationTimeMetrics.put( - StringMetrics.LIBRARY_HIBERNATE_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group == "org.jetbrains.kotlin" && dependency.name.startsWith("kotlin-stdlib") -> configurationTimeMetrics.put( - StringMetrics.KOTLIN_STDLIB_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group == "org.jetbrains.kotlinx" && dependency.name == "kotlinx-coroutines" -> configurationTimeMetrics.put( - StringMetrics.KOTLIN_COROUTINES_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group == "org.jetbrains.kotlin" && dependency.name == "kotlin-reflect" -> configurationTimeMetrics.put( - StringMetrics.KOTLIN_REFLECT_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group == "org.jetbrains.kotlinx" && dependency.name - .startsWith("kotlinx-serialization-runtime") -> configurationTimeMetrics.put( - StringMetrics.KOTLIN_SERIALIZATION_VERSION, - dependency.version ?: "0.0.0" - ) - dependency.group == "com.android.tools.build" && dependency.name.startsWith("gradle") -> configurationTimeMetrics.put( - StringMetrics.ANDROID_GRADLE_PLUGIN_VERSION, - dependency.version ?: "0.0.0" - ) - } - } - } - - internal fun reportGlobalMetrics(metricConsumer: StatisticsValuesConsumer) { - runSafe("${KotlinBuildStatHandler::class.java}.reportGlobalMetrics") { - System.getProperty("os.name")?.also { metricConsumer.report(StringMetrics.OS_TYPE, System.getProperty("os.name")) } - metricConsumer.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong()) - metricConsumer.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null) - metricConsumer.report(NumericalMetrics.GRADLE_DAEMON_HEAP_SIZE, Runtime.getRuntime().maxMemory()) - } - } - - } - - fun buildFinished( - beanName: ObjectName, - ) { - runSafe("${KotlinBuildStatHandler::class.java}.buildFinished") { - val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() - if (mbs.isRegistered(beanName)) { - mbs.unregisterMBean(beanName) - } - } - } - - internal fun reportBuildFinished( - sessionLogger: BuildSessionLogger, - action: String?, - buildFailed: Boolean, - buildId: String, - metricsContainer: NonSynchronizedMetricsContainer, - ) { - runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") { - metricsContainer.sendToConsumer(sessionLogger) - sessionLogger.finishBuildSession(action, buildFailed, buildId) - } - } - - internal fun report( - sessionLogger: BuildSessionLogger, - metric: BooleanMetrics, - value: Boolean, - subprojectName: String?, - weight: Long? = null, - ) = runSafe("report metric ${metric.name}") { - sessionLogger.report(metric, value, subprojectName, weight) - } ?: false - - internal fun report( - sessionLogger: BuildSessionLogger, - metric: NumericalMetrics, - value: Long, - subprojectName: String?, - weight: Long? = null, - ) = runSafe("report metric ${metric.name}") { - sessionLogger.report(metric, value, subprojectName, weight) - } ?: false - - internal fun report( - sessionLogger: BuildSessionLogger, - metric: StringMetrics, - value: String, - subprojectName: String?, - weight: Long? = null, - ) = runSafe("report metric ${metric.name}") { - sessionLogger.report(metric, value, subprojectName, weight) - } ?: false -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsBeanService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsBeanService.kt new file mode 100644 index 00000000000..bfe18c12a8f --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsBeanService.kt @@ -0,0 +1,189 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import org.gradle.api.Project +import org.gradle.api.invocation.Gradle +import org.gradle.api.logging.Logging +import org.gradle.api.provider.ProviderFactory +import org.jetbrains.kotlin.gradle.plugin.internal.ConfigurationTimePropertiesAccessor +import org.jetbrains.kotlin.gradle.plugin.internal.configurationTimePropertiesAccessor +import org.jetbrains.kotlin.gradle.plugin.internal.usedAtConfigurationTime +import org.jetbrains.kotlin.gradle.utils.runMetricMethodSafely +import org.jetbrains.kotlin.statistics.BuildSessionLogger +import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME +import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics +import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer +import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics +import org.jetbrains.kotlin.statistics.metrics.StringMetrics +import java.io.Closeable +import java.io.File +import java.lang.management.ManagementFactory +import javax.management.MBeanServer +import javax.management.ObjectName + +/** + * Interface for populating statistics collection method via JXM interface + * JMX could be used for reporting both from other JVMs, other versions + * of Kotlin Plugin and other classloaders + */ + +interface KotlinBuildStatsMXBean { + fun reportBoolean(name: String, value: Boolean, subprojectName: String?, weight: Long?): Boolean + + fun reportNumber(name: String, value: Long, subprojectName: String?, weight: Long?): Boolean + + fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean +} + +internal abstract class KotlinBuildStatsBeanService internal constructor( + project: Project, + private val beanName: ObjectName, +) : StatisticsValuesConsumer, Closeable { + + companion object { + // Property name for disabling saving statistical information + private const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile" + + // default state + private const val DEFAULT_STATISTICS_STATE = true + + // "emergency file" collecting statistics is disabled it the file exists + private const val DISABLE_STATISTICS_FILE_NAME = "${STATISTICS_FOLDER_NAME}/.disable" + + @JvmStatic + internal val logger = Logging.getLogger(KotlinBuildStatsBeanService::class.java) + + /** + * Method for creating new instance of [StatisticsValuesConsumer] + * It could be invoked only when applying Kotlin gradle plugin. + * When executed, this method checks, whether it is already executed in the current build (whether it was already executed + * in the same classpath (i.e., with the same version of Kotlin plugin)). + * If it was not executed, the new instance of StatisticsValuesConsumer is created + * + * [closeServices] must be called at the end of the build in order to release resources. + */ + @JvmStatic + @Synchronized + internal fun initStatsService(project: Project) { + runMetricMethodSafely(logger, "${KotlinBuildStatsBeanService::class.java}.initStatsService") { + val gradle = project.gradle + val configurationTimePropertiesAccessor = project.configurationTimePropertiesAccessor + val statisticsIsEnabled = checkStatisticsEnabled(gradle, project.providers, configurationTimePropertiesAccessor) + if (!statisticsIsEnabled) { + null + } else { + //register JMX service to support old kotlin version compatibility + + val registry = kotlinBuildStatsServicesRegistry ?: KotlinBuildStatsServicesRegistry().also { + kotlinBuildStatsServicesRegistry = it + } + + registry.registerServices(project) + } + } + } + + internal fun recordBuildStart(buildId: String) { + kotlinBuildStatsServicesRegistry?.recordBuildStart(buildId) + } + + fun closeServices() { + kotlinBuildStatsServicesRegistry?.close() + kotlinBuildStatsServicesRegistry = null + } + + private var kotlinBuildStatsServicesRegistry: KotlinBuildStatsServicesRegistry? = null + + private fun checkStatisticsEnabled( + gradle: Gradle, + providerFactory: ProviderFactory, + configurationTimePropertiesAccessor: ConfigurationTimePropertiesAccessor, + ): Boolean { + return if (File(gradle.gradleUserHomeDir, DISABLE_STATISTICS_FILE_NAME).exists()) { + false + } else { + providerFactory.gradleProperty(ENABLE_STATISTICS_PROPERTY_NAME) + .usedAtConfigurationTime(configurationTimePropertiesAccessor) + .orNull?.toBoolean() ?: DEFAULT_STATISTICS_STATE + } + } + } + + protected val kotlinBuildLogger = KotlinBuildStatsLoggerService(KotlinBuildStatsConfiguration(project)) + + @Synchronized + override fun close() { + runMetricMethodSafely(logger, "${KotlinBuildStatsBeanService::class.java}.buildFinished") { + val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() + if (mbs.isRegistered(beanName)) { + mbs.unregisterMBean(beanName) + logger.debug("Unregistered $this") + } + } + } + + @Synchronized + fun recordBuildStart(buildId: String) { + kotlinBuildLogger.initSessionLogger(buildId) + } + + internal fun report( + sessionLogger: BuildSessionLogger, + metric: BooleanMetrics, + value: Boolean, + subprojectName: String?, + weight: Long? = null, + ) = runMetricMethodSafely(logger, "report metric ${metric.name}") { + sessionLogger.report(metric, value, subprojectName, weight) + } ?: false + + internal fun report( + sessionLogger: BuildSessionLogger, + metric: NumericalMetrics, + value: Long, + subprojectName: String?, + weight: Long? = null, + ) = runMetricMethodSafely(logger, "report metric ${metric.name}") { + sessionLogger.report(metric, value, subprojectName, weight) + } ?: false + + internal fun report( + sessionLogger: BuildSessionLogger, + metric: StringMetrics, + value: String, + subprojectName: String?, + weight: Long? = null, + ) = runMetricMethodSafely(logger, "report metric ${metric.name}") { + sessionLogger.report(metric, value, subprojectName, weight) + } ?: false +} + +internal class DefaultKotlinBuildStatsBeanService internal constructor( + project: Project, + beanName: ObjectName, +) : KotlinBuildStatsBeanService(project, beanName), + KotlinBuildStatsMXBean { + + override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?): Boolean = + report(kotlinBuildLogger.sessionLogger, metric, value, subprojectName, weight) + + override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?): Boolean = + report(kotlinBuildLogger.sessionLogger, metric, value, subprojectName, weight) + + override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?): Boolean = + report(kotlinBuildLogger.sessionLogger, metric, value, subprojectName, weight) + + override fun reportBoolean(name: String, value: Boolean, subprojectName: String?, weight: Long?): Boolean = + report(BooleanMetrics.valueOf(name), value, subprojectName, weight) + + override fun reportNumber(name: String, value: Long, subprojectName: String?, weight: Long?): Boolean = + report(NumericalMetrics.valueOf(name), value, subprojectName, weight) + + override fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean = + report(StringMetrics.valueOf(name), value, subprojectName, weight) + +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsLoggerService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsLoggerService.kt new file mode 100644 index 00000000000..a8cd4cd7e69 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsLoggerService.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import org.gradle.api.Project +import org.gradle.api.logging.Logging +import org.jetbrains.kotlin.gradle.plugin.PropertiesBuildService +import org.jetbrains.kotlin.statistics.BuildSessionLogger +import org.jetbrains.kotlin.gradle.utils.runMetricMethodSafely +import java.io.File +import java.io.Serializable + +class KotlinBuildStatsConfiguration( + internal val forcePropertiesValidation: Boolean, + internal val sessionLoggerRootPath: File, +) : Serializable { + companion object { + // Property used for tests. + private const val CUSTOM_LOGGER_ROOT_PATH = "kotlin.session.logger.root.path" + + // Property used for tests. Build will fail fast if collected value doesn't fit regexp + private const val FORCE_VALUES_VALIDATION = "kotlin_performance_profile_force_validation" + private val logger = Logging.getLogger(KotlinBuildStatsConfiguration::class.java) + } + + constructor(project: Project) : this( + PropertiesBuildService.registerIfAbsent(project).get().get(FORCE_VALUES_VALIDATION, project)?.toBoolean() ?: false, + PropertiesBuildService.registerIfAbsent(project).get().get(CUSTOM_LOGGER_ROOT_PATH, project) + ?.also { + logger.warn("$CUSTOM_LOGGER_ROOT_PATH property for test purpose only") + }?.let { File(it) } ?: project.gradle.gradleUserHomeDir + ) +} + +internal open class KotlinBuildStatsLoggerService( + configuration: KotlinBuildStatsConfiguration, +) { + companion object { + private val logger = Logging.getLogger(KotlinBuildStatsLoggerService::class.java) + } + + internal val sessionLogger = BuildSessionLogger( + configuration.sessionLoggerRootPath, + forceValuesValidation = configuration.forcePropertiesValidation, + ) + + fun initSessionLogger(buildId: String) { + runMetricMethodSafely(logger, "${KotlinBuildStatsLoggerService::class.java}.initSessionLogger") { + if (!sessionLogger.isBuildSessionStarted()) { + sessionLogger.startBuildSession(buildId) + } else { + logger.debug("Unable to initiate build logger for \"$buildId\" build. It is already initialized for \"${sessionLogger.getActiveBuildId()}\" build") + } + } + } + + internal fun reportBuildFinished(metricsContainer: NonSynchronizedMetricsContainer? = null) { + runMetricMethodSafely(logger, "${KotlinBuildStatsLoggerService::class.java}.reportBuildFinish") { + metricsContainer?.sendToConsumer(sessionLogger) + sessionLogger.finishBuildSession() + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt deleted file mode 100644 index f10c5fbd822..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.statistics - -import org.gradle.api.Project -import org.gradle.api.invocation.Gradle -import org.gradle.api.logging.Logging -import org.gradle.api.provider.ProviderFactory -import org.gradle.initialization.BuildRequestMetaData -import org.gradle.invocation.DefaultGradle -import org.gradle.tooling.events.OperationCompletionListener -import org.gradle.tooling.events.task.TaskFinishEvent -import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder -import org.jetbrains.kotlin.gradle.plugin.PropertiesBuildService -import org.jetbrains.kotlin.gradle.plugin.internal.ConfigurationTimePropertiesAccessor -import org.jetbrains.kotlin.gradle.plugin.internal.configurationTimePropertiesAccessor -import org.jetbrains.kotlin.gradle.plugin.internal.usedAtConfigurationTime -import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatHandler.Companion.runSafe -import org.jetbrains.kotlin.statistics.BuildSessionLogger -import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME -import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics -import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer -import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics -import org.jetbrains.kotlin.statistics.metrics.StringMetrics -import java.io.Closeable -import java.io.File -import javax.management.MBeanServer -import javax.management.ObjectName -import kotlin.system.measureTimeMillis - -/** - * Interface for populating statistics collection method via JXM interface - * JMX could be used for reporting both from other JVMs, other versions - * of Kotlin Plugin and other classloaders - */ - -interface KotlinBuildStatsMXBean { - fun reportBoolean(name: String, value: Boolean, subprojectName: String?, weight: Long?): Boolean - - fun reportNumber(name: String, value: Long, subprojectName: String?, weight: Long?): Boolean - - fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean -} - -internal abstract class KotlinBuildStatsService internal constructor() : StatisticsValuesConsumer, Closeable { - companion object { - // Property name for disabling saving statistical information - private const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile" - - // Property used for tests. Build will fail fast if collected value doesn't fit regexp - const val FORCE_VALUES_VALIDATION = "kotlin_performance_profile_force_validation" - - // default state - private const val DEFAULT_STATISTICS_STATE = true - - // "emergency file" collecting statistics is disabled it the file exists - private const val DISABLE_STATISTICS_FILE_NAME = "${STATISTICS_FOLDER_NAME}/.disable" - - /** - * Method for getting IStatisticsValuesConsumer for reporting some statistics - * Could be invoked during any build phase after applying first Kotlin plugin and - * until build is completed - */ - @JvmStatic - @Synchronized - private fun getInstance(): KotlinBuildStatsService? { - if (statisticsIsEnabled != true) { - return null - } - return kotlinBuildStatsServicesRegistry?.getDefaultService() - } - - /** - * Method for creating new instance of IStatisticsValuesConsumer - * It could be invoked only when applying Kotlin gradle plugin. - * When executed, this method checks, whether it is already executed in the current build. - * If it was not executed, the new instance of IStatisticsValuesConsumer is created - * - * If it was already executed in the same classpath (i.e., with the same version of Kotlin plugin), - * the previously returned instance is returned. - * - * If it was already executed in the other classpath, a JXM implementation is returned. - * - * [closeServices] must be called at the end of the build in order to release resources. - */ - @JvmStatic - @Synchronized - internal fun getOrCreateInstance(project: Project): KotlinBuildStatsService? { - - return runSafe("${KotlinBuildStatsService::class.java}.getOrCreateInstance") { - val gradle = project.gradle - val configurationTimePropertiesAccessor = project.configurationTimePropertiesAccessor - statisticsIsEnabled = statisticsIsEnabled - ?: checkStatisticsEnabled(gradle, project.providers, configurationTimePropertiesAccessor) - if (statisticsIsEnabled != true) { - null - } else { - val registry = kotlinBuildStatsServicesRegistry ?: KotlinBuildStatsServicesRegistry().also { - kotlinBuildStatsServicesRegistry = it - } - - val defaultServiceName = - KotlinBuildStatsServicesRegistry.getBeanName(KotlinBuildStatsServicesRegistry.DEFAULT_SERVICE_QUALIFIER) - val instance = kotlinBuildStatsServicesRegistry?.getDefaultService() - if (instance != null) { - registry.logger.debug("$defaultServiceName is already instantiated. Current instance is $instance") - return@runSafe instance - } - - registry.registerServices(project) - - BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(project.provider { - OperationCompletionListener { event -> - if (event is TaskFinishEvent) { - reportTaskIfNeed(event.descriptor.name) - } - } - }) - - registry.getDefaultService() ?: error("The default kotlin build stats $defaultServiceName service wasn't initialized") - } - } - } - - fun closeServices() { - kotlinBuildStatsServicesRegistry?.close() - kotlinBuildStatsServicesRegistry = null - } - - protected fun reportTaskIfNeed(task: String) { - val metric = when (task.substringAfterLast(":")) { - "dokkaHtml" -> BooleanMetrics.ENABLED_DOKKA_HTML - "dokkaGfm" -> BooleanMetrics.ENABLED_DOKKA_GFM - "dokkaJavadoc" -> BooleanMetrics.ENABLED_DOKKA_JAVADOC - "dokkaJekyll" -> BooleanMetrics.ENABLED_DOKKA_JEKYLL - "dokkaHtmlMultiModule" -> BooleanMetrics.ENABLED_DOKKA_HTML_MULTI_MODULE - "dokkaGfmMultiModule" -> BooleanMetrics.ENABLED_DOKKA_GFM_MULTI_MODULE - "dokkaJekyllMultiModule" -> BooleanMetrics.ENABLED_DOKKA_JEKYLL_MULTI_MODULE - "dokkaHtmlCollector" -> BooleanMetrics.ENABLED_DOKKA_HTML_COLLECTOR - "dokkaGfmCollector" -> BooleanMetrics.ENABLED_DOKKA_GFM_COLLECTOR - "dokkaJavadocCollector" -> BooleanMetrics.ENABLED_DOKKA_JAVADOC_COLLECTOR - "dokkaJekyllCollector" -> BooleanMetrics.ENABLED_DOKKA_JEKYLL_COLLECTOR - else -> null - } - metric?.also { getInstance()?.report(it, true) } - } - - - /** - * Invokes provided collector if the reporting service is initialised. - * The duration of collector's wall time is reported into overall overhead metric. - */ - fun applyIfInitialised(collector: (KotlinBuildStatsService) -> Unit) { - getInstance()?.apply { - try { - val duration = measureTimeMillis { - collector.invoke(this) - } - this.report(NumericalMetrics.STATISTICS_COLLECT_METRICS_OVERHEAD, duration) - } catch (e: Throwable) { - KotlinBuildStatHandler.logException("Could collect statistics metrics", e) - } - } - } - - private var kotlinBuildStatsServicesRegistry: KotlinBuildStatsServicesRegistry? = null - - private var statisticsIsEnabled: Boolean? = null - - private fun checkStatisticsEnabled( - gradle: Gradle, - providerFactory: ProviderFactory, - configurationTimePropertiesAccessor: ConfigurationTimePropertiesAccessor, - ): Boolean { - return if (File(gradle.gradleUserHomeDir, DISABLE_STATISTICS_FILE_NAME).exists()) { - false - } else { - providerFactory.gradleProperty(ENABLE_STATISTICS_PROPERTY_NAME) - .usedAtConfigurationTime(configurationTimePropertiesAccessor) - .orNull?.toBoolean() ?: DEFAULT_STATISTICS_STATE - } - } - } - - override fun close() { - } - /** - * Collects metrics at the end of a build - */ - open fun recordBuildFinish(action: String?, buildFailed: Boolean, metricsContainer: NonSynchronizedMetricsContainer, buildId: String) {} - - open fun recordProjectsEvaluated(project: Project) {} -} - -internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) : - KotlinBuildStatsService() { - - private fun callJmx(method: String, type: String, metricName: String, value: Any, subprojectName: String?, weight: Long?): Any? { - return mbs.invoke( - beanName, - method, - arrayOf(metricName, value, subprojectName, weight), - arrayOf("java.lang.String", type, "java.lang.String", "java.lang.Long") - ) - } - - override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) = - runSafe("report metric ${metric.name}") { - callJmx("reportBoolean", "boolean", metric.name, value, subprojectName, weight) - } as? Boolean ?: false - - override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?) = - runSafe("report metric ${metric.name}") { - callJmx("reportNumber", "long", metric.name, value, subprojectName, weight) - } as? Boolean ?: false - - override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?) = - runSafe("report metric ${metric.name}") { - callJmx("reportString", "java.lang.String", metric.name, value, subprojectName, weight) - } as? Boolean ?: false - -} - -internal abstract class AbstractKotlinBuildStatsService( - project: Project, - private val beanName: ObjectName, -) : KotlinBuildStatsService() { - companion object { - //test only - const val CUSTOM_LOGGER_ROOT_PATH = "kotlin.session.logger.root.path" - - private val logger = Logging.getLogger(AbstractKotlinBuildStatsService::class.java) - } - - private val forcePropertiesValidation: Boolean - - private val customSessionLoggerRootPath: String? - - init { - val propertiesBuildService = PropertiesBuildService.registerIfAbsent(project).get() - forcePropertiesValidation = propertiesBuildService.get(FORCE_VALUES_VALIDATION, project)?.toBoolean() ?: false - customSessionLoggerRootPath = propertiesBuildService.get(CUSTOM_LOGGER_ROOT_PATH, project)?.also { - logger.warn("$CUSTOM_LOGGER_ROOT_PATH property for test purpose only") - } - } - - private val sessionLoggerRootPath = - customSessionLoggerRootPath?.let { File(it) } ?: project.gradle.gradleUserHomeDir - - protected val sessionLogger = BuildSessionLogger( - sessionLoggerRootPath, - forceValuesValidation = forcePropertiesValidation, - ) - - private fun gradleBuildStartTime(gradle: Gradle): Long? { - return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime - } - - override fun recordProjectsEvaluated(project: Project) { - runSafe("${DefaultKotlinBuildStatsService::class.java}.projectEvaluated") { - if (!sessionLogger.isBuildSessionStarted()) { - sessionLogger.startBuildSession( - DaemonReuseCounter.incrementAndGetOrdinal(), - gradleBuildStartTime(project.gradle), - ) - } - } - } - - @Synchronized - override fun close() { - KotlinBuildStatHandler().buildFinished(beanName) - super.close() - } -} - -internal class DefaultKotlinBuildStatsService internal constructor( - project: Project, - beanName: ObjectName, -) : AbstractKotlinBuildStatsService(project, beanName), - KotlinBuildStatsMXBean { - - override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?): Boolean = - KotlinBuildStatHandler().report(sessionLogger, metric, value, subprojectName, weight) - - override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?): Boolean = - KotlinBuildStatHandler().report(sessionLogger, metric, value, subprojectName, weight) - - override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?): Boolean = - KotlinBuildStatHandler().report(sessionLogger, metric, value, subprojectName, weight) - - override fun reportBoolean(name: String, value: Boolean, subprojectName: String?, weight: Long?): Boolean = - report(BooleanMetrics.valueOf(name), value, subprojectName, weight) - - override fun reportNumber(name: String, value: Long, subprojectName: String?, weight: Long?): Boolean = - report(NumericalMetrics.valueOf(name), value, subprojectName, weight) - - override fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean = - report(StringMetrics.valueOf(name), value, subprojectName, weight) - - //only one jmx bean service should report global metrics - override fun recordBuildFinish(action: String?, buildFailed: Boolean, metricsContainer: NonSynchronizedMetricsContainer, buildId: String) { - KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, buildId, metricsContainer) - } -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsServicesRegistry.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsServicesRegistry.kt index 0246a632783..c9840aedbdf 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsServicesRegistry.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsServicesRegistry.kt @@ -8,7 +8,7 @@ package org.jetbrains.kotlin.gradle.plugin.statistics import org.gradle.api.Project import org.gradle.api.logging.Logging import org.jetbrains.kotlin.gradle.plugin.statistics.old.Pre232IdeaKotlinBuildStatsMXBean -import org.jetbrains.kotlin.gradle.plugin.statistics.old.Pre232IdeaKotlinBuildStatsService +import org.jetbrains.kotlin.gradle.plugin.statistics.old.Pre232IdeaKotlinBuildStatsBeanService import java.io.Closeable import java.lang.management.ManagementFactory import javax.management.MBeanServer @@ -25,58 +25,53 @@ import javax.management.StandardMBean */ internal class KotlinBuildStatsServicesRegistry : Closeable { internal val logger = Logging.getLogger(this::class.java) - private val services = hashMapOf() + private val services = hashMapOf() /** * Registers the Kotlin build stats services for the given project. * - * After a call of that method, [getDefaultService] is expected to return a non-null value. - * * The registry must be closed at the end of the usage. */ fun registerServices(project: Project) { - val defaultBeanName = getBeanName(DEFAULT_SERVICE_QUALIFIER) val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() - if (mbs.isRegistered(defaultBeanName)) { - // because we use only the default service, and it's already registered in the MBean server, then there's no need in caring of legacy services as they're already registered - logger.debug("${KotlinBuildStatsService::class.simpleName} $defaultBeanName is already instantiated in another classpath. Creating JMX wrapper for the main service") - services[DEFAULT_SERVICE_QUALIFIER] = JMXKotlinBuildStatsService(mbs, defaultBeanName) - } else { - registerStatsService( - mbs, - DefaultKotlinBuildStatsService(project, getBeanName(DEFAULT_SERVICE_QUALIFIER)), - KotlinBuildStatsMXBean::class.java, - DEFAULT_SERVICE_QUALIFIER - ) - // to support backward compatibility with Idea before version 232 - registerStatsService( - mbs, - Pre232IdeaKotlinBuildStatsService(project, getBeanName(LEGACY_SERVICE_QUALIFIER)), - Pre232IdeaKotlinBuildStatsMXBean::class.java, - LEGACY_SERVICE_QUALIFIER - ) - } + registerStatsService( + mbs, + DefaultKotlinBuildStatsBeanService(project, getBeanName(DEFAULT_SERVICE_QUALIFIER)), + KotlinBuildStatsMXBean::class.java, + DEFAULT_SERVICE_QUALIFIER + ) + // to support backward compatibility with Idea before version 232 + registerStatsService( + mbs, + Pre232IdeaKotlinBuildStatsBeanService(project, getBeanName(LEGACY_SERVICE_QUALIFIER)), + Pre232IdeaKotlinBuildStatsMXBean::class.java, + LEGACY_SERVICE_QUALIFIER + ) + } - private fun registerStatsService( + fun recordBuildStart(buildId: String) { + services.forEach { it.value.recordBuildStart(buildId) } + } + + + private fun registerStatsService( mbs: MBeanServer, service: T, beanInterfaceType: Class, qualifier: String, ) { val beanName = getBeanName(qualifier) + val loggedServiceName = "${KotlinBuildStatsBeanService::class.java}" + if (qualifier.isNotEmpty()) "_$qualifier" else "" if (!mbs.isRegistered(beanName)) { mbs.registerMBean(StandardMBean(service, beanInterfaceType), beanName) services[qualifier] = service - val loggedServiceName = "${KotlinBuildStatsService::class.java}" + if (qualifier.isNotEmpty()) "_$qualifier" else "" logger.debug("Instantiated $loggedServiceName: new instance $service") + } else { + logger.debug("$loggedServiceName is already instantiated in another classpath.") } } - /** - * Retrieves the default Kotlin build stats service. That's the main service we are working with. - */ - fun getDefaultService() = services[DEFAULT_SERVICE_QUALIFIER] /** * Unregisters all the registered JMX services and may release other resources allocated by a service. diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt new file mode 100644 index 00000000000..cf7937e6b08 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt @@ -0,0 +1,266 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import org.gradle.api.Project +import org.gradle.api.artifacts.DependencySet +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.invocation.Gradle +import org.gradle.api.logging.Logger +import org.jetbrains.kotlin.gradle.plugin.statistics.plugins.ObservablePlugins +import org.jetbrains.kotlin.gradle.report.BuildReportType +import org.jetbrains.kotlin.gradle.utils.* +import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics +import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics +import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer +import org.jetbrains.kotlin.statistics.metrics.StringMetrics +import kotlin.system.measureTimeMillis + +/** + * Collect general configuration metrics + **/ +internal fun collectGeneralConfigurationTimeMetrics( + gradle: Gradle, + buildReportOutputs: List, + useClasspathSnapshot: Boolean, + pluginVersion: String, + isProjectIsolationEnabled: Boolean, +): MetricContainer { + val configurationTimeMetrics = MetricContainer() + + val statisticOverhead = measureTimeMillis { + configurationTimeMetrics.put(StringMetrics.KOTLIN_COMPILER_VERSION, pluginVersion) + configurationTimeMetrics.put(StringMetrics.USE_CLASSPATH_SNAPSHOT, useClasspathSnapshot.toString()) + buildReportOutputs.forEach { + when (it) { + BuildReportType.BUILD_SCAN -> configurationTimeMetrics.put(BooleanMetrics.BUILD_SCAN_BUILD_REPORT, true) + BuildReportType.FILE -> configurationTimeMetrics.put(BooleanMetrics.FILE_BUILD_REPORT, true) + BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true) + BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true) + BuildReportType.TRY_NEXT_CONSOLE -> {}//ignore + } + } + configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath) + configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion) + + //will be updated with KT-58266 + if (!isProjectIsolationEnabled) { + gradle.taskGraph.whenReady { taskExecutionGraph -> + val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct() + configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install")) + } + } + } + configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) + + return configurationTimeMetrics + +} + +/** + * Collect project's configuration metrics including applied plugins. It should be called inside afterEvaluate block. + */ +internal fun collectProjectConfigurationTimeMetrics( + project: Project, +): MetricContainer { + val configurationTimeMetrics = MetricContainer() + + val statisticOverhead = measureTimeMillis { + collectAppliedPluginsStatistics(project, configurationTimeMetrics) + + val configurations = project.configurations.asMap.values + for (configuration in configurations) { + try { + val configurationName = configuration.name + val dependencies = configuration.dependencies + + when (configurationName) { + "KoverEngineConfig" -> { + configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true) + } + "kapt" -> { + configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true) + for (dependency in dependencies) { + when (dependency.group) { + "com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true) + "com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true) + } + } + } + API -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + IMPLEMENTATION -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + COMPILE -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + COMPILE_ONLY -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_ONLY_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + RUNTIME -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + RUNTIME_ONLY -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_ONLY_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + } + } catch (e: Throwable) { + // Ignore exceptions for FUS metrics + } + } + + configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1) + + + configurationTimeMetrics.put( + BooleanMetrics.KOTLIN_KTS_USED, + project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false + ) + + addTaskMetrics(project, configurationTimeMetrics) + + if (project.name == "buildSrc") { + configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1) + configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true) + } + } + configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) + + return configurationTimeMetrics +} + +private fun addTaskMetrics( + project: Project, + configurationTimeMetrics: MetricContainer, +) { + try { + val taskNames = project.tasks.names.toList() + configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong()) + configurationTimeMetrics.put( + NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS, + taskNames.count { name -> + try { + project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider") + } catch (_: Exception) { + true + } + }.toLong() + ) + } catch (e: Exception) { + //ignore exceptions for KT-62131. + } +} + +private fun collectAppliedPluginsStatistics( + project: Project, + configurationTimeMetrics: MetricContainer, +) { + for (plugin in ObservablePlugins.values()) { + project.plugins.withId(plugin.title) { + configurationTimeMetrics.put(plugin.metric, true) + } + } +} + +private fun reportLibrariesVersions( + configurationTimeMetrics: MetricContainer, + dependencies: DependencySet?, +) { + dependencies?.filter { it !is ProjectDependency }?.forEach { dependency -> + when { + dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_SPRING_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group?.startsWith("com.vaadin") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_VAADIN_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group?.startsWith("com.google.gwt") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_GWT_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group?.startsWith("org.hibernate") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_HIBERNATE_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlin" && dependency.name.startsWith("kotlin-stdlib") -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_STDLIB_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlinx" && dependency.name == "kotlinx-coroutines" -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_COROUTINES_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlin" && dependency.name == "kotlin-reflect" -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_REFLECT_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlinx" && dependency.name + .startsWith("kotlinx-serialization-runtime") -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_SERIALIZATION_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "com.android.tools.build" && dependency.name.startsWith("gradle") -> configurationTimeMetrics.put( + StringMetrics.ANDROID_GRADLE_PLUGIN_VERSION, + dependency.version ?: "0.0.0" + ) + } + } +} + +internal fun reportGlobalMetrics(logger: Logger, metricConsumer: StatisticsValuesConsumer) { + runMetricMethodSafely(logger, "reportGlobalMetrics") { + System.getProperty("os.name")?.also { metricConsumer.report(StringMetrics.OS_TYPE, System.getProperty("os.name")) } + metricConsumer.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong()) + metricConsumer.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null) + metricConsumer.report(NumericalMetrics.GRADLE_DAEMON_HEAP_SIZE, Runtime.getRuntime().maxMemory()) + metricConsumer.report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, DaemonReuseCounter.incrementAndGetOrdinal()) + } +} + +internal fun reportBuildFinished( + logger: Logger, + buildFailed: Boolean, + buildStartedTime: Long?, + projectEvaluatedTime: Long?, + metricsContainer: NonSynchronizedMetricsContainer, +) { + runMetricMethodSafely(logger, "reportBuildFinish") { + val finishTime = System.currentTimeMillis() + if (buildStartedTime != null) { + metricsContainer.report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - buildStartedTime) + } + if (projectEvaluatedTime != null) { + metricsContainer.report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - projectEvaluatedTime) + } + metricsContainer.report(NumericalMetrics.BUILD_FINISH_TIME, finishTime) + metricsContainer.report(BooleanMetrics.BUILD_FAILED, buildFailed) + } +} + +internal fun getMetricToReport(task: String) = when (task.substringAfterLast(":")) { + "dokkaHtml" -> BooleanMetrics.ENABLED_DOKKA_HTML + "dokkaGfm" -> BooleanMetrics.ENABLED_DOKKA_GFM + "dokkaJavadoc" -> BooleanMetrics.ENABLED_DOKKA_JAVADOC + "dokkaJekyll" -> BooleanMetrics.ENABLED_DOKKA_JEKYLL + "dokkaHtmlMultiModule" -> BooleanMetrics.ENABLED_DOKKA_HTML_MULTI_MODULE + "dokkaGfmMultiModule" -> BooleanMetrics.ENABLED_DOKKA_GFM_MULTI_MODULE + "dokkaJekyllMultiModule" -> BooleanMetrics.ENABLED_DOKKA_JEKYLL_MULTI_MODULE + "dokkaHtmlCollector" -> BooleanMetrics.ENABLED_DOKKA_HTML_COLLECTOR + "dokkaGfmCollector" -> BooleanMetrics.ENABLED_DOKKA_GFM_COLLECTOR + "dokkaJavadocCollector" -> BooleanMetrics.ENABLED_DOKKA_JAVADOC_COLLECTOR + "dokkaJekyllCollector" -> BooleanMetrics.ENABLED_DOKKA_JEKYLL_COLLECTOR + else -> null +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/Pre232IdeaKotlinBuildStatsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/Pre232IdeaKotlinBuildStatsBeanService.kt similarity index 80% rename from libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/Pre232IdeaKotlinBuildStatsService.kt rename to libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/Pre232IdeaKotlinBuildStatsBeanService.kt index 5bfdc0a1065..cd20bcf1b34 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/Pre232IdeaKotlinBuildStatsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/old/Pre232IdeaKotlinBuildStatsBeanService.kt @@ -6,27 +6,26 @@ package org.jetbrains.kotlin.gradle.plugin.statistics.old import org.gradle.api.Project -import org.jetbrains.kotlin.gradle.plugin.statistics.AbstractKotlinBuildStatsService -import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatHandler +import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsBeanService import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics import javax.management.ObjectName -internal class Pre232IdeaKotlinBuildStatsService internal constructor( +internal class Pre232IdeaKotlinBuildStatsBeanService internal constructor( project: Project, beanName: ObjectName -) : AbstractKotlinBuildStatsService(project, beanName), +) : KotlinBuildStatsBeanService(project, beanName), Pre232IdeaKotlinBuildStatsMXBean { override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?): Boolean = - KotlinBuildStatHandler().report(sessionLogger, metric, value, subprojectName, weight) + report(kotlinBuildLogger.sessionLogger, metric, value, subprojectName, weight) override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?): Boolean = - KotlinBuildStatHandler().report(sessionLogger, metric, value, subprojectName, weight) + report(kotlinBuildLogger.sessionLogger, metric, value, subprojectName, weight) override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?): Boolean = - KotlinBuildStatHandler().report(sessionLogger, metric, value, subprojectName, weight) + report(kotlinBuildLogger.sessionLogger, metric, value, subprojectName, weight) override fun reportBoolean(name: String, value: Boolean, subprojectName: String?, weight: Long?): Boolean = report(BooleanMetrics.valueOf(name), value, subprojectName, weight) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/reportUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/reportUtils.kt index 63cffc5f5cf..f974727388d 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/reportUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/reportUtils.kt @@ -5,25 +5,40 @@ package org.jetbrains.kotlin.gradle.utils import org.gradle.api.Project -import org.gradle.api.provider.Provider +import org.gradle.api.logging.Logger import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFusService import org.jetbrains.kotlin.gradle.plugin.statistics.MetricContainer +import org.jetbrains.kotlin.statistics.MetricValueValidationFailed /** * Invokes build FUS service if the reporting service is initialised and add configuration time metrics. * - * BuildFusService.reportFusMetrics should be called for execution time metrics + * New value will be present in configuration cache if BuildFusService.Parameters are not calculated yet + * [BuildFusService.reportFusMetrics] should be called for execution time metrics */ internal fun Project.addConfigurationMetrics(reportAction: (MetricContainer) -> Unit) { project.gradle.sharedServices.registrations.findByName(BuildFusService.serviceName)?.also { - @Suppress("UNCHECKED_CAST") - (it.service as Provider).also { - it.get().parameters.configurationMetrics.add(project.provider { + with(it.parameters as BuildFusService.Parameters) { + configurationMetrics.add(project.provider { val configurationTimeMetrics = MetricContainer() reportAction(configurationTimeMetrics) configurationTimeMetrics }) } } -} \ No newline at end of file +} + +internal fun runMetricMethodSafely(logger: Logger, methodName: String, action: () -> T?): T? { + return try { + logger.debug("Executing [$methodName]") + action.invoke() + } catch (e: MetricValueValidationFailed) { + throw e + } catch (e: Throwable) { + val description = "Could not execute [$methodName]" + logger.info(description) + logger.debug(e.message, e) + null + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildStatServiceTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildStatServiceTest.kt deleted file mode 100644 index 042976f34d8..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildStatServiceTest.kt +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.statistics - -import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics -import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics -import org.jetbrains.kotlin.statistics.metrics.StringMetrics -import org.junit.Test -import java.lang.management.ManagementFactory -import java.util.concurrent.atomic.AtomicInteger -import javax.management.MBeanServer -import javax.management.StandardMBean -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -class BuildStatServiceTest { - - @Test - fun testJmxDoesNotFail() { - //Test that JMX service does not throw exception even if JMX beans are not configured - val beanName = KotlinBuildStatsServicesRegistry.getBeanName(KotlinBuildStatsServicesRegistry.DEFAULT_SERVICE_QUALIFIER) - val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() - - val jmxService = JMXKotlinBuildStatsService(mbs, beanName) - jmxService.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3") - jmxService.report(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 10) - jmxService.report(BooleanMetrics.ENABLED_DATABINDING, true) - } - - @Test - fun testJmxServerWorks() { - val callsCount = AtomicInteger(0) - val instance = object : KotlinBuildStatsMXBean { - override fun reportBoolean(name: String, value: Boolean, subprojectName: String?, weight: Long?): Boolean { - callsCount.incrementAndGet() - return true - } - - override fun reportNumber(name: String, value: Long, subprojectName: String?, weight: Long?): Boolean { - callsCount.incrementAndGet() - return true - } - - override fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean { - callsCount.incrementAndGet() - return true - } - - } - - val beanName = KotlinBuildStatsServicesRegistry.getBeanName(KotlinBuildStatsServicesRegistry.DEFAULT_SERVICE_QUALIFIER) - val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() - mbs.registerMBean(StandardMBean(instance, KotlinBuildStatsMXBean::class.java), beanName) - - try { - val jmxService = JMXKotlinBuildStatsService(mbs, beanName) - assertTrue(jmxService.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3")) - assertTrue(jmxService.report(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 10)) - assertTrue(jmxService.report(BooleanMetrics.ENABLED_DATABINDING, true)) - assertEquals(3, callsCount.get()) - } finally { - mbs.unregisterMBean(beanName) - } - - } -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/statistics/BuildSessionLoggerTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/statistics/BuildSessionLoggerTest.kt index b1fb9090b61..9817cf51a99 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/statistics/BuildSessionLoggerTest.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/statistics/BuildSessionLoggerTest.kt @@ -43,18 +43,18 @@ class BuildSessionLoggerTest { //check that starting session creates a new log file assertEquals(false, logger1.isBuildSessionStarted()) - logger1.startBuildSession(0, 0) + logger1.startBuildSession("build_1") // check that file locks work properly and do not use the same file - logger2.startBuildSession(0, 0) + logger2.startBuildSession("build_2") assertEquals(true, logger1.isBuildSessionStarted()) - logger1.finishBuildSession("", false, buildId = "1") + logger1.finishBuildSession() // created single folder with kotlin statistics assertEquals(1, statFilesCount()) - logger2.finishBuildSession("", false, buildId = "2") + logger2.finishBuildSession() assertEquals(2, statFilesCount()) rootFolder.listFiles()?.first()?.listFiles()?.forEach { file -> @@ -69,8 +69,8 @@ class BuildSessionLoggerTest { fun testDeleteExtraFiles() { val maxFiles = 100 val logger = BuildSessionLogger(rootFolder, maxFiles) - logger.startBuildSession(0, null) - logger.finishBuildSession("", false, buildId = "1") + logger.startBuildSession("buildId_1") + logger.finishBuildSession() assertEquals(1, statFilesCount()) val statsFolder = rootFolder.listFiles()?.single() ?: fail("${rootFolder.absolutePath} was not created") @@ -81,8 +81,8 @@ class BuildSessionLoggerTest { File(statsFolder, "${UUID.randomUUID()}.profile").createNewFile() } - logger.startBuildSession(0, 0) - logger.finishBuildSession("", false, buildId = "1") + logger.startBuildSession("buildId_1") + logger.finishBuildSession() assertTrue( statsFolder.listFiles()?.count { it.name == singleStatFile.name } == 1, @@ -109,22 +109,21 @@ class BuildSessionLoggerTest { val logger = BuildSessionLogger(rootFolder) logger.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3.4-snapshot") - val startTime = System.currentTimeMillis() - 1001 - logger.startBuildSession(1, startTime) + logger.startBuildSession(buildId) val reportFile = rootFolder.resolve(BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME) .resolve(buildId + BuildSessionLogger.Companion.PROFILE_FILE_NAME_SUFFIX) reportFile.createNewFile() reportFile.appendText("${StringMetrics.USE_FIR.name}=true\n") - logger.finishBuildSession("Build", false, buildId) + logger.finishBuildSession() assertEquals(1, statFilesCount()) val statFile = rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file") statFile.appendBytes("break format of the file".toByteArray()) //this line should be filtered by MetricsContainer.readFromFile - logger.startBuildSession(1, startTime) - logger.finishBuildSession("", false, buildId) + logger.startBuildSession(buildId) + logger.finishBuildSession() val metrics = ArrayList() MetricsContainer.readFromFile(statFile) { @@ -146,15 +145,12 @@ class BuildSessionLoggerTest { null, metrics[1].getMetric(StringMetrics.KOTLIN_COMPILER_VERSION)?.getValue() ) - - val buildDuration = metrics[0].getMetric(NumericalMetrics.GRADLE_BUILD_DURATION)?.getValue() ?: 0L - assertTrue(buildDuration > 1000, "It was expected that build duration is > 1000, but got $buildDuration") } @Test fun testSaveAndReadAllMetrics() { val logger = BuildSessionLogger(rootFolder) - logger.startBuildSession(1, 1) + logger.startBuildSession("test") for (metric in StringMetrics.values()) { logger.report(metric, "value") logger.report(metric, metric.name) @@ -165,7 +161,7 @@ class BuildSessionLoggerTest { for (metric in NumericalMetrics.values()) { logger.report(metric, System.currentTimeMillis()) } - logger.finishBuildSession("Build", false, buildId = "test") + logger.finishBuildSession() MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) { for (metric in StringMetrics.values()) { @@ -183,12 +179,12 @@ class BuildSessionLoggerTest { @Test fun testWeight() { val logger = BuildSessionLogger(rootFolder) - logger.startBuildSession(1, 1) + logger.startBuildSession("build_1") logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 10, null, 9) logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 100, null, 1) - logger.finishBuildSession("Build", false, buildId = "1") + logger.finishBuildSession() MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) { assertEquals(19L, it.getMetric(NumericalMetrics.ANALYSIS_LINES_PER_SECOND)?.getValue()) } diff --git a/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSession.kt b/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSession.kt index dff2adc093c..8f3a80a18db 100644 --- a/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSession.kt +++ b/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSession.kt @@ -5,6 +5,4 @@ package org.jetbrains.kotlin.statistics -class BuildSession(val buildStartedTime: Long?) { - val projectEvaluatedTime = System.currentTimeMillis() -} +class BuildSession(val buildUid: String) diff --git a/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt b/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt index 01b93c4e786..4a55debd11f 100644 --- a/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt +++ b/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt @@ -46,15 +46,16 @@ class BuildSessionLogger( private val metricsContainer = MetricsContainer(forceValuesValidation) @Synchronized - fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) { - buildSession = BuildSession(buildStartedTime) - report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart) - + fun startBuildSession(buildUid: String) { + buildSession = BuildSession(buildUid) } @Synchronized fun isBuildSessionStarted() = buildSession != null + @Synchronized + fun getActiveBuildId() = buildSession?.buildUid + /** * Initializes a new build report file * The following contracts are implemented: @@ -94,28 +95,12 @@ class BuildSessionLogger( @Synchronized - fun finishBuildSession( - @Suppress("UNUSED_PARAMETER") action: String?, - buildFailed: Boolean, - buildId: String, - ) { - try { - // nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration - // could be affected by system clock correction - val finishTime = System.currentTimeMillis() - buildSession?.also { - if (it.buildStartedTime != null) { - report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime) - } - report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime) - report(NumericalMetrics.BUILD_FINISH_TIME, finishTime) - report(BooleanMetrics.BUILD_FAILED, buildFailed) - } - } finally { - buildSession = null - storeMetricsIntoFile(buildId) - clearOldFiles() + fun finishBuildSession() { + buildSession?.also { + storeMetricsIntoFile(it.buildUid) } + buildSession = null + clearOldFiles() } override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) = diff --git a/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt b/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt index 59fbb162c00..15d3429e079 100644 --- a/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt +++ b/libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt @@ -150,6 +150,7 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta allMetrics.putAll(booleanMetrics) allMetrics.putAll(stringMetrics) } + writer.appendLine() for (entry in allMetrics.entries) { val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}" writer.appendLine("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")