Remove BuildStatsService.applyIfInitialized outside BuildFlowService

#KT-58768 In Progress
This commit is contained in:
Nataliya.Valtman
2023-10-25 13:07:49 +02:00
committed by Space Team
parent 62ebb9932f
commit c332e1beb5
21 changed files with 454 additions and 364 deletions
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.gradle package org.jetbrains.kotlin.gradle
import org.gradle.util.GradleVersion import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.gradle.report.BuildReportType
import org.jetbrains.kotlin.gradle.testbase.* import org.jetbrains.kotlin.gradle.testbase.*
import org.jetbrains.kotlin.gradle.util.replaceText import org.jetbrains.kotlin.gradle.util.replaceText
import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.DisplayName
@@ -175,12 +176,14 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
project( project(
"incrementalMultiproject", gradleVersion, "incrementalMultiproject", gradleVersion,
) { ) {
//after KT-58768 and KT-62616 are done it will be possible to check build metrics also //Collect metrics from BuildMetricsService also
build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") { build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath",
buildOptions = defaultBuildOptions.copy(buildReport = listOf(BuildReportType.FILE))) {
assertFileContains( assertFileContains(
fusStatisticsPath, fusStatisticsPath,
"CONFIGURATION_IMPLEMENTATION_COUNT=2", "CONFIGURATION_IMPLEMENTATION_COUNT=2",
"NUMBER_OF_SUBPROJECTS=2", "NUMBER_OF_SUBPROJECTS=2",
"COMPILATIONS_COUNT=2"
) )
} }
} }
@@ -17,6 +17,7 @@ import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.build.report.metrics.* import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
import org.jetbrains.kotlin.gradle.tasks.* import org.jetbrains.kotlin.gradle.tasks.*
import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import java.io.File import java.io.File
import javax.inject.Inject import javax.inject.Inject
@@ -29,7 +30,8 @@ internal class GradleCompilerRunnerWithWorkers(
compilerExecutionSettings: CompilerExecutionSettings, compilerExecutionSettings: CompilerExecutionSettings,
buildMetrics: BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric>, buildMetrics: BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric>,
private val workerExecutor: WorkerExecutor, private val workerExecutor: WorkerExecutor,
) : GradleCompilerRunner(taskProvider, jdkToolsJar, compilerExecutionSettings, buildMetrics) { fusMetricsConsumer: StatisticsValuesConsumer?,
) : GradleCompilerRunner(taskProvider, jdkToolsJar, compilerExecutionSettings, buildMetrics, fusMetricsConsumer) {
override fun runCompilerAsync( override fun runCompilerAsync(
workArgs: GradleKotlinCompilerWorkArguments, workArgs: GradleKotlinCompilerWorkArguments,
taskOutputsBackup: TaskOutputsBackup?, taskOutputsBackup: TaskOutputsBackup?,
@@ -36,13 +36,13 @@ import org.jetbrains.kotlin.gradle.plugin.internal.BuildIdService
import org.jetbrains.kotlin.gradle.plugin.internal.JavaSourceSetsAccessor import org.jetbrains.kotlin.gradle.plugin.internal.JavaSourceSetsAccessor
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.plugin.variantImplementationFactory import org.jetbrains.kotlin.gradle.plugin.variantImplementationFactory
import org.jetbrains.kotlin.gradle.tasks.* import org.jetbrains.kotlin.gradle.tasks.*
import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.incremental.IncrementalModuleEntry import org.jetbrains.kotlin.incremental.IncrementalModuleEntry
import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import org.jetbrains.kotlin.statistics.metrics.StringMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import java.io.File import java.io.File
import java.lang.ref.WeakReference import java.lang.ref.WeakReference
@@ -68,6 +68,7 @@ internal fun createGradleCompilerRunner(
cachedClassLoadersService: Property<ClassLoadersCachingBuildService>, cachedClassLoadersService: Property<ClassLoadersCachingBuildService>,
buildFinishedListenerService: Provider<BuildFinishedListenerService>, buildFinishedListenerService: Provider<BuildFinishedListenerService>,
buildIdService: Provider<BuildIdService>, buildIdService: Provider<BuildIdService>,
fusMetricsConsumer: StatisticsValuesConsumer?,
): GradleCompilerRunner { ): GradleCompilerRunner {
return if (runViaBuildToolsApi) { return if (runViaBuildToolsApi) {
GradleBuildToolsApiCompilerRunner( GradleBuildToolsApiCompilerRunner(
@@ -79,6 +80,7 @@ internal fun createGradleCompilerRunner(
cachedClassLoadersService, cachedClassLoadersService,
buildFinishedListenerService, buildFinishedListenerService,
buildIdService, buildIdService,
fusMetricsConsumer
) )
} else { } else {
GradleCompilerRunnerWithWorkers( GradleCompilerRunnerWithWorkers(
@@ -87,6 +89,7 @@ internal fun createGradleCompilerRunner(
compilerExecutionSettings, compilerExecutionSettings,
buildMetricsReporter, buildMetricsReporter,
workerExecutor, workerExecutor,
fusMetricsConsumer
) )
} }
} }
@@ -101,6 +104,7 @@ internal open class GradleCompilerRunner(
protected val jdkToolsJar: File?, protected val jdkToolsJar: File?,
protected val compilerExecutionSettings: CompilerExecutionSettings, protected val compilerExecutionSettings: CompilerExecutionSettings,
protected val buildMetrics: BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric>, protected val buildMetrics: BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric>,
protected val fusMetricsConsumer: StatisticsValuesConsumer?,
) { ) {
internal val pathProvider = taskProvider.path.get() internal val pathProvider = taskProvider.path.get()
@@ -120,7 +124,7 @@ internal open class GradleCompilerRunner(
args: K2JVMCompilerArguments, args: K2JVMCompilerArguments,
environment: GradleCompilerEnvironment, environment: GradleCompilerEnvironment,
jdkHome: File, jdkHome: File,
taskOutputsBackup: TaskOutputsBackup? taskOutputsBackup: TaskOutputsBackup?,
): WorkQueue? { ): WorkQueue? {
if (args.jdkHome == null && !args.noJdk) args.jdkHome = jdkHome.absolutePath if (args.jdkHome == null && !args.noJdk) args.jdkHome = jdkHome.absolutePath
loggerProvider.kotlinInfo("Kotlin compilation 'jdkHome' argument: ${args.jdkHome}") loggerProvider.kotlinInfo("Kotlin compilation 'jdkHome' argument: ${args.jdkHome}")
@@ -134,7 +138,7 @@ internal open class GradleCompilerRunner(
fun runJsCompilerAsync( fun runJsCompilerAsync(
args: K2JSCompilerArguments, args: K2JSCompilerArguments,
environment: GradleCompilerEnvironment, environment: GradleCompilerEnvironment,
taskOutputsBackup: TaskOutputsBackup? taskOutputsBackup: TaskOutputsBackup?,
): WorkQueue? { ): WorkQueue? {
return runCompilerAsync(KotlinCompilerClass.JS, args, environment, taskOutputsBackup) return runCompilerAsync(KotlinCompilerClass.JS, args, environment, taskOutputsBackup)
} }
@@ -145,7 +149,7 @@ internal open class GradleCompilerRunner(
*/ */
fun runMetadataCompilerAsync( fun runMetadataCompilerAsync(
args: K2MetadataCompilerArguments, args: K2MetadataCompilerArguments,
environment: GradleCompilerEnvironment environment: GradleCompilerEnvironment,
): WorkQueue? { ): WorkQueue? {
return runCompilerAsync(KotlinCompilerClass.METADATA, args, environment) return runCompilerAsync(KotlinCompilerClass.METADATA, args, environment)
} }
@@ -154,7 +158,7 @@ internal open class GradleCompilerRunner(
compilerClassName: String, compilerClassName: String,
compilerArgs: CommonCompilerArguments, compilerArgs: CommonCompilerArguments,
environment: GradleCompilerEnvironment, environment: GradleCompilerEnvironment,
taskOutputsBackup: TaskOutputsBackup? = null taskOutputsBackup: TaskOutputsBackup? = null,
): WorkQueue? { ): WorkQueue? {
if (compilerArgs.version) { if (compilerArgs.version) {
loggerProvider.lifecycle( loggerProvider.lifecycle(
@@ -168,42 +172,39 @@ internal open class GradleCompilerRunner(
// compilerArgs arguments may have some attributes which are overrided by freeCompilerArguments. // compilerArgs arguments may have some attributes which are overrided by freeCompilerArguments.
// Here we perform the work which is repeated in compiler in order to obtain correct values. This extra work could be avoided when // Here we perform the work which is repeated in compiler in order to obtain correct values. This extra work could be avoided when
// compiler would report metrics by itself via JMX // compiler would report metrics by itself via JMX
KotlinBuildStatsService.applyIfInitialised { fusMetricsConsumer?.let { metricsConsumer ->
when (compilerArgs) { when (compilerArgs) {
is K2JVMCompilerArguments -> { is K2JVMCompilerArguments -> {
KotlinBuildStatsService.getInstance()?.apply { val args = K2JVMCompilerArguments()
val args = K2JVMCompilerArguments() parseCommandLineArguments(argsArray.toList(), args)
parseCommandLineArguments(argsArray.toList(), args) metricsConsumer.report(StringMetrics.JVM_DEFAULTS, args.jvmDefault)
report(StringMetrics.JVM_DEFAULTS, args.jvmDefault) metricsConsumer.report(StringMetrics.USE_FIR, args.useK2.toString())
report(StringMetrics.USE_FIR, args.useK2.toString())
val pluginPatterns = listOf(Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_ALL_OPEN, "kotlin-allopen-.*jar"), val pluginPatterns = listOf(
Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_NO_ARG, "kotlin-noarg-.*jar"), Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_ALL_OPEN, "kotlin-allopen-.*jar"),
Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_SAM_WITH_RECEIVER, "kotlin-sam-with-receiver-.*jar"), Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_NO_ARG, "kotlin-noarg-.*jar"),
Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_LOMBOK, "kotlin-lombok-.*jar"), Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_SAM_WITH_RECEIVER, "kotlin-sam-with-receiver-.*jar"),
Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_PARSELIZE, "kotlin-parcelize-compiler-.*jar"), Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_LOMBOK, "kotlin-lombok-.*jar"),
Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_ATOMICFU, "atomicfu-.*jar") Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_PARSELIZE, "kotlin-parcelize-compiler-.*jar"),
) Pair(BooleanMetrics.ENABLED_COMPILER_PLUGIN_ATOMICFU, "atomicfu-.*jar")
val pluginJars = args.pluginClasspaths?.map { it.replace("\\", "/").split("/").last() } )
if (pluginJars != null) { val pluginJars = args.pluginClasspaths?.map { it.replace("\\", "/").split("/").last() }
for (pluginPattern in pluginPatterns) { if (pluginJars != null) {
if (pluginJars.any { it.matches(pluginPattern.second.toRegex())}) { for (pluginPattern in pluginPatterns) {
report(pluginPattern.first, true) if (pluginJars.any { it.matches(pluginPattern.second.toRegex()) }) {
} metricsConsumer.report(pluginPattern.first, true)
} }
} }
} }
} }
is K2JSCompilerArguments -> { is K2JSCompilerArguments -> {
KotlinBuildStatsService.getInstance()?.apply { val args = K2JSCompilerArguments()
val args = K2JSCompilerArguments() parseCommandLineArguments(argsArray.toList(), args)
parseCommandLineArguments(argsArray.toList(), args) if (!args.isPreIrBackendDisabled() || args.irProduceJs) {
if (!args.isPreIrBackendDisabled() || args.irProduceJs) { metricsConsumer.report(BooleanMetrics.JS_SOURCE_MAP, args.sourceMap)
report(BooleanMetrics.JS_SOURCE_MAP, args.sourceMap) }
} if (args.irProduceJs) {
if (args.irProduceJs) { metricsConsumer.report(StringMetrics.JS_PROPERTY_LAZY_INITIALIZATION, args.irPropertyLazyInitialization.toString())
report(StringMetrics.JS_PROPERTY_LAZY_INITIALIZATION, args.irPropertyLazyInitialization.toString())
}
} }
} }
} }
@@ -245,7 +246,7 @@ internal open class GradleCompilerRunner(
protected open fun runCompilerAsync( protected open fun runCompilerAsync(
workArgs: GradleKotlinCompilerWorkArguments, workArgs: GradleKotlinCompilerWorkArguments,
taskOutputsBackup: TaskOutputsBackup? taskOutputsBackup: TaskOutputsBackup?,
): WorkQueue? { ): WorkQueue? {
try { try {
buildMetrics.addTimeMetric(GradleBuildPerformanceMetric.CALL_WORKER) buildMetrics.addTimeMetric(GradleBuildPerformanceMetric.CALL_WORKER)
@@ -272,7 +273,7 @@ internal open class GradleCompilerRunner(
compilerFullClasspath: List<File>, compilerFullClasspath: List<File>,
messageCollector: MessageCollector, messageCollector: MessageCollector,
daemonJvmArgs: List<String>?, daemonJvmArgs: List<String>?,
isDebugEnabled: Boolean isDebugEnabled: Boolean,
): CompileServiceSession? { ): CompileServiceSession? {
val compilerId = CompilerId.makeCompilerId(compilerFullClasspath) val compilerId = CompilerId.makeCompilerId(compilerFullClasspath)
val daemonJvmOptions = configureDaemonJVMOptions( val daemonJvmOptions = configureDaemonJVMOptions(
@@ -347,7 +348,10 @@ internal open class GradleCompilerRunner(
nameToModules.getOrPut(module.name) { HashSet() }.add(module) nameToModules.getOrPut(module.name) { HashSet() }.add(module)
if (task is Kotlin2JsCompile) { if (task is Kotlin2JsCompile) {
(jarForJavaSourceSet(project, task.sourceSetName.get()) ?: jarForSingleTargetJs(project, task.sourceSetName.get()))?.let { (jarForJavaSourceSet(project, task.sourceSetName.get()) ?: jarForSingleTargetJs(
project,
task.sourceSetName.get()
))?.let {
jarToModule[it] = module jarToModule[it] = module
} }
} }
@@ -404,7 +408,7 @@ internal open class GradleCompilerRunner(
private fun jarForJavaSourceSet( private fun jarForJavaSourceSet(
project: Project, project: Project,
sourceSetName: String sourceSetName: String,
): File? { ): File? {
val sourceSets = project.variantImplementationFactory<JavaSourceSetsAccessor.JavaSourceSetsAccessorVariantFactory>() val sourceSets = project.variantImplementationFactory<JavaSourceSetsAccessor.JavaSourceSetsAccessorVariantFactory>()
.getInstance(project) .getInstance(project)
@@ -9,7 +9,6 @@ import org.gradle.api.provider.Provider
import org.gradle.workers.WorkQueue import org.gradle.workers.WorkQueue
import org.gradle.workers.WorkerExecutor import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
import org.jetbrains.kotlin.compilerRunner.CompilerExecutionSettings import org.jetbrains.kotlin.compilerRunner.CompilerExecutionSettings
@@ -20,6 +19,7 @@ import org.jetbrains.kotlin.gradle.plugin.BuildFinishedListenerService
import org.jetbrains.kotlin.gradle.plugin.internal.BuildIdService import org.jetbrains.kotlin.gradle.plugin.internal.BuildIdService
import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider
import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup
import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import java.io.File import java.io.File
internal class GradleBuildToolsApiCompilerRunner( internal class GradleBuildToolsApiCompilerRunner(
@@ -31,7 +31,8 @@ internal class GradleBuildToolsApiCompilerRunner(
private val cachedClassLoadersService: Provider<ClassLoadersCachingBuildService>, private val cachedClassLoadersService: Provider<ClassLoadersCachingBuildService>,
private val buildFinishedListenerService: Provider<BuildFinishedListenerService>, private val buildFinishedListenerService: Provider<BuildFinishedListenerService>,
private val buildIdService: Provider<BuildIdService>, private val buildIdService: Provider<BuildIdService>,
) : GradleCompilerRunner(taskProvider, jdkToolsJar, compilerExecutionSettings, buildMetrics) { fusMetricsConsumer: StatisticsValuesConsumer?,
) : GradleCompilerRunner(taskProvider, jdkToolsJar, compilerExecutionSettings, buildMetrics, fusMetricsConsumer) {
override fun runCompilerAsync( override fun runCompilerAsync(
@@ -33,8 +33,7 @@ import org.jetbrains.kotlin.gradle.plugin.internal.*
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinUsages import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinUsages
import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory
import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFlowService import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFusService
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.report.BuildMetricsService import org.jetbrains.kotlin.gradle.report.BuildMetricsService
import org.jetbrains.kotlin.gradle.targets.js.KotlinJsCompilerAttribute import org.jetbrains.kotlin.gradle.targets.js.KotlinJsCompilerAttribute
import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin
@@ -48,7 +47,6 @@ import org.jetbrains.kotlin.gradle.targets.native.internal.CommonizerTargetAttri
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompileTool import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompileTool
import org.jetbrains.kotlin.gradle.testing.internal.KotlinTestsRegistry import org.jetbrains.kotlin.gradle.testing.internal.KotlinTestsRegistry
import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import org.jetbrains.kotlin.tooling.core.KotlinToolingVersion import org.jetbrains.kotlin.tooling.core.KotlinToolingVersion
import kotlin.reflect.KClass import kotlin.reflect.KClass
@@ -63,10 +61,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
override fun apply(project: Project) { override fun apply(project: Project) {
project.registerDefaultVariantImplementations() project.registerDefaultVariantImplementations()
KotlinBuildStatsService.getOrCreateInstance(project)?.apply { val buildFusService = BuildFusService.registerIfAbsent(project, pluginVersion)
report(StringMetrics.KOTLIN_COMPILER_VERSION, pluginVersion)
}
BuildFlowService.registerIfAbsent(project)
project.gradle.projectsEvaluated { project.gradle.projectsEvaluated {
whenBuildEvaluated(project) whenBuildEvaluated(project)
@@ -87,7 +82,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
kotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, pluginVersion) kotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, pluginVersion)
} }
BuildMetricsService.registerIfAbsent(project) BuildMetricsService.registerIfAbsent(project, buildFusService)
} }
private fun addKotlinCompilerConfiguration(project: Project) { private fun addKotlinCompilerConfiguration(project: Project) {
@@ -10,7 +10,7 @@ import org.gradle.api.flow.*
import org.gradle.api.provider.Property import org.gradle.api.provider.Property
import org.gradle.api.services.ServiceReference import org.gradle.api.services.ServiceReference
import org.gradle.api.tasks.Input import org.gradle.api.tasks.Input
import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFlowService import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFusService
import org.jetbrains.kotlin.gradle.report.BuildMetricsService import org.jetbrains.kotlin.gradle.report.BuildMetricsService
import org.jetbrains.kotlin.gradle.report.BuildScanExtensionHolder import org.jetbrains.kotlin.gradle.report.BuildScanExtensionHolder
import javax.inject.Inject import javax.inject.Inject
@@ -32,10 +32,7 @@ internal abstract class StatisticsBuildFlowManager @Inject constructor(
} }
} }
fun subscribeForBuildScan(project: Project) { fun subscribeForBuildScan(buildScanHolder: BuildScanExtensionHolder) {
val buildScanExtension = project.rootProject.extensions.findByName("buildScan")
val buildScanHolder = buildScanExtension?.let { BuildScanExtensionHolder(it) }
flowScope.always( flowScope.always(
BuildScanFlowAction::class.java BuildScanFlowAction::class.java
) { spec -> ) { spec ->
@@ -47,10 +44,10 @@ internal abstract class StatisticsBuildFlowManager @Inject constructor(
internal class BuildScanFlowAction : FlowAction<BuildScanFlowAction.Parameters> { internal class BuildScanFlowAction : FlowAction<BuildScanFlowAction.Parameters> {
interface Parameters : FlowParameters { interface Parameters : FlowParameters {
@get:ServiceReference @get:ServiceReference
val buildMetricService: Property<BuildMetricsService?> val buildMetricService: Property<BuildMetricsService>
@get: Input @get: Input
val buildScanExtensionHolder: Property<BuildScanExtensionHolder?> val buildScanExtensionHolder: Property<BuildScanExtensionHolder>
} }
override fun execute(parameters: Parameters) { override fun execute(parameters: Parameters) {
@@ -61,7 +58,7 @@ internal class BuildScanFlowAction : FlowAction<BuildScanFlowAction.Parameters>
internal class BuildFinishFlowAction : FlowAction<BuildFinishFlowAction.Parameters> { internal class BuildFinishFlowAction : FlowAction<BuildFinishFlowAction.Parameters> {
interface Parameters : FlowParameters { interface Parameters : FlowParameters {
@get:ServiceReference @get:ServiceReference
val buildFlowServiceProperty: Property<BuildFlowService> val buildFusServiceProperty: Property<BuildFusService>
@get:Input @get:Input
val action: Property<String?> val action: Property<String?>
@@ -72,7 +69,7 @@ internal class BuildFinishFlowAction : FlowAction<BuildFinishFlowAction.Paramete
} }
override fun execute(parameters: Parameters) { override fun execute(parameters: Parameters) {
parameters.buildFlowServiceProperty.get().recordBuildFinished( parameters.buildFusServiceProperty.orNull?.recordBuildFinished(
parameters.action.orNull, parameters.buildFailed.get() parameters.action.orNull, parameters.buildFailed.get()
) )
} }
@@ -6,12 +6,14 @@
package org.jetbrains.kotlin.gradle.plugin.statistics package org.jetbrains.kotlin.gradle.plugin.statistics
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.logging.Logging import org.gradle.api.logging.Logging
import org.gradle.api.provider.ListProperty import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal
import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.FinishEvent
import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.OperationCompletionListener
import org.gradle.tooling.events.task.TaskFailureResult import org.gradle.tooling.events.task.TaskFailureResult
@@ -22,15 +24,21 @@ import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager
import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested
import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled
import org.jetbrains.kotlin.gradle.report.BuildReportType
import org.jetbrains.kotlin.gradle.report.reportingSettings import org.jetbrains.kotlin.gradle.report.reportingSettings
import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.IStatisticsValuesConsumer import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import java.io.Serializable import java.io.Serializable
internal abstract class BuildFlowService : BuildService<BuildFlowService.Parameters>, AutoCloseable, OperationCompletionListener {
internal interface UsesBuildFusService : Task {
@get:Internal
val buildFusService: Property<BuildFusService?>
}
abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoCloseable, OperationCompletionListener {
private var buildFailed: Boolean = false private var buildFailed: Boolean = false
private val log = Logging.getLogger(this.javaClass) private val log = Logging.getLogger(this.javaClass)
@@ -43,8 +51,20 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
val fusStatisticsAvailable: Property<Boolean> val fusStatisticsAvailable: Property<Boolean>
} }
internal val fusMetricsConsumer: NonSynchronizedMetricsContainer? by lazy {
// parameters should be already injected on this property access
if (parameters.fusStatisticsAvailable.get())
NonSynchronizedMetricsContainer()
else
null
}
internal fun reportFusMetrics(reportAction: (StatisticsValuesConsumer) -> Unit) {
fusMetricsConsumer?.let { reportAction(it) }
}
companion object { companion object {
private val serviceName = "${BuildFlowService::class.simpleName}_${BuildFlowService::class.java.classLoader.hashCode()}" private val serviceName = "${BuildFusService::class.simpleName}_${BuildFusService::class.java.classLoader.hashCode()}"
private fun fusStatisticsAvailable(project: Project): Boolean { private fun fusStatisticsAvailable(project: Project): Boolean {
return when { return when {
@@ -55,29 +75,42 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
else -> !project.isConfigurationCacheRequested else -> !project.isConfigurationCacheRequested
} }
} }
fun registerIfAbsent(
fun registerIfAbsent(project: Project, pluginVersion: String) = registerIfAbsentImpl(project, pluginVersion).also { serviceProvider ->
SingleActionPerProject.run(project, UsesBuildFusService::class.java.name) {
project.tasks.withType<UsesBuildFusService>().configureEach { task ->
task.buildFusService.value(serviceProvider).disallowChanges()
task.usesService(serviceProvider)
}
}
}
private fun registerIfAbsentImpl(
project: Project, project: Project,
): Provider<BuildFlowService> { pluginVersion: String,
): Provider<BuildFusService> {
val isProjectIsolationEnabled = project.isProjectIsolationEnabled val isProjectIsolationEnabled = project.isProjectIsolationEnabled
project.gradle.sharedServices.registrations.findByName(serviceName)?.let { project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
return (it.service as Provider<BuildFlowService>).also { return (it.service as Provider<BuildFusService>).also {
it.get().parameters.configurationMetrics.add(project.provider { it.get().parameters.configurationMetrics.add(project.provider {
KotlinBuildStatsService.getInstance()?.collectProjectConfigurationMetrics(project, isProjectIsolationEnabled) KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project, isProjectIsolationEnabled)
}) })
} }
} }
//init buildStatsService
KotlinBuildStatsService.getOrCreateInstance(project)
val fusStatisticsAvailable = fusStatisticsAvailable(project) val fusStatisticsAvailable = fusStatisticsAvailable(project)
val buildReportOutputs = reportingSettings(project).buildReportOutputs val buildReportOutputs = reportingSettings(project).buildReportOutputs
val buildScanReportEnabled = buildReportOutputs.contains(BuildReportType.BUILD_SCAN)
//Workaround for known issues for Gradle 8+: https://github.com/gradle/gradle/issues/24887: //Workaround for known issues for Gradle 8+: https://github.com/gradle/gradle/issues/24887:
// when this OperationCompletionListener is called services can be already closed for Gradle 8, // when this OperationCompletionListener is called services can be already closed for Gradle 8,
// so there is a change that no VariantImplementationFactory will be found // so there is a change that no VariantImplementationFactory will be found
return project.gradle.sharedServices.registerIfAbsent(serviceName, BuildFlowService::class.java) { spec -> return project.gradle.sharedServices.registerIfAbsent(serviceName, BuildFusService::class.java) { spec ->
if (fusStatisticsAvailable) { if (fusStatisticsAvailable) {
KotlinBuildStatsService.applyIfInitialised { KotlinBuildStatsService.applyIfInitialised {
it.recordProjectsEvaluated(project.gradle) it.recordProjectsEvaluated(project.gradle)
@@ -85,11 +118,11 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
} }
spec.parameters.configurationMetrics.add(project.provider { spec.parameters.configurationMetrics.add(project.provider {
KotlinBuildStatsService.getInstance()?.collectGeneralConfigurationMetrics(project, isProjectIsolationEnabled, buildReportOutputs) KotlinBuildStatHandler.collectGeneralConfigurationTimeMetrics(project, isProjectIsolationEnabled, buildReportOutputs, pluginVersion)
}) })
spec.parameters.configurationMetrics.add(project.provider { spec.parameters.configurationMetrics.add(project.provider {
KotlinBuildStatsService.getInstance()?.collectProjectConfigurationMetrics(project, isProjectIsolationEnabled) KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project, isProjectIsolationEnabled)
}) })
spec.parameters.fusStatisticsAvailable.set(fusStatisticsAvailable) spec.parameters.fusStatisticsAvailable.set(fusStatisticsAvailable)
}.also { buildService -> }.also { buildService ->
@@ -100,9 +133,6 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
else -> StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult() else -> StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult()
} }
} }
if (buildScanReportEnabled && GradleVersion.current().baseVersion >= GradleVersion.version("8.1")) {
StatisticsBuildFlowManager.getInstance(project).subscribeForBuildScan(project)
}
} }
} }
} }
@@ -123,26 +153,31 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
} }
internal fun recordBuildFinished(action: String?, buildFailed: Boolean) { internal fun recordBuildFinished(action: String?, buildFailed: Boolean) {
KotlinBuildStatsService.applyIfInitialised { fusMetricsConsumer?.let { metricsConsumer ->
it.recordBuildFinish(action, buildFailed, parameters.configurationMetrics.orElse(emptyList()).get()) KotlinBuildStatHandler.reportGlobalMetrics(metricsConsumer)
parameters.configurationMetrics.orElse(emptyList()).get().forEach { it.addToConsumer(metricsConsumer) }
KotlinBuildStatsService.applyIfInitialised {
it.recordBuildFinish(action, buildFailed, metricsConsumer)
}
} }
} }
} }
internal class MetricContainer : Serializable { class MetricContainer : Serializable {
private val numericalMetrics = HashMap<NumericalMetrics, Long>() private val numericalMetrics = HashMap<NumericalMetrics, Long>()
private val booleanMetrics = HashMap<BooleanMetrics, Boolean>() private val booleanMetrics = HashMap<BooleanMetrics, Boolean>()
private val stringMetrics = HashMap<StringMetrics, String>() private val stringMetrics = HashMap<StringMetrics, String>()
fun report(sessionLogger: IStatisticsValuesConsumer) { fun addToConsumer(metricsConsumer: StatisticsValuesConsumer) {
for ((key, value) in numericalMetrics) { for ((key, value) in numericalMetrics) {
sessionLogger.report(key, value) metricsConsumer.report(key, value)
} }
for ((key, value) in booleanMetrics) { for ((key, value) in booleanMetrics) {
sessionLogger.report(key, value) metricsConsumer.report(key, value)
} }
for ((key, value) in stringMetrics) { for ((key, value) in stringMetrics) {
sessionLogger.report(key, value) metricsConsumer.report(key, value)
} }
} }
@@ -16,6 +16,7 @@ import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import org.jetbrains.kotlin.statistics.MetricValueValidationFailed import org.jetbrains.kotlin.statistics.MetricValueValidationFailed
import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import java.lang.management.ManagementFactory import java.lang.management.ManagementFactory
import javax.management.MBeanServer import javax.management.MBeanServer
import javax.management.ObjectName import javax.management.ObjectName
@@ -44,6 +45,216 @@ class KotlinBuildStatHandler {
getLogger().debug(e.message, e) getLogger().debug(e.message, e)
} }
/**
* Collect general configuration metrics
*/
internal fun collectGeneralConfigurationTimeMetrics(
project: Project,
isProjectIsolationEnabled: Boolean,
buildReportOutputs: List<BuildReportType>,
pluginVersion: String,
): MetricContainer {
val configurationTimeMetrics = MetricContainer()
configurationTimeMetrics.put(StringMetrics.KOTLIN_COMPILER_VERSION, pluginVersion)
val statisticOverhead = measureTimeMillis {
buildReportOutputs.forEach {
when (it) {
BuildReportType.BUILD_SCAN -> configurationTimeMetrics.put(BooleanMetrics.BUILD_SCAN_BUILD_REPORT, true)
BuildReportType.FILE -> configurationTimeMetrics.put(BooleanMetrics.FILE_BUILD_REPORT, true)
BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true)
BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true)
BuildReportType.TRY_K2_CONSOLE -> {}//ignore
}
}
val gradle = project.gradle
configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
if (!isProjectIsolationEnabled) {
gradle.taskGraph.whenReady { taskExecutionGraph ->
val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct()
configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install"))
}
}
}
configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
return configurationTimeMetrics
}
/**
* Collect project's configuration metrics
*/
internal fun collectProjectConfigurationTimeMetrics(
project: Project,
isProjectIsolationEnabled: Boolean,
): MetricContainer {
val configurationTimeMetrics = MetricContainer()
if (isProjectIsolationEnabled) { //support project isolation - KT-58768
return configurationTimeMetrics
}
val statisticOverhead = measureTimeMillis {
collectAppliedPluginsStatistics(project, configurationTimeMetrics)
val configurations = project.configurations.asMap.values
for (configuration in configurations) {
try {
val configurationName = configuration.name
val dependencies = configuration.dependencies
when (configurationName) {
"KoverEngineConfig" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true)
}
"kapt" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true)
for (dependency in dependencies) {
when (dependency.group) {
"com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true)
"com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true)
}
}
}
API -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
IMPLEMENTATION -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_ONLY_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_ONLY_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
}
} catch (e: Throwable) {
// log?
}
}
configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1)
configurationTimeMetrics.put(
BooleanMetrics.KOTLIN_KTS_USED,
project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false
)
addTaskMetrics(project, configurationTimeMetrics)
if (project.name == "buildSrc") {
configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1)
configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true)
}
}
configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
return configurationTimeMetrics
}
private fun addTaskMetrics(
project: Project,
configurationTimeMetrics: MetricContainer,
) {
try {
val taskNames = project.tasks.names.toList()
configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong())
configurationTimeMetrics.put(
NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS,
taskNames.count { name ->
try {
project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider")
} catch (_: Exception) {
true
}
}.toLong()
)
} catch (e: Exception) {
//ignore exceptions for KT-62131.
}
}
private fun collectAppliedPluginsStatistics(
project: Project,
configurationTimeMetrics: MetricContainer,
) {
for (plugin in ObservablePlugins.values()) {
project.plugins.withId(plugin.title) {
configurationTimeMetrics.put(plugin.metric, true)
}
}
}
private fun reportLibrariesVersions(configurationTimeMetrics: MetricContainer, dependencies: DependencySet?) {
dependencies?.forEach { dependency ->
when {
dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_SPRING_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("com.vaadin") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_VAADIN_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("com.google.gwt") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_GWT_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("org.hibernate") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_HIBERNATE_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlin" && dependency.name.startsWith("kotlin-stdlib") -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_STDLIB_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlinx" && dependency.name == "kotlinx-coroutines" -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_COROUTINES_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlin" && dependency.name == "kotlin-reflect" -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_REFLECT_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlinx" && dependency.name
.startsWith("kotlinx-serialization-runtime") -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_SERIALIZATION_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "com.android.tools.build" && dependency.name.startsWith("gradle") -> configurationTimeMetrics.put(
StringMetrics.ANDROID_GRADLE_PLUGIN_VERSION,
dependency.version ?: "0.0.0"
)
}
}
}
internal fun reportGlobalMetrics(metricConsumer: StatisticsValuesConsumer) {
runSafe("${KotlinBuildStatHandler::class.java}.reportGlobalMetrics") {
System.getProperty("os.name")?.also { metricConsumer.report(StringMetrics.OS_TYPE, System.getProperty("os.name")) }
metricConsumer.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong())
metricConsumer.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null)
metricConsumer.report(NumericalMetrics.GRADLE_DAEMON_HEAP_SIZE, Runtime.getRuntime().maxMemory())
}
}
} }
fun buildFinished( fun buildFinished(
@@ -57,223 +268,18 @@ class KotlinBuildStatHandler {
} }
} }
fun reportGlobalMetrics(
sessionLogger: BuildSessionLogger,
) {
runSafe("${KotlinBuildStatHandler::class.java}.reportGlobalMetrics") {
System.getProperty("os.name")?.also { sessionLogger.report(StringMetrics.OS_TYPE, System.getProperty("os.name")) }
sessionLogger.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong())
sessionLogger.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null)
sessionLogger.report(NumericalMetrics.GRADLE_DAEMON_HEAP_SIZE, Runtime.getRuntime().maxMemory())
}
}
internal fun reportBuildFinished( internal fun reportBuildFinished(
sessionLogger: BuildSessionLogger, sessionLogger: BuildSessionLogger,
action: String?, action: String?,
buildFailed: Boolean, buildFailed: Boolean,
configurationMetrics: List<MetricContainer>, metrics: NonSynchronizedMetricsContainer,
) { ) {
runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") { runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") {
configurationMetrics.forEach { it.report(sessionLogger) } metrics.sendToConsumer(sessionLogger)
sessionLogger.finishBuildSession(action, buildFailed) sessionLogger.finishBuildSession(action, buildFailed)
} }
} }
internal fun collectGeneralConfigurationTimeMetrics(
project: Project,
sessionLogger: BuildSessionLogger,
isProjectIsolationEnabled: Boolean,
buildReportOutputs: List<BuildReportType>,
): MetricContainer {
val configurationTimeMetrics = MetricContainer()
val statisticOverhead = measureTimeMillis {
buildReportOutputs.forEach {
when (it) {
BuildReportType.BUILD_SCAN -> configurationTimeMetrics.put(BooleanMetrics.BUILD_SCAN_BUILD_REPORT, true)
BuildReportType.FILE -> configurationTimeMetrics.put(BooleanMetrics.FILE_BUILD_REPORT, true)
BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true)
BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true)
BuildReportType.TRY_K2_CONSOLE -> {}//ignore
}
}
val gradle = project.gradle
configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
if (!isProjectIsolationEnabled) {
gradle.taskGraph.whenReady { taskExecutionGraph ->
val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct()
configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install"))
}
}
}
sessionLogger.report(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
return configurationTimeMetrics
}
internal fun collectProjectConfigurationTimeMetrics(
project: Project,
sessionLogger: BuildSessionLogger,
isProjectIsolationEnabled: Boolean,
): MetricContainer {
val configurationTimeMetrics = MetricContainer()
if (isProjectIsolationEnabled) { //support project isolation - KT-58768
return configurationTimeMetrics
}
val statisticOverhead = measureTimeMillis {
collectAppliedPluginsStatistics(project, configurationTimeMetrics)
val configurations = project.configurations.asMap.values
for (configuration in configurations) {
try {
val configurationName = configuration.name
val dependencies = configuration.dependencies
when (configurationName) {
"KoverEngineConfig" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true)
}
"kapt" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true)
for (dependency in dependencies) {
when (dependency.group) {
"com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true)
"com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true)
}
}
}
API -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
IMPLEMENTATION -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_ONLY_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_ONLY_COUNT,1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
}
} catch (e: Throwable) {
// log?
}
}
configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1)
configurationTimeMetrics.put(
BooleanMetrics.KOTLIN_KTS_USED,
project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false
)
addTaskMetrics(project, configurationTimeMetrics)
if (project.name == "buildSrc") {
configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1)
configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true)
}
}
sessionLogger.report(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
return configurationTimeMetrics
}
private fun addTaskMetrics(
project: Project,
configurationTimeMetrics: MetricContainer,
) {
try {
val taskNames = project.tasks.names.toList()
configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong())
configurationTimeMetrics.put(
NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS,
taskNames.count { name ->
try {
project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider")
} catch (_: Exception) {
true
}
}.toLong()
)
} catch (e: Exception) {
//ignore exceptions for KT-62131.
}
}
private fun collectAppliedPluginsStatistics(
project: Project,
configurationTimeMetrics: MetricContainer,
) {
for (plugin in ObservablePlugins.values()) {
project.plugins.withId(plugin.title) {
configurationTimeMetrics.put(plugin.metric, true)
}
}
}
private fun reportLibrariesVersions(configurationTimeMetrics: MetricContainer, dependencies: DependencySet?) {
dependencies?.forEach { dependency ->
when {
dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_SPRING_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("com.vaadin") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_VAADIN_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("com.google.gwt") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_GWT_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("org.hibernate") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_HIBERNATE_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlin" && dependency.name.startsWith("kotlin-stdlib") -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_STDLIB_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlinx" && dependency.name == "kotlinx-coroutines" -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_COROUTINES_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlin" && dependency.name == "kotlin-reflect" -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_REFLECT_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "org.jetbrains.kotlinx" && dependency.name
.startsWith("kotlinx-serialization-runtime") -> configurationTimeMetrics.put(
StringMetrics.KOTLIN_SERIALIZATION_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group == "com.android.tools.build" && dependency.name.startsWith("gradle") -> configurationTimeMetrics.put(
StringMetrics.ANDROID_GRADLE_PLUGIN_VERSION,
dependency.version ?: "0.0.0"
)
}
}
}
internal fun report( internal fun report(
sessionLogger: BuildSessionLogger, sessionLogger: BuildSessionLogger,
metric: BooleanMetrics, metric: BooleanMetrics,
@@ -19,11 +19,10 @@ import org.jetbrains.kotlin.gradle.plugin.internal.ConfigurationTimePropertiesAc
import org.jetbrains.kotlin.gradle.plugin.internal.configurationTimePropertiesAccessor import org.jetbrains.kotlin.gradle.plugin.internal.configurationTimePropertiesAccessor
import org.jetbrains.kotlin.gradle.plugin.internal.usedAtConfigurationTime import org.jetbrains.kotlin.gradle.plugin.internal.usedAtConfigurationTime
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatHandler.Companion.runSafe import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatHandler.Companion.runSafe
import org.jetbrains.kotlin.gradle.report.BuildReportType
import org.jetbrains.kotlin.statistics.BuildSessionLogger import org.jetbrains.kotlin.statistics.BuildSessionLogger
import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.IStatisticsValuesConsumer import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import java.io.Closeable import java.io.Closeable
@@ -46,7 +45,7 @@ interface KotlinBuildStatsMXBean {
fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean fun reportString(name: String, value: String, subprojectName: String?, weight: Long?): Boolean
} }
internal abstract class KotlinBuildStatsService internal constructor() : IStatisticsValuesConsumer, Closeable { internal abstract class KotlinBuildStatsService internal constructor() : StatisticsValuesConsumer, Closeable {
companion object { companion object {
// Property name for disabling saving statistical information // Property name for disabling saving statistical information
private const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile" private const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile"
@@ -188,21 +187,7 @@ internal abstract class KotlinBuildStatsService internal constructor() : IStatis
override fun close() { override fun close() {
} }
open fun recordBuildFinish(action: String?, buildFailed: Boolean, metric: NonSynchronizedMetricsContainer) {}
/**
* Collects metrics at the end of a build
*/
open fun recordBuildFinish(action: String?, buildFailed: Boolean, configurationTimeMetrics: List<MetricContainer>) {}
/**
* Collect project's configuration metrics
*/
open fun collectProjectConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean): MetricContainer = MetricContainer()
/**
* Collect general configuration metrics
*/
open fun collectGeneralConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean, buildReportOutputs: List<BuildReportType>): MetricContainer = MetricContainer()
open fun recordProjectsEvaluated(gradle: Gradle) {} open fun recordProjectsEvaluated(gradle: Gradle) {}
} }
@@ -314,14 +299,7 @@ internal class DefaultKotlinBuildStatsService internal constructor(
report(StringMetrics.valueOf(name), value, subprojectName, weight) report(StringMetrics.valueOf(name), value, subprojectName, weight)
//only one jmx bean service should report global metrics //only one jmx bean service should report global metrics
override fun recordBuildFinish(action: String?, buildFailed: Boolean, configurationTimeMetrics: List<MetricContainer>) { override fun recordBuildFinish(action: String?, buildFailed: Boolean, metrics: NonSynchronizedMetricsContainer) {
KotlinBuildStatHandler().reportGlobalMetrics(sessionLogger) KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, metrics)
KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, configurationTimeMetrics)
} }
override fun collectProjectConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean) =
KotlinBuildStatHandler().collectProjectConfigurationTimeMetrics(project, sessionLogger, isProjectIsolationEnabled)
override fun collectGeneralConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean, buildReportOutputs: List<BuildReportType>) =
KotlinBuildStatHandler().collectGeneralConfigurationTimeMetrics(project, sessionLogger, isProjectIsolationEnabled, buildReportOutputs)
} }
@@ -0,0 +1,57 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.plugin.statistics
import org.jetbrains.kotlin.statistics.metrics.*
import java.io.Serializable
import kotlin.collections.HashMap
class NonSynchronizedMetricsContainer : StatisticsValuesConsumer, Serializable {
data class MetricDescriptor<T : Comparable<T>>(val name: T, val subprojectName: String?) : Comparable<MetricDescriptor<T>> {
override fun compareTo(other: MetricDescriptor<T>): Int {
val compareNames = name.compareTo(other.name)
return when {
compareNames != 0 -> compareNames
subprojectName == other.subprojectName -> 0
else -> (subprojectName ?: "").compareTo(other.subprojectName ?: "")
}
}
}
private val numericalMetrics = HashMap<MetricDescriptor<NumericalMetrics>, IMetricContainer<Long>>()
private val booleanMetrics = HashMap<MetricDescriptor<BooleanMetrics>, IMetricContainer<Boolean>>()
private val stringMetrics = HashMap<MetricDescriptor<StringMetrics>, IMetricContainer<String>>()
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?): Boolean {
booleanMetrics.getOrPut(MetricDescriptor(metric, subprojectName)) { metric.type.newMetricContainer() }.let { metricContainer ->
metricContainer.addValue(value, weight)
}
return true
}
override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?): Boolean {
numericalMetrics.getOrPut(MetricDescriptor(metric, subprojectName)) { metric.type.newMetricContainer() }.let { metricContainer ->
metricContainer.addValue(value, weight)
}
return true
}
override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?): Boolean {
stringMetrics.getOrPut(MetricDescriptor(metric, subprojectName)) { metric.type.newMetricContainer() }.let { metricContainer ->
metricContainer.addValue(value, weight)
}
return true
}
fun sendToConsumer(metricConsumer: StatisticsValuesConsumer) {
booleanMetrics.forEach { metricConsumer.report(it.key.name, it.value.getValue()!!, it.key.subprojectName) }
numericalMetrics.forEach { metricConsumer.report(it.key.name, it.value.getValue()!!, it.key.subprojectName) }
stringMetrics.forEach { metricConsumer.report(it.key.name, it.value.getValue()!!, it.key.subprojectName) }
}
}
@@ -8,7 +8,6 @@ package org.jetbrains.kotlin.gradle.report
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.api.Task import org.gradle.api.Task
import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.DirectoryProperty
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging import org.gradle.api.logging.Logging
import org.gradle.api.provider.ListProperty import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property import org.gradle.api.provider.Property
@@ -32,7 +31,6 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters
import org.jetbrains.kotlin.build.report.statistics.StatTag import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.buildtools.api.SourcesChanges import org.jetbrains.kotlin.buildtools.api.SourcesChanges
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.report.BuildReportsService.Companion.getStartParameters import org.jetbrains.kotlin.gradle.report.BuildReportsService.Companion.getStartParameters
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.gradle.tasks.withType import org.jetbrains.kotlin.gradle.tasks.withType
@@ -42,7 +40,9 @@ import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.ConcurrentLinkedQueue
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager
import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested
import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFusService
import java.lang.management.ManagementFactory import java.lang.management.ManagementFactory
internal interface UsesBuildMetricsService : Task { internal interface UsesBuildMetricsService : Task {
@@ -63,6 +63,7 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
val projectName: Property<String> val projectName: Property<String>
val kotlinVersion: Property<String> val kotlinVersion: Property<String>
val buildConfigurationTags: ListProperty<StatTag> val buildConfigurationTags: ListProperty<StatTag>
val fusService: Property<BuildFusService>
} }
private val log = Logging.getLogger(this.javaClass) private val log = Logging.getLogger(this.javaClass)
@@ -114,9 +115,11 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
taskExecutionResult?.buildMetrics?.also { taskExecutionResult?.buildMetrics?.also {
buildMetrics.addAll(it) buildMetrics.addAll(it)
KotlinBuildStatsService.applyIfInitialised { collector -> parameters.fusService.orNull?.reportFusMetrics { collector ->
collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs) collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult) collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap() val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
val linesOfCode = metricsMap[GradleBuildPerformanceMetric.ANALYZED_LINES_NUMBER] val linesOfCode = metricsMap[GradleBuildPerformanceMetric.ANALYZED_LINES_NUMBER]
@@ -130,7 +133,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode) collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
} }
} }
collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
collector.report( collector.report(
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT, NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0 if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
@@ -188,6 +190,7 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
private fun registerIfAbsentImpl( private fun registerIfAbsentImpl(
project: Project, project: Project,
fusService: Provider<BuildFusService>
): Provider<BuildMetricsService>? { ): Provider<BuildMetricsService>? {
// Return early if the service was already registered to avoid the overhead of reading the reporting settings below // Return early if the service was already registered to avoid the overhead of reading the reporting settings below
project.gradle.sharedServices.registrations.findByName(serviceName)?.let { project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
@@ -221,6 +224,7 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
it.parameters.projectDir.set(project.rootProject.layout.projectDirectory) it.parameters.projectDir.set(project.rootProject.layout.projectDirectory)
//init gradle tags for build scan and http reports //init gradle tags for build scan and http reports
it.parameters.buildConfigurationTags.value(setupTags(project)) it.parameters.buildConfigurationTags.value(setupTags(project))
it.parameters.fusService.set(fusService)
}.also { }.also {
subscribeForTaskEvents(project, it) subscribeForTaskEvents(project, it)
} }
@@ -281,19 +285,22 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
}) })
} }
else -> {}//do nothing, BuildScanFlowAction is used else -> {
} StatisticsBuildFlowManager.getInstance(project).subscribeForBuildScan(buildScanHolder)
}
fun registerIfAbsent(project: Project) = registerIfAbsentImpl(project)?.also { serviceProvider ->
SingleActionPerProject.run(project, UsesBuildMetricsService::class.java.name) {
project.tasks.withType<UsesBuildMetricsService>().configureEach { task ->
task.buildMetricsService.value(serviceProvider).disallowChanges()
task.usesService(serviceProvider)
} }
} }
} }
fun registerIfAbsent(project: Project, fusService: Provider<BuildFusService>) =
registerIfAbsentImpl(project, fusService)?.also { serviceProvider ->
SingleActionPerProject.run(project, UsesBuildMetricsService::class.java.name) {
project.tasks.withType<UsesBuildMetricsService>().configureEach { task ->
task.buildMetricsService.value(serviceProvider).disallowChanges()
task.usesService(serviceProvider)
}
}
}
private fun setupTags(project: Project): ArrayList<StatTag> { private fun setupTags(project: Project): ArrayList<StatTag> {
val gradle = project.gradle val gradle = project.gradle
val additionalTags = ArrayList<StatTag>() val additionalTags = ArrayList<StatTag>()
@@ -10,14 +10,14 @@ import org.gradle.api.tasks.*
import org.gradle.internal.hash.FileHasher import org.gradle.internal.hash.FileHasher
import org.gradle.work.DisableCachingByDefault import org.gradle.work.DisableCachingByDefault
import org.jetbrains.kotlin.gradle.logging.kotlinInfo import org.jetbrains.kotlin.gradle.logging.kotlinInfo
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import java.io.File import java.io.File
import java.net.URI import java.net.URI
import javax.inject.Inject import javax.inject.Inject
@DisableCachingByDefault @DisableCachingByDefault
abstract class AbstractSetupTask<Env : AbstractEnv, Settings : AbstractSettings<Env>> : DefaultTask() { abstract class AbstractSetupTask<Env : AbstractEnv, Settings : AbstractSettings<Env>> : DefaultTask(), UsesBuildFusService {
@get:Internal @get:Internal
protected abstract val settings: Settings protected abstract val settings: Settings
@@ -77,8 +77,9 @@ abstract class AbstractSetupTask<Env : AbstractEnv, Settings : AbstractSettings<
configuration!!.get().files.single().also { configuration!!.get().files.single().also {
val downloadDuration = System.currentTimeMillis() - startDownloadTime val downloadDuration = System.currentTimeMillis() - startDownloadTime
if (downloadDuration > 0) { if (downloadDuration > 0) {
KotlinBuildStatsService.getInstance() buildFusService.orNull?.reportFusMetrics { metricsConsumer ->
?.report(NumericalMetrics.ARTIFACTS_DOWNLOAD_SPEED, it.length() * 1000 / downloadDuration) metricsConsumer.report(NumericalMetrics.ARTIFACTS_DOWNLOAD_SPEED, it.length() * 1000 / downloadDuration)
}
} }
} }
} }
@@ -20,7 +20,7 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinJsCompilerOptionsDefault
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.ContributeCompilerArgumentsContext import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.ContributeCompilerArgumentsContext
import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService
import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode
import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
@@ -39,7 +39,7 @@ abstract class KotlinJsIrLink @Inject constructor(
objectFactory.newInstance(KotlinJsCompilerOptionsDefault::class.java).configureExperimentalTryK2(project), objectFactory.newInstance(KotlinJsCompilerOptionsDefault::class.java).configureExperimentalTryK2(project),
objectFactory, objectFactory,
workerExecutor workerExecutor
) { ), UsesBuildFusService {
init { init {
// Not check sources, only klib module // Not check sources, only klib module
@@ -117,7 +117,7 @@ abstract class KotlinJsIrLink @Inject constructor(
} }
override fun processArgsBeforeCompile(args: K2JSCompilerArguments) { override fun processArgsBeforeCompile(args: K2JSCompilerArguments) {
KotlinBuildStatsService.applyIfInitialised { buildFusService.orNull?.reportFusMetrics {
it.report(BooleanMetrics.JS_IR_INCREMENTAL, incrementalJsIr) it.report(BooleanMetrics.JS_IR_INCREMENTAL, incrementalJsIr)
val newArgs = K2JSCompilerArguments() val newArgs = K2JSCompilerArguments()
parseCommandLineArguments(ArgumentUtils.convertArgumentsToStringList(args), newArgs) parseCommandLineArguments(ArgumentUtils.convertArgumentsToStringList(args), newArgs)
@@ -37,7 +37,7 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.Create
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext.Companion.create import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext.Companion.create
import org.jetbrains.kotlin.gradle.plugin.cocoapods.asValidFrameworkName import org.jetbrains.kotlin.gradle.plugin.cocoapods.asValidFrameworkName
import org.jetbrains.kotlin.gradle.plugin.mpp.* import org.jetbrains.kotlin.gradle.plugin.mpp.*
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService
import org.jetbrains.kotlin.gradle.report.* import org.jetbrains.kotlin.gradle.report.*
import org.jetbrains.kotlin.gradle.report.UsesBuildMetricsService import org.jetbrains.kotlin.gradle.report.UsesBuildMetricsService
import org.jetbrains.kotlin.gradle.targets.native.KonanPropertiesBuildService import org.jetbrains.kotlin.gradle.targets.native.KonanPropertiesBuildService
@@ -45,7 +45,6 @@ import org.jetbrains.kotlin.gradle.targets.native.tasks.*
import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.gradle.utils.GradleLoggerAdapter import org.jetbrains.kotlin.gradle.utils.GradleLoggerAdapter
import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty
import org.jetbrains.kotlin.incremental.deleteDirectoryContents
import org.jetbrains.kotlin.ir.linkage.partial.PartialLinkageMode import org.jetbrains.kotlin.ir.linkage.partial.PartialLinkageMode
import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER
import org.jetbrains.kotlin.konan.properties.saveToFile import org.jetbrains.kotlin.konan.properties.saveToFile
@@ -307,7 +306,8 @@ internal constructor(
KotlinCompile<KotlinCommonOptions>, KotlinCompile<KotlinCommonOptions>,
K2MultiplatformCompilationTask, K2MultiplatformCompilationTask,
UsesBuildMetricsService, UsesBuildMetricsService,
KotlinCompilationTask<KotlinNativeCompilerOptions> { KotlinCompilationTask<KotlinNativeCompilerOptions>,
UsesBuildFusService {
@get:Input @get:Input
override val outputKind = LIBRARY override val outputKind = LIBRARY
@@ -536,13 +536,13 @@ internal constructor(
} }
private fun collectCommonCompilerStats() { private fun collectCommonCompilerStats() {
KotlinBuildStatsService.getInstance()?.apply { buildFusService.orNull?.reportFusMetrics {
report(BooleanMetrics.KOTLIN_PROGRESSIVE_MODE, compilerOptions.progressiveMode.get()) it.report(BooleanMetrics.KOTLIN_PROGRESSIVE_MODE, compilerOptions.progressiveMode.get())
compilerOptions.apiVersion.orNull?.also { v -> compilerOptions.apiVersion.orNull?.also { v ->
report(StringMetrics.KOTLIN_API_VERSION, v.version) it.report(StringMetrics.KOTLIN_API_VERSION, v.version)
} }
compilerOptions.languageVersion.orNull?.also { v -> compilerOptions.languageVersion.orNull?.also { v ->
report(StringMetrics.KOTLIN_LANGUAGE_VERSION, v.version) it.report(StringMetrics.KOTLIN_LANGUAGE_VERSION, v.version)
} }
} }
} }
@@ -725,7 +725,7 @@ internal class CacheBuilder(
private val executionContext: KotlinToolRunner.GradleExecutionContext, private val executionContext: KotlinToolRunner.GradleExecutionContext,
private val settings: Settings, private val settings: Settings,
private val konanPropertiesService: KonanPropertiesBuildService, private val konanPropertiesService: KonanPropertiesBuildService,
private val metricsReporter: BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric> private val metricsReporter: BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric>,
) { ) {
class Settings( class Settings(
val runnerSettings: KotlinNativeCompilerRunner.Settings, val runnerSettings: KotlinNativeCompilerRunner.Settings,
@@ -955,7 +955,8 @@ internal class CacheBuilder(
} }
private fun ensureCompilerProvidedLibsPrecached() { private fun ensureCompilerProvidedLibsPrecached() {
val distribution = Distribution(settings.runnerSettings.parent.konanHome, konanDataDir = settings.runnerSettings.parent.konanDataDir) val distribution =
Distribution(settings.runnerSettings.parent.konanHome, konanDataDir = settings.runnerSettings.parent.konanDataDir)
val platformLibs = mutableListOf<File>().apply { val platformLibs = mutableListOf<File>().apply {
this += File(distribution.stdlib) this += File(distribution.stdlib)
this += File(distribution.platformLibs(konanTarget)).listFiles().orEmpty() this += File(distribution.platformLibs(konanTarget)).listFiles().orEmpty()
@@ -1172,7 +1173,7 @@ abstract class CInteropProcess @Inject internal constructor(params: Params) :
addAll(extraOpts) addAll(extraOpts)
} }
addBuildMetricsForTaskAction(buildMetrics, languageVersion = null) { addBuildMetricsForTaskAction(buildMetrics, languageVersion = null) {
outputFile.parentFile.mkdirs() outputFile.parentFile.mkdirs()
KotlinNativeCInteropRunner.createExecutionContext( KotlinNativeCInteropRunner.createExecutionContext(
task = this, task = this,
@@ -35,7 +35,7 @@ import org.jetbrains.kotlin.gradle.plugin.UsesBuildFinishedListenerService
import org.jetbrains.kotlin.gradle.plugin.UsesVariantImplementationFactories import org.jetbrains.kotlin.gradle.plugin.UsesVariantImplementationFactories
import org.jetbrains.kotlin.gradle.plugin.diagnostics.UsesKotlinToolingDiagnostics import org.jetbrains.kotlin.gradle.plugin.diagnostics.UsesKotlinToolingDiagnostics
import org.jetbrains.kotlin.gradle.plugin.internal.UsesBuildIdProviderService import org.jetbrains.kotlin.gradle.plugin.internal.UsesBuildIdProviderService
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService
import org.jetbrains.kotlin.gradle.report.* import org.jetbrains.kotlin.gradle.report.*
import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
@@ -60,6 +60,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
UsesClassLoadersCachingBuildService, UsesClassLoadersCachingBuildService,
UsesKotlinToolingDiagnostics, UsesKotlinToolingDiagnostics,
UsesBuildIdProviderService, UsesBuildIdProviderService,
UsesBuildFusService,
BaseKotlinCompile { BaseKotlinCompile {
init { init {
@@ -201,6 +202,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
classLoadersCachingService, classLoadersCachingService,
buildFinishedListenerService, buildFinishedListenerService,
buildIdService, buildIdService,
buildFusService.orNull?.fusMetricsConsumer
) )
} }
} }
@@ -247,7 +249,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
val buildMetrics = metrics.get() val buildMetrics = metrics.get()
buildMetrics.addTimeMetric(GradleBuildPerformanceMetric.START_TASK_ACTION_EXECUTION) buildMetrics.addTimeMetric(GradleBuildPerformanceMetric.START_TASK_ACTION_EXECUTION)
buildMetrics.measure(GradleBuildTime.OUT_OF_WORKER_TASK_ACTION) { buildMetrics.measure(GradleBuildTime.OUT_OF_WORKER_TASK_ACTION) {
KotlinBuildStatsService.applyIfInitialised { buildFusService.orNull?.reportFusMetrics {
if (name.contains("Test")) if (name.contains("Test"))
it.report(BooleanMetrics.TESTS_EXECUTED, true) it.report(BooleanMetrics.TESTS_EXECUTED, true)
else else
@@ -289,13 +291,13 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
} }
private fun collectCommonCompilerStats() { private fun collectCommonCompilerStats() {
KotlinBuildStatsService.getInstance()?.apply { buildFusService.orNull?.reportFusMetrics {
report(BooleanMetrics.KOTLIN_PROGRESSIVE_MODE, compilerOptions.progressiveMode.get()) it.report(BooleanMetrics.KOTLIN_PROGRESSIVE_MODE, compilerOptions.progressiveMode.get())
compilerOptions.apiVersion.orNull?.also { v -> compilerOptions.apiVersion.orNull?.also { v ->
report(StringMetrics.KOTLIN_API_VERSION, v.version) it.report(StringMetrics.KOTLIN_API_VERSION, v.version)
} }
compilerOptions.languageVersion.orNull?.also { v -> compilerOptions.languageVersion.orNull?.also { v ->
report(StringMetrics.KOTLIN_LANGUAGE_VERSION, v.version) it.report(StringMetrics.KOTLIN_LANGUAGE_VERSION, v.version)
} }
} }
} }
@@ -31,7 +31,7 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.Contri
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext.Companion.create import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext.Companion.create
import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService
import org.jetbrains.kotlin.gradle.report.BuildReportMode import org.jetbrains.kotlin.gradle.report.BuildReportMode
import org.jetbrains.kotlin.gradle.targets.js.internal.LibraryFilterCachingService import org.jetbrains.kotlin.gradle.targets.js.internal.LibraryFilterCachingService
import org.jetbrains.kotlin.gradle.targets.js.internal.UsesLibraryFilterCachingService import org.jetbrains.kotlin.gradle.targets.js.internal.UsesLibraryFilterCachingService
@@ -41,7 +41,6 @@ import org.jetbrains.kotlin.gradle.targets.js.ir.PRODUCE_UNZIPPED_KLIB
import org.jetbrains.kotlin.gradle.targets.js.ir.PRODUCE_ZIPPED_KLIB import org.jetbrains.kotlin.gradle.targets.js.ir.PRODUCE_ZIPPED_KLIB
import org.jetbrains.kotlin.gradle.tasks.internal.KotlinJsOptionsCompat import org.jetbrains.kotlin.gradle.tasks.internal.KotlinJsOptionsCompat
import org.jetbrains.kotlin.gradle.utils.getFile import org.jetbrains.kotlin.gradle.utils.getFile
import org.jetbrains.kotlin.gradle.utils.existsCompat
import org.jetbrains.kotlin.gradle.utils.isParentOf import org.jetbrains.kotlin.gradle.utils.isParentOf
import org.jetbrains.kotlin.gradle.utils.newInstance import org.jetbrains.kotlin.gradle.utils.newInstance
import org.jetbrains.kotlin.gradle.utils.toPathsArray import org.jetbrains.kotlin.gradle.utils.toPathsArray
@@ -60,6 +59,7 @@ abstract class Kotlin2JsCompile @Inject constructor(
) : AbstractKotlinCompile<K2JSCompilerArguments>(objectFactory, workerExecutor), ) : AbstractKotlinCompile<K2JSCompilerArguments>(objectFactory, workerExecutor),
KotlinCompilationTask<KotlinJsCompilerOptions>, KotlinCompilationTask<KotlinJsCompilerOptions>,
UsesLibraryFilterCachingService, UsesLibraryFilterCachingService,
UsesBuildFusService,
KotlinJsCompile, KotlinJsCompile,
K2MultiplatformCompilationTask { K2MultiplatformCompilationTask {
@@ -82,14 +82,14 @@ abstract class Kotlin2JsCompile @Inject constructor(
PRODUCE_JS in freeArgs -> false PRODUCE_JS in freeArgs -> false
PRODUCE_UNZIPPED_KLIB in freeArgs -> { PRODUCE_UNZIPPED_KLIB in freeArgs -> {
KotlinBuildStatsService.applyIfInitialised { buildFusService.orNull?.reportFusMetrics {
it.report(BooleanMetrics.JS_KLIB_INCREMENTAL, incrementalJsKlib) it.report(BooleanMetrics.JS_KLIB_INCREMENTAL, incrementalJsKlib)
} }
incrementalJsKlib incrementalJsKlib
} }
PRODUCE_ZIPPED_KLIB in freeArgs -> { PRODUCE_ZIPPED_KLIB in freeArgs -> {
KotlinBuildStatsService.applyIfInitialised { buildFusService.orNull?.reportFusMetrics {
it.report(BooleanMetrics.JS_KLIB_INCREMENTAL, incrementalJsKlib) it.report(BooleanMetrics.JS_KLIB_INCREMENTAL, incrementalJsKlib)
} }
incrementalJsKlib incrementalJsKlib
@@ -21,7 +21,7 @@ class BuildSessionLogger(
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE, private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE, private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
private val forceValuesValidation: Boolean = false, private val forceValuesValidation: Boolean = false,
) : IStatisticsValuesConsumer { ) : StatisticsValuesConsumer {
companion object { companion object {
const val STATISTICS_FOLDER_NAME = "kotlin-profile" const val STATISTICS_FOLDER_NAME = "kotlin-profile"
@@ -16,7 +16,7 @@ import java.nio.file.Paths
import java.nio.file.StandardOpenOption import java.nio.file.StandardOpenOption
import java.util.* import java.util.*
class MetricsContainer(private val forceValuesValidation: Boolean = false) : IStatisticsValuesConsumer { class MetricsContainer(private val forceValuesValidation: Boolean = false) : StatisticsValuesConsumer {
data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> { data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
override fun compareTo(other: MetricDescriptor): Int { override fun compareTo(other: MetricDescriptor): Int {
val compareNames = name.compareTo(other.name) val compareNames = name.compareTo(other.name)
@@ -5,9 +5,10 @@
package org.jetbrains.kotlin.statistics.metrics package org.jetbrains.kotlin.statistics.metrics
import java.io.Serializable
import java.util.* import java.util.*
interface IMetricContainer<T> { interface IMetricContainer<T> : Serializable {
fun addValue(t: T, weight: Long? = null) fun addValue(t: T, weight: Long? = null)
fun toStringRepresentation(): String fun toStringRepresentation(): String
@@ -5,7 +5,7 @@
package org.jetbrains.kotlin.statistics.metrics package org.jetbrains.kotlin.statistics.metrics
interface IStatisticsValuesConsumer { interface StatisticsValuesConsumer {
fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null, weight: Long? = null): Boolean fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null, weight: Long? = null): Boolean
fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null, weight: Long? = null): Boolean fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null, weight: Long? = null): Boolean
@@ -17,10 +17,10 @@ import kotlin.test.Test
import kotlin.test.assertEquals import kotlin.test.assertEquals
private const val SOURCE_CODE_RELATIVE_PATH = private const val SOURCE_CODE_RELATIVE_PATH =
"libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics" "libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/metrics"
private const val BOOLEAN_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/metrics/BooleanMetrics.kt" private const val BOOLEAN_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/BooleanMetrics.kt"
private const val STRING_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/metrics/StringMetrics.kt" private const val STRING_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/StringMetrics.kt"
private const val NUMERICAL_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/metrics/NumericalMetrics.kt" private const val NUMERICAL_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/NumericalMetrics.kt"
private val STRING_METRICS_EXPECTED_VERSION_AND_HASH = Pair(1, "90347332db2ce54b51e7daa64595371e") private val STRING_METRICS_EXPECTED_VERSION_AND_HASH = Pair(1, "90347332db2ce54b51e7daa64595371e")
private val BOOLEAN_METRICS_EXPECTED_VERSION_AND_HASH = Pair(3, "3c8c4ca636adee168e99862244a22520") private val BOOLEAN_METRICS_EXPECTED_VERSION_AND_HASH = Pair(3, "3c8c4ca636adee168e99862244a22520")
@@ -30,7 +30,7 @@ private val SOURCE_FOLDER_EXPECTED_VERSION_AND_HASH =
STRING_METRICS_EXPECTED_VERSION_AND_HASH.first + STRING_METRICS_EXPECTED_VERSION_AND_HASH.first +
BOOLEAN_METRICS_EXPECTED_VERSION_AND_HASH.first + BOOLEAN_METRICS_EXPECTED_VERSION_AND_HASH.first +
NUMERICAL_METRICS_EXPECTED_VERSION_AND_HASH.first, NUMERICAL_METRICS_EXPECTED_VERSION_AND_HASH.first,
"ce47debcab820f23e0ea1333d4124e9e" "8b845ce616f5bdb9b885ededcdfd812d"
) )
private const val HASH_ALG = "MD5" private const val HASH_ALG = "MD5"