diff --git a/build-common/build.gradle.kts b/build-common/build.gradle.kts index 91af78ef459..f2cc7945b5f 100644 --- a/build-common/build.gradle.kts +++ b/build-common/build.gradle.kts @@ -19,12 +19,14 @@ dependencies { compileOnly(intellijCore()) compileOnly(commonDependency("org.jetbrains.intellij.deps:asm-all")) compileOnly(commonDependency("org.jetbrains.intellij.deps:trove4j")) + compileOnly(project(":kotlin-build-statistic")) testCompileOnly(project(":compiler:cli-common")) testApi(projectTests(":compiler:tests-common")) testApi(commonDependency("junit:junit")) testApi(protobufFull()) testApi(kotlinStdlib()) + testImplementation(project(":kotlin-build-statistic")) testImplementation(commonDependency("org.jetbrains.kotlin:kotlin-reflect")) { isTransitive = false } testImplementation("org.reflections:reflections:0.10.2") } diff --git a/build.gradle.kts b/build.gradle.kts index fccf9d69e24..b538b5b3ba5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -143,6 +143,7 @@ val commonCompilerModules = arrayOf( ":analysis:project-structure", ":analysis:kt-references", ":kotlin-build-common", + ":kotlin-build-statistic", ).also { extra["commonCompilerModules"] = it } val firCompilerCoreModules = arrayOf( @@ -262,7 +263,8 @@ extra["kotlinJpsPluginMavenDependencies"] = listOf( ":kotlin-util-io", ":kotlin-util-klib", ":kotlin-util-klib-metadata", - ":native:kotlin-native-utils" + ":native:kotlin-native-utils", + ":kotlin-build-statistic", ) extra["kotlinJpsPluginMavenDependenciesNonTransitiveLibs"] = listOf( diff --git a/buildSrc/src/main/kotlin/tasks.kt b/buildSrc/src/main/kotlin/tasks.kt index 08e10c87943..71a473a9e2c 100644 --- a/buildSrc/src/main/kotlin/tasks.kt +++ b/buildSrc/src/main/kotlin/tasks.kt @@ -81,6 +81,7 @@ val kotlinGradlePluginAndItsRequired = arrayOf( ":native:kotlin-klib-commonizer-api", ":compiler:build-tools:kotlin-build-tools-api", ":compiler:build-tools:kotlin-build-tools-impl", + ":compiler:kotlin-build-statistic" ) fun Task.dependsOnKotlinGradlePluginInstall() { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/utils.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/utils.kt index ba596c58711..a162182fa6a 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/utils.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/utils.kt @@ -87,7 +87,7 @@ fun getLibraryFromHome( fun MessageCollector.toLogger(): Logger = object : Logger { - override fun error(message: String) { + override fun error(message: String, throwable: Throwable?) { report(CompilerMessageSeverity.ERROR, message) } @@ -103,5 +103,9 @@ fun MessageCollector.toLogger(): Logger = override fun log(message: String) { report(CompilerMessageSeverity.LOGGING, message) } + + override fun lifecycle(message: String) { + report(CompilerMessageSeverity.INFO, message) + } } diff --git a/compiler/incremental-compilation-impl/build.gradle.kts b/compiler/incremental-compilation-impl/build.gradle.kts index a20ed6f4733..79213542376 100644 --- a/compiler/incremental-compilation-impl/build.gradle.kts +++ b/compiler/incremental-compilation-impl/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { api(project(":compiler:backend.jvm.entrypoint")) api(project(":kotlin-build-common")) api(project(":daemon-common")) + api(project(":kotlin-build-statistic")) compileOnly(intellijCore()) testApi(commonDependency("junit:junit")) @@ -28,6 +29,7 @@ dependencies { testApi(intellijCore()) testApi(commonDependency("org.jetbrains.intellij.deps:log4j")) testApi(commonDependency("org.jetbrains.intellij.deps:jdom")) + testApi(projectTests(":kotlin-build-statistic")) testImplementation(commonDependency("com.google.code.gson:gson")) testRuntimeOnly(commonDependency("org.jetbrains.kotlin:kotlin-reflect")) { isTransitive = false } diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index 1bfd29e3679..daca0a0a225 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -106,8 +106,9 @@ val CompilerConfiguration.resolverLogger: Logger null -> DummyLogger else -> object : Logger { override fun log(message: String) = messageLogger.report(IrMessageLogger.Severity.INFO, message, null) - override fun error(message: String) = messageLogger.report(IrMessageLogger.Severity.ERROR, message, null) + override fun error(message: String, throwable: Throwable?) = messageLogger.report(IrMessageLogger.Severity.ERROR, message, null) override fun warning(message: String) = messageLogger.report(IrMessageLogger.Severity.WARNING, message, null) + override fun lifecycle(message: String) = messageLogger.report(IrMessageLogger.Severity.INFO, message, null) override fun fatal(message: String): Nothing { messageLogger.report(IrMessageLogger.Severity.ERROR, message, null) diff --git a/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt index 482dab42f4e..dbb86866f7a 100644 --- a/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt +++ b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt @@ -175,8 +175,9 @@ object GenerationUtils { fun messageCollectorLogger(collector: MessageCollector) = object : Logger { override fun warning(message: String) = collector.report(CompilerMessageSeverity.STRONG_WARNING, message) - override fun error(message: String) = collector.report(CompilerMessageSeverity.ERROR, message) + override fun error(message: String, throwable: Throwable?) = collector.report(CompilerMessageSeverity.ERROR, message) override fun log(message: String) = collector.report(CompilerMessageSeverity.LOGGING, message) + override fun lifecycle(message: String) = collector.report(CompilerMessageSeverity.INFO, message) override fun fatal(message: String): Nothing { collector.report(CompilerMessageSeverity.ERROR, message) (collector as? GroupingMessageCollector)?.flush() diff --git a/compiler/util-io/src/org/jetbrains/kotlin/util/WithLogger.kt b/compiler/util-io/src/org/jetbrains/kotlin/util/WithLogger.kt index df03ffc5863..9ce4c361921 100644 --- a/compiler/util-io/src/org/jetbrains/kotlin/util/WithLogger.kt +++ b/compiler/util-io/src/org/jetbrains/kotlin/util/WithLogger.kt @@ -4,9 +4,10 @@ import kotlin.system.exitProcess interface Logger { fun log(message: String) - fun error(message: String) + fun error(message: String, throwable: Throwable? = null) fun warning(message: String) fun fatal(message: String): Nothing + fun lifecycle(message: String) } interface WithLogger { @@ -15,10 +16,18 @@ interface WithLogger { object DummyLogger : Logger { override fun log(message: String) = println(message) - override fun error(message: String) = println("e: $message") + override fun error(message: String, throwable: Throwable?) { + println("e: $message") + throwable?.also { + println("${it.message}:\n${it.stackTraceToString()} ") + } + } + override fun warning(message: String) = println("w: $message") override fun fatal(message: String): Nothing { println("e: $message") exitProcess(1) } + + override fun lifecycle(message: String) = println("i: $message") } diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/SingleFileResolve.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/SingleFileResolve.kt index fc7e3a562ec..58c76296e61 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/SingleFileResolve.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/SingleFileResolve.kt @@ -16,9 +16,10 @@ fun resolveSingleFileKlib( libraryFile: File, logger: Logger = object : Logger { override fun log(message: String) {} - override fun error(message: String) = kotlin.error("e: $message") + override fun error(message: String, throwable: Throwable?) = kotlin.error("e: $message") override fun warning(message: String) {} override fun fatal(message: String) = kotlin.error("e: $message") + override fun lifecycle(message: String) {} }, strategy: SingleFileKlibResolveStrategy = CompilerSingleFileKlibResolveStrategy ): KotlinLibrary = strategy.resolve(libraryFile, logger) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 36f85c75ce3..b811c57a126 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -20,6 +20,7 @@ + diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index df2360696f7..2b2497803fb 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -42,6 +42,7 @@ import org.jetbrains.kotlin.jps.KotlinJpsBundle import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager import org.jetbrains.kotlin.jps.model.kotlinKind +import org.jetbrains.kotlin.jps.statistic.KotlinBuilderReportService import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader @@ -67,6 +68,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { System.getProperty("kotlin.jps.classesToLoadByParent")?.split(',')?.map { it.trim() } ?: emptyList() private val classPrefixesToLoadByParentFromRegistry = System.getProperty("kotlin.jps.classPrefixesToLoadByParent")?.split(',')?.map { it.trim() } ?: emptyList() + private val reportService = KotlinBuilderReportService() val classesToLoadByParent: ClassCondition get() = ClassCondition { className -> @@ -100,6 +102,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { override fun buildStarted(context: CompileContext) { logSettings(context) + reportService.buildStarted(context) } private fun logSettings(context: CompileContext) { @@ -160,6 +163,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { override fun buildFinished(context: CompileContext) { ensureKotlinContextDisposed(context) + reportService.buildFinished(context) } private fun ensureKotlinContextDisposed(context: CompileContext) { diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsLoggerAdapter.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsLoggerAdapter.kt new file mode 100644 index 00000000000..9854b42fa95 --- /dev/null +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsLoggerAdapter.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.statistic + +import com.intellij.openapi.diagnostic.Logger + +class JpsLoggerAdapter(private val log: Logger) : org.jetbrains.kotlin.util.Logger { + + override fun log(message: String) { + log.info(message) + } + + override fun warning(message: String) { + log.warn(message) + } + + override fun fatal(message: String): Nothing { + log.error(message) + kotlin.error(message) + } + + override fun error(message: String, throwable: Throwable?) { + log.error(message, throwable) + } + + override fun lifecycle(message: String) { + log.info(message) + } +} \ No newline at end of file diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/KotlinBuilderReportService.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/KotlinBuilderReportService.kt new file mode 100644 index 00000000000..81e8cdfccae --- /dev/null +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/KotlinBuilderReportService.kt @@ -0,0 +1,131 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.statistic + +import com.intellij.openapi.diagnostic.Logger +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.kotlin.build.report.FileReportSettings +import org.jetbrains.kotlin.build.report.HttpReportSettings +import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.statistic.BuildDataType +import org.jetbrains.kotlin.build.report.statistic.CompileStatisticsData +import org.jetbrains.kotlin.build.report.statistic.HttpReportServiceImpl +import org.jetbrains.kotlin.build.report.statistic.file.FileReportService +import java.io.File +import java.util.* +import java.net.InetAddress + +interface JpsBuilderMetricReporter : BuildMetricsReporter { + fun flush(context: CompileContext): CompileStatisticsData +} + +//single thread execution +class JpsBuilderMetricReporterImpl(private val reporter: BuildMetricsReporterImpl) : JpsBuilderMetricReporter, BuildMetricsReporter by reporter { + + companion object { + private val log = Logger.getInstance("#org.jetbrains.kotlin.jps.statistic.KotlinBuilderMetricImpl") + private val hostName: String? = try { + InetAddress.getLocalHost().hostName + } catch (_: Exception) { + //do nothing + null + } + } + + private val uuid = UUID.randomUUID() + private val startTime = System.currentTimeMillis() + + override fun flush(context: CompileContext/*, listener: BuildListener*/): CompileStatisticsData { + val buildMetrics = reporter.getMetrics() + return CompileStatisticsData( + projectName = context.projectDescriptor.project.name, + label = "JPS build", //TODO + taskName = "JPS build", + taskResult = "Unknown",//TODO + startTimeMs = startTime, + durationMs = System.currentTimeMillis() - startTime, + tags = emptySet(), + buildUuid = uuid.toString(), + changes = emptyList(), //TODO + kotlinVersion = "kotlin_version", //TODO + hostName = hostName, + finishTime = System.currentTimeMillis(), + buildTimesMetrics = buildMetrics.buildTimes.asMapMs(), + performanceMetrics = buildMetrics.buildPerformanceMetrics.asMap(), + compilerArguments = emptyList(), //TODO + nonIncrementalAttributes = emptySet(), + type = BuildDataType.JPS_DATA.name, + fromKotlinPlugin = true, + compiledSources = emptyList(), + skipMessage = null, + icLogLines = emptyList(), + gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(), + gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(), + kotlinLanguageVersion = null + ) + } + +} + +// TODO test UserDataHolder in CompileContext to store CompileStatisticsData.Build or KotlinBuilderMetric +class KotlinBuilderReportService( + private val fileReportSettings: FileReportSettings?, + private val httpReportSettings: HttpReportSettings? +) { + constructor() : this( + initFileReportSettings(), + initHttpReportSettings(), + ) + + companion object { + private fun initFileReportSettings(): FileReportSettings? { + return System.getProperty("kotlin.build.report.file.output_dir")?.let { FileReportSettings(File(it)) } + } + + private fun initHttpReportSettings(): HttpReportSettings? { + val httpReportUrl = System.getProperty("kotlin.build.report.http.url") ?: return null + val httpReportUser = System.getProperty("kotlin.build.report.http.user") + val httpReportPassword = System.getProperty("kotlin.build.report.http.password") + val includeGitBranch = System.getProperty("kotlin.build.report.http.git_branch", "false").toBoolean() + val verboseEnvironment = System.getProperty("kotlin.build.report.http.environment.verbose", "false").toBoolean() + return HttpReportSettings(httpReportUrl, httpReportUser, httpReportPassword, verboseEnvironment, includeGitBranch) + } + } + + private val contextMetrics = HashMap() + private val log = Logger.getInstance("#org.jetbrains.kotlin.jps.statistic.KotlinBuilderReportService") + private val loggerAdapter = JpsLoggerAdapter(log) + private val httpService = httpReportSettings?.let { HttpReportServiceImpl(it.url, it.user, it.password) } + fun buildStarted(context: CompileContext) { + if (contextMetrics[context] != null) { + log.error("Service already initialized for context") + } + contextMetrics[context] = JpsBuilderMetricReporterImpl(BuildMetricsReporterImpl()) + } + + fun buildFinished(context: CompileContext) { + val metrics = contextMetrics.remove(context) + if (metrics == null) { + log.error("Service hasn't initialized for context") + return + } + + httpService?.sendData(metrics.flush(context), loggerAdapter) + fileReportSettings?.also { FileReportService(it.buildReportDir, true, loggerAdapter) } + } + + fun addMetric(context: CompileContext, metric: BuildTime, value: Long) { + val metrics = contextMetrics[context] + if (metrics == null) { + log.error("Service hasn't initialized for context") + return + } + metrics.addTimeMetricNs(metric, value) + } +} + + + diff --git a/kotlin-build-statistic/build.gradle.kts b/kotlin-build-statistic/build.gradle.kts new file mode 100644 index 00000000000..65f2f83506c --- /dev/null +++ b/kotlin-build-statistic/build.gradle.kts @@ -0,0 +1,47 @@ +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion as GradleKotlinVersion +import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask + +description = "Kotlin Build Report Common" + +plugins { + kotlin("jvm") + id("jps-compatible") +} + +dependencies { + compileOnly(project(":core:util.runtime")) + compileOnly(project(":compiler:util")) + compileOnly(project(":kotlin-util-io")) + compileOnly(commonDependency("org.jetbrains.kotlin:kotlin-reflect")) { isTransitive = false } + + compileOnly(kotlinStdlib()) + compileOnly(intellijCore()) + implementation(commonDependency("com.google.code.gson:gson")) + testApi(kotlinStdlib()) +} + +sourceSets { + "main" { projectDefault() } + "test" { projectDefault() } +} + +publish() + +runtimeJar() +sourcesJar() +javadocJar() + +testsJar() + +projectTest(parallel = true) + +projectTest("testJUnit5", jUnitMode = JUnitMode.JUnit5, parallel = true) { + useJUnitPlatform() +} + +// 1.9 level breaks Kotlin Gradle plugins via changes in enums (KT-48872) +// We limit api and LV until KGP will stop using Kotlin compiler directly (KT-56574) +tasks.withType>().configureEach { + compilerOptions.apiVersion.value(GradleKotlinVersion.KOTLIN_1_8).finalizeValueOnRead() + compilerOptions.languageVersion.value(GradleKotlinVersion.KOTLIN_1_8).finalizeValueOnRead() +} diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt similarity index 97% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt index 5190bbead38..6473e8d7adc 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt similarity index 92% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt index 867d2adbcc7..6402383e7ce 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt similarity index 92% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt index 47a3e456a9a..362ac468d80 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt similarity index 95% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt index 39d01c7bc1b..6b4c202583a 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt similarity index 95% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt index a38664d2c33..8ef136c77e6 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ @@ -57,6 +57,7 @@ class BuildMetricsReporterImpl : BuildMetricsReporter, Serializable { when (metric.type) { ValueType.NANOSECONDS -> myBuildMetrics.add(metric, System.nanoTime()) ValueType.MILLISECONDS -> myBuildMetrics.add(metric, System.currentTimeMillis()) + ValueType.TIME -> myBuildMetrics.add(metric, System.currentTimeMillis()) else -> error("Unable to add time metric for '${metric.type}' type") } diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt similarity index 94% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index 1dd6dc55657..e2ecc1f8477 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ @@ -41,13 +41,14 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va LOAD_CLASSPATH_ENTRY_SNAPSHOT_CACHE_HITS(parent = LOAD_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Number of cache hits when loading classpath entry snapshots", type = ValueType.NUMBER), LOAD_CLASSPATH_ENTRY_SNAPSHOT_CACHE_MISSES(parent = LOAD_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Number of cache misses when loading classpath entry snapshots", type = ValueType.NUMBER), - //exact time - START_TASK_ACTION_EXECUTION(readableString = "Start time of task action", type = ValueType.MILLISECONDS), + //time metrics + START_TASK_ACTION_EXECUTION(readableString = "Start time of task action", type = ValueType.TIME), + FINISH_KOTLIN_DAEMON_EXECUTION(readableString = "Finish time of kotlin daemon execution", type = ValueType.TIME), + CALL_KOTLIN_DAEMON(readableString = "Finish gradle part of task execution", type = ValueType.NANOSECONDS), CALL_WORKER(readableString = "Worker submit time", type = ValueType.NANOSECONDS), START_WORKER_EXECUTION(readableString = "Start time of worker execution", type = ValueType.NANOSECONDS), START_KOTLIN_DAEMON_EXECUTION(readableString = "Start time of kotlin daemon task execution", type = ValueType.NANOSECONDS), - FINISH_KOTLIN_DAEMON_EXECUTION(readableString = "Finish kotlin daemon execution", type = ValueType.MILLISECONDS), ; companion object { const val serialVersionUID = 0L @@ -63,4 +64,5 @@ enum class ValueType { NUMBER, NANOSECONDS, MILLISECONDS, + TIME } diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt similarity index 92% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt index 677bf43bf40..aad35f9afa6 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt similarity index 99% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index 41fe3c432e7..db841dce7df 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt similarity index 100% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt similarity index 94% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt index f0a322aa1b3..32adeab22b2 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt similarity index 100% rename from build-common/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/metrics/GcMetrics.kt diff --git a/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/reportSettings.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/reportSettings.kt new file mode 100644 index 00000000000..112e29e20ce --- /dev/null +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/reportSettings.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report + +import java.io.File +import java.io.Serializable + +data class FileReportSettings( + val buildReportDir: File, + val includeMetricsInReport: Boolean = false, +) : Serializable { + companion object { + const val serialVersionUID: Long = 0 + } +} + +data class HttpReportSettings( + val url: String, + val password: String?, + val user: String?, + val verboseEnvironment: Boolean, + val includeGitBranchName: Boolean +) : Serializable { + companion object { + const val serialVersionUID: Long = 0 + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/CompileStatisticsData.kt similarity index 83% rename from libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/CompileStatisticsData.kt index a79c4fd07b0..1cc7de22598 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/CompileStatisticsData.kt @@ -1,31 +1,31 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.gradle.plugin.stat +package org.jetbrains.kotlin.build.report.statistic import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.BuildTime import java.text.SimpleDateFormat import java.util.* -import org.jetbrains.kotlin.gradle.dsl.KotlinVersion //Sensitive data. This object is used directly for statistic via http private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC")} data class CompileStatisticsData( - val version: Int = 2, + val version: Int = 3, val projectName: String?, val label: String?, - val taskName: String?, - val taskResult: String, + val taskName: String, + val taskResult: String?, + val startTimeMs: Long, val durationMs: Long, - val tags: List, + val tags: Set, val changes: List, val buildUuid: String = "Unset", val kotlinVersion: String, - val kotlinLanguageVersion: KotlinVersion?, + val kotlinLanguageVersion: String?, val hostName: String? = "Unset", val finishTime: Long, val timestamp: String = formatter.format(finishTime), @@ -36,7 +36,11 @@ data class CompileStatisticsData( val performanceMetrics: Map, val gcTimeMetrics: Map?, val gcCountMetrics: Map?, - val type: String = BuildDataType.TASK_DATA.name + val type: String = BuildDataType.TASK_DATA.name, + val fromKotlinPlugin: Boolean?, + val compiledSources: List = emptyList(), + val skipMessage: String?, + val icLogLines: List, ) @@ -57,7 +61,8 @@ enum class StatTag(val readableString: String) { enum class BuildDataType { TASK_DATA, - BUILD_DATA + BUILD_DATA, + JPS_DATA } //Sensitive data. This object is used directly for statistic via http @@ -80,7 +85,7 @@ data class BuildFinishStatisticsData( val finishTime: Long, val timestamp: String = formatter.format(finishTime), val hostName: String? = "Unset", - val tags: List, + val tags: Set, val gitBranch: String = "Unset" ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/HttpReportService.kt similarity index 80% rename from libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/HttpReportService.kt index eb508516a78..97d29cbf602 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/HttpReportService.kt @@ -1,12 +1,12 @@ /* - * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.gradle.report +package org.jetbrains.kotlin.build.report.statistic import com.google.gson.Gson -import org.gradle.api.logging.Logger +import org.jetbrains.kotlin.util.Logger import java.io.IOException import java.io.Serializable import java.net.HttpURLConnection @@ -23,7 +23,6 @@ class HttpReportServiceImpl( private val password: String?, private val user: String?, ) : HttpReportService, Serializable { - constructor(httpSettings: HttpReportSettings) : this(httpSettings.url, httpSettings.password, httpSettings.user) private var invalidUrl = false private var requestPreviousFailed = false @@ -33,9 +32,7 @@ class HttpReportServiceImpl( if (isResponseBad) { val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}" if (!requestPreviousFailed) { - log.warn(message) - } else { - log.debug(message) + log.warning(message) } requestPreviousFailed = true } @@ -49,7 +46,7 @@ class HttpReportServiceImpl( val connection = try { URL(url).openConnection() as HttpURLConnection } catch (e: IOException) { - log.warn("Unable to open connection to ${url}: ${e.message}") + log.warning("Unable to open connection to ${url}: ${e.message}") invalidUrl = true return } @@ -70,12 +67,12 @@ class HttpReportServiceImpl( connection.connect() checkResponseAndLog(connection, log) } catch (e: Exception) { - log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}") + log.warning("Unexpected exception happened ${e.message}: ${e.stackTrace}") checkResponseAndLog(connection, log) } finally { connection.disconnect() } } - log.debug("Report statistic by http takes $elapsedTime ms") + log.log("Report statistic by http takes $elapsedTime ms") } } diff --git a/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/FileReportService.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/FileReportService.kt new file mode 100644 index 00000000000..70f3852d09f --- /dev/null +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/FileReportService.kt @@ -0,0 +1,294 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistic.file + +import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.statistic.asString +import org.jetbrains.kotlin.build.report.statistic.formatSize +import org.jetbrains.kotlin.build.report.statistic.formatTime +import org.jetbrains.kotlin.build.report.statistic.CompileStatisticsData +import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters +import org.jetbrains.kotlin.util.Logger +import java.io.File +import java.io.Serializable +import java.text.SimpleDateFormat +import java.util.* + +class FileReportService( + private val outputFile: File, + private val printMetrics: Boolean, + private val logger: Logger +) : Serializable { + companion object { + private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC")} + fun reportBuildStatInFile( + buildReportDir: File, + projectName: String, + includeMetricsInReport: Boolean, + buildData: List, + startParameters: GradleBuildStartParameters, + failureMessages: List, + logger: Logger + ) { + val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time) + val reportFile = buildReportDir.resolve("$projectName-build-$ts.txt") + + FileReportService( + outputFile = reportFile, + printMetrics = includeMetricsInReport, + logger = logger + ).process(buildData, startParameters, failureMessages) + } + } + + private lateinit var p: Printer + + fun process( + statisticsData: List, + startParameters: GradleBuildStartParameters, + failureMessages: List + ) { + val buildReportPath = outputFile.toPath().toUri().toString() + try { + outputFile.parentFile.mkdirs() + if (!(outputFile.parentFile.exists() && outputFile.parentFile.isDirectory)) { + logger.error("Kotlin build report cannot be created: '$outputFile.parentFile' is a file or do not have permissions to create") + return + } + + outputFile.bufferedWriter().use { writer -> + p = Printer(writer) + printBuildReport(statisticsData, startParameters, failureMessages) + } + + logger.lifecycle("Kotlin build report is written to $buildReportPath") + } catch (e: Exception) { + logger.error("Could not write Kotlin build report to $buildReportPath", e) + } + } + + private fun printBuildReport( + statisticsData: List, + startParameters: GradleBuildStartParameters, + failureMessages: List + ) { + // NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms. + // Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound + // a bit unfamiliar. + // TODO: If it is confusing, consider renaming "tasks" to "build operations" in this class. + printBuildInfo(startParameters, failureMessages) + if (printMetrics) { + printMetrics( + statisticsData.map { it.buildTimesMetrics }.reduce { agg, value -> + (agg.keys + value.keys).associateWith { (agg[it] ?: 0) + (value[it] ?: 0) } + }, + statisticsData.map { it.performanceMetrics }.reduce { agg, value -> + (agg.keys + value.keys).associateWith { (agg[it] ?: 0) + (value[it] ?: 0) } + }, + statisticsData.map { it.nonIncrementalAttributes.asSequence() }.reduce { agg, value -> agg + value }.toList(), + aggregatedMetric = true + ) + p.println() + } + printTaskOverview(statisticsData) + printTasksLog(statisticsData) + } + + private fun printBuildInfo(startParameters: GradleBuildStartParameters, failureMessages: List) { + p.withIndent("Gradle start parameters:") { + startParameters.let { + p.println("tasks = ${it.tasks}") + p.println("excluded tasks = ${it.excludedTasks}") + p.println("current dir = ${it.currentDir}") + p.println("project properties args = ${it.projectProperties}") + p.println("system properties args = ${it.systemProperties}") + } + } + p.println() + + if (failureMessages.isNotEmpty()) { + p.println("Build failed: ${failureMessages}") + p.println() + } + } + + private fun printMetrics( + buildTimesMetrics: Map, + performanceMetrics: Map, + nonIncrementalAttributes: Collection, + gcTimeMetrics: Map? = emptyMap(), + gcCountMetrics: Map? = emptyMap(), + aggregatedMetric: Boolean = false + ) { + printBuildTimes(buildTimesMetrics) + if (aggregatedMetric) p.println() + + printBuildPerformanceMetrics(performanceMetrics) + if (aggregatedMetric) p.println() + + printBuildAttributes(nonIncrementalAttributes) + + //TODO: KT-57310 Implement build GC metric in + if (!aggregatedMetric) { + printGcMetrics(gcTimeMetrics, gcCountMetrics) + } + } + + private fun printGcMetrics( + gcTimeMetrics: Map?, + gcCountMetrics: Map? + ) { + val keys = HashSet() + gcCountMetrics?.keys?.also { keys.addAll(it) } + gcTimeMetrics?.keys?.also { keys.addAll(it) } + if (keys.isEmpty()) return + + p.withIndent("GC metrics:") { + for (key in keys) { + p.println("$key:") + p.withIndent { + gcCountMetrics?.get(key)?.also { p.println("GC count: ${it}") } + gcTimeMetrics?.get(key)?.also { p.println("GC time: ${formatTime(it)}") } + } + } + } + } + + private fun printBuildTimes(buildTimes: Map) { + if (buildTimes.isEmpty()) return + + p.println("Time metrics:") + p.withIndent { + val visitedBuildTimes = HashSet() + fun printBuildTime(buildTime: BuildTime) { + if (!visitedBuildTimes.add(buildTime)) return + + val timeMs = buildTimes[buildTime] + if (timeMs != null) { + p.println("${buildTime.readableString}: ${formatTime(timeMs)}") + p.withIndent { + BuildTime.children[buildTime]?.forEach { printBuildTime(it) } + } + } else { + //Skip formatting if parent metric does not set + BuildTime.children[buildTime]?.forEach { printBuildTime(it) } + } + } + + for (buildTime in BuildTime.values()) { + if (buildTime.parent != null) continue + + printBuildTime(buildTime) + } + } + } + + private fun printBuildPerformanceMetrics(buildMetrics: Map) { + if (buildMetrics.isEmpty()) return + + p.withIndent("Size metrics:") { + for (metric in BuildPerformanceMetric.values()) { + buildMetrics[metric]?.let { printSizeMetric(metric, it) } + } + } + } + + private fun printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) { + fun BuildPerformanceMetric.numberOfAncestors(): Int { + var count = 0 + var parent: BuildPerformanceMetric? = parent + while (parent != null) { + count++ + parent = parent.parent + } + return count + } + + val indentLevel = sizeMetric.numberOfAncestors() + + repeat(indentLevel) { p.pushIndent() } + when (sizeMetric.type) { + ValueType.BYTES -> p.println("${sizeMetric.readableString}: ${formatSize(value)}") + ValueType.NUMBER -> p.println("${sizeMetric.readableString}: $value") + ValueType.NANOSECONDS -> p.println("${sizeMetric.readableString}: $value") + ValueType.MILLISECONDS -> p.println("${sizeMetric.readableString}: ${formatTime(value)}") + ValueType.TIME -> p.println("${sizeMetric.readableString}: ${formatter.format(value)}") + } + repeat(indentLevel) { p.popIndent() } + } + + private fun printBuildAttributes(buildAttributes: Collection) { + if (buildAttributes.isEmpty()) return + + val buildAttributesMap = buildAttributes.groupingBy { it }.eachCount() + p.withIndent("Build attributes:") { + val attributesByKind = buildAttributesMap.entries.groupBy { it.key.kind }.toSortedMap() + for ((kind, attributesCounts) in attributesByKind) { + printMap(p, kind.name, attributesCounts.associate { (k, v) -> k.readableString to v }) + } + } + } + + private fun printTaskOverview(statisticsData: Collection) { + var allTasksTimeMs = 0L + var kotlinTotalTimeMs = 0L + val kotlinTasks = ArrayList() + + for (task in statisticsData) { + val taskTimeMs = task.durationMs + allTasksTimeMs += taskTimeMs + + if (task.fromKotlinPlugin == true) { + kotlinTotalTimeMs += taskTimeMs + kotlinTasks.add(task) + } + } + + if (kotlinTasks.isEmpty()) { + p.println("No Kotlin task was run") + return + } + + val ktTaskPercent = (kotlinTotalTimeMs.toDouble() / allTasksTimeMs * 100).asString(1) + p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)") + + val table = TextTable("Time", "% of Kotlin time", "Task") + for (task in kotlinTasks.sortedWith(compareBy({ -it.durationMs }, { it.startTimeMs }))) { + val timeMs = task.durationMs + val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1) + table.addRow(formatTime(timeMs), "$percent %", task.taskName) + } + table.printTo(p) + p.println() + } + + private fun printTasksLog(statisticsData: List) { + for (task in statisticsData.sortedWith(compareBy({ -it.durationMs }, { it.startTimeMs }))) { + printTaskLog(task) + p.println() + } + } + + private fun printTaskLog(statisticsData: CompileStatisticsData) { + val skipMessage = statisticsData.skipMessage + if (skipMessage != null) { + p.println("Task '${statisticsData.taskName}' was skipped: $skipMessage") + } else { + p.println("Task '${statisticsData.taskName}' finished in ${formatTime(statisticsData.durationMs)}") + } + + if (statisticsData.icLogLines.isNotEmpty()) { + p.withIndent("Compilation log for task '${statisticsData.taskName}':") { + statisticsData.icLogLines.forEach { p.println(it) } + } + } + + if (printMetrics) { + printMetrics(statisticsData.buildTimesMetrics, statisticsData.performanceMetrics, statisticsData.nonIncrementalAttributes, + statisticsData.gcTimeMetrics, statisticsData.gcCountMetrics) + } + } +} \ No newline at end of file diff --git a/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/PlainTextBuildReportWriter.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/PlainTextBuildReportWriter.kt new file mode 100644 index 00000000000..9d37c959f2b --- /dev/null +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/PlainTextBuildReportWriter.kt @@ -0,0 +1,51 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistic.file + +import java.util.* +import kotlin.math.max + +internal fun printMap(p: Printer, name: String, mapping: Map) { + if (mapping.isEmpty()) return + + if (mapping.size == 1) { + p.println("$name: ${mapping.keys.single()}") + return + } + + p.withIndent("$name:") { + val sortedEnumMap = mapping.toSortedMap() + for ((k, v) in sortedEnumMap) { + p.println("$k($v)") + } + } +} + +internal class TextTable(vararg columnNames: String) { + private val rows = ArrayList>() + private val columnsCount = columnNames.size + private val maxLengths = IntArray(columnsCount) { columnNames[it].length } + + init { + rows.add(columnNames.toList()) + } + + fun addRow(vararg row: String) { + check(row.size == columnsCount) { "Row size ${row.size} differs from columns count $columnsCount" } + rows.add(row.toList()) + + for ((i, col) in row.withIndex()) { + maxLengths[i] = max(maxLengths[i], col.length) + } + } + + fun printTo(p: Printer) { + for (row in rows) { + val rowStr = row.withIndex().joinToString("|") { (i, col) -> col.padEnd(maxLengths[i], ' ') } + p.println(rowStr) + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/Printer.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/Printer.kt similarity index 68% rename from libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/Printer.kt rename to kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/Printer.kt index 71ef31d50de..ccec16865b8 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/Printer.kt +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/file/Printer.kt @@ -1,19 +1,9 @@ /* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.gradle.utils + +package org.jetbrains.kotlin.build.report.statistic.file import java.io.IOException diff --git a/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/formattingUtils.kt b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/formattingUtils.kt new file mode 100644 index 00000000000..cb769d6143f --- /dev/null +++ b/kotlin-build-statistic/src/org/jetbrains/kotlin/build/report/statistic/formattingUtils.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistic + +internal fun formatTime(ms: Long): String { + val seconds = ms.toDouble() / 1_000 + return seconds.asString(2) + " s" +} + +private const val kbSize = 1024 +private const val mbSize = kbSize * 1024 +private const val gbSize = mbSize * 1024 + +public fun formatSize(sizeInBytes: Long): String = when { + sizeInBytes / gbSize >= 1 -> "${(sizeInBytes.toDouble() / gbSize).asString(1)} GB" + sizeInBytes / mbSize >= 1 -> "${(sizeInBytes.toDouble() / mbSize).asString(1)} MB" + sizeInBytes / kbSize >= 1 -> "${(sizeInBytes.toDouble() / kbSize).asString(1)} KB" + else -> "$sizeInBytes B" +} + +internal fun Double.asString(decPoints: Int): String = "%,.${decPoints}f".format(this) \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt index 5ff733af5f4..9f6f19303d6 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt @@ -38,12 +38,13 @@ class KonanLibrariesResolveSupport( object : Logger { private val collector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) override fun warning(message: String)= collector.report(CompilerMessageSeverity.STRONG_WARNING, message) - override fun error(message: String) = collector.report(CompilerMessageSeverity.ERROR, message) + override fun error(message: String, throwable: Throwable?) = collector.report(CompilerMessageSeverity.ERROR, message) override fun log(message: String) = collector.report(CompilerMessageSeverity.LOGGING, message) override fun fatal(message: String): Nothing { collector.report(CompilerMessageSeverity.ERROR, message) throw KonanCompilationException() } + override fun lifecycle(message: String) =collector.report(CompilerMessageSeverity.LOGGING, message) } private val resolver = defaultResolver( diff --git a/kotlin-native/klib/src/main/kotlin/org/jetbrains/kotlin/cli/klib/main.kt b/kotlin-native/klib/src/main/kotlin/org/jetbrains/kotlin/cli/klib/main.kt index 08385bcbcfa..2c03e1c6433 100644 --- a/kotlin-native/klib/src/main/kotlin/org/jetbrains/kotlin/cli/klib/main.kt +++ b/kotlin-native/klib/src/main/kotlin/org/jetbrains/kotlin/cli/klib/main.kt @@ -88,9 +88,10 @@ fun error(text: String): Nothing { object KlibToolLogger : Logger { override fun warning(message: String) = org.jetbrains.kotlin.cli.klib.warn(message) - override fun error(message: String) = org.jetbrains.kotlin.cli.klib.warn(message) + override fun error(message: String, throwable: Throwable?) = org.jetbrains.kotlin.cli.klib.warn(message) override fun fatal(message: String) = org.jetbrains.kotlin.cli.klib.error(message) override fun log(message: String) = println(message) + override fun lifecycle(message: String) = println(message) } val defaultRepository = File(DependencyProcessor.localKonanDir.resolve("klib").absolutePath) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin-integration-tests/build.gradle.kts index 58a5e5a042b..291e220bba1 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/build.gradle.kts @@ -74,6 +74,7 @@ dependencies { testImplementation(project(":kotlin-compiler-embeddable")) testImplementation(commonDependency("org.jetbrains.intellij.deps:jdom")) testImplementation(project(":compiler:cli-common")) + testImplementation(project(":kotlin-build-statistic")) // testCompileOnly dependency on non-shaded artifacts is needed for IDE support // testRuntimeOnly on shaded artifact is needed for running tests with shaded compiler testCompileOnly(project(":kotlin-gradle-plugin-test-utils-embeddable")) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt index 5a68ae82bdb..0c706d486ab 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt @@ -17,9 +17,8 @@ import io.ktor.server.engine.* import io.ktor.server.netty.* import io.ktor.util.* import io.ktor.util.collections.* -import org.gradle.api.logging.LogLevel import org.gradle.util.GradleVersion -import org.jetbrains.kotlin.gradle.plugin.stat.* +import org.jetbrains.kotlin.build.report.statistic.* import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.testbase.* import org.junit.jupiter.api.DisplayName diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index 4d9ddd881e5..850ab169b19 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -52,7 +52,6 @@ dependencies { commonCompileOnly(intellijCore()) commonCompileOnly(commonDependency("org.jetbrains.teamcity:serviceMessages")) commonCompileOnly("com.gradle:gradle-enterprise-gradle-plugin:3.12.4") - commonCompileOnly(commonDependency("com.google.code.gson:gson")) commonCompileOnly(commonDependency("com.google.guava:guava")) commonCompileOnly("de.undercouch:gradle-download-task:4.1.1") commonCompileOnly("com.github.gundy:semver4j:0.16.4:nodeps") { @@ -67,6 +66,7 @@ dependencies { commonImplementation(project(":native:kotlin-klib-commonizer-api")) commonImplementation(project(":kotlin-project-model")) commonImplementation(project(":compiler:build-tools:kotlin-build-tools-api")) + commonImplementation(project(":kotlin-build-statistic")) commonRuntimeOnly(project(":kotlin-compiler-embeddable")) commonRuntimeOnly(project(":kotlin-annotation-processing-embeddable")) @@ -111,6 +111,7 @@ dependencies { testImplementation(commonDependency("junit:junit")) testImplementation(project(":kotlin-gradle-statistics")) testImplementation(project(":kotlin-tooling-metadata")) + testImplementation(projectTests(":kotlin-build-statistic")) } configurations.commonCompileClasspath.get().exclude("org.jetbrains.kotlinx", "kotlinx-coroutines-core") diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 95f71103f26..ef75034450b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -15,6 +15,7 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinVersion import org.jetbrains.kotlin.gradle.logging.* import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers +import org.jetbrains.kotlin.build.report.statistic.StatTag import org.jetbrains.kotlin.gradle.report.* import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState @@ -26,6 +27,7 @@ import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.jetbrains.kotlin.incremental.util.ExceptionLocation import org.jetbrains.kotlin.incremental.util.reportException import org.jetbrains.kotlin.util.removeSuffixIfPresent +import org.jetbrains.kotlin.utils.addToStdlib.ifTrue import org.slf4j.LoggerFactory import java.io.* import java.net.URLClassLoader @@ -34,6 +36,7 @@ import java.util.* import java.util.concurrent.Callable import java.util.concurrent.Executors import javax.inject.Inject +import kotlin.collections.HashSet internal class ProjectFilesForCompilation( val projectRootFile: File, @@ -41,10 +44,9 @@ internal class ProjectFilesForCompilation( val sessionFlagFile: File, val buildDir: File ) : Serializable { - //TODO - constructor(logger: Logger, projectDir:File, buildDir: File, prjectName: String, projectCacheDirProvider: File, sessionDir: File) : this( + constructor(logger: Logger, projectDir:File, buildDir: File, projectName: String, projectCacheDirProvider: File, sessionDir: File) : this( projectRootFile = projectDir, - clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(logger, prjectName), + clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(logger, projectName), sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(logger, sessionDir, projectCacheDirProvider), buildDir = buildDir ) @@ -114,7 +116,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( TaskLoggers.get(taskPath)?.let { GradleKotlinLogger(it).apply { debug("Using '$taskPath' logger") } } ?: run { val logger = LoggerFactory.getLogger("GradleKotlinCompilerWork") - val kotlinLogger = if (logger is org.gradle.api.logging.Logger) { + val kotlinLogger = if (logger is Logger) { GradleKotlinLogger(logger) } else SL4JKotlinLogger(logger) @@ -144,8 +146,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( kotlinLanguageVersion = kotlinLanguageVersion, changedFiles = incrementalCompilationEnvironment?.changedFiles, compilerArguments = if (reportingSettings.includeCompilerArguments) compilerArgs else emptyArray(), - withAbiSnapshot = incrementalCompilationEnvironment?.withAbiSnapshot, - withArtifactTransform = incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled + tags = collectStatTags(), ) metrics.endMeasure(BuildTime.RUN_COMPILATION_IN_WORKER) val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines, taskInfo = taskInfo) @@ -153,6 +154,15 @@ internal class GradleKotlinCompilerWork @Inject constructor( } } + private fun collectStatTags(): Set { + val statTags = HashSet() + incrementalCompilationEnvironment?.withAbiSnapshot?.ifTrue { statTags.add(StatTag.ABI_SNAPSHOT) } + if (incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) { + statTags.add(StatTag.ARTIFACT_TRANSFORM) + } + return statTags + } + private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): Pair { with(log) { kotlinDebug { "Kotlin compiler class: $compilerClassName" } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt index 115dda4ed12..12d71e2c9e9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt @@ -12,8 +12,6 @@ import org.gradle.api.provider.Provider import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters import org.jetbrains.kotlin.gradle.logging.kotlinDebug -import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults -import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.utils.projectCacheDir import java.io.File @@ -65,9 +63,6 @@ internal abstract class KotlinGradleBuildServices : BuildService - val buildEventsListenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project) - buildEventsListenerRegistryHolder.listenerRegistry.onTaskCompletion(buildMetricsService) - BuildReportsService.registerIfAbsent(project, buildMetricsService).also { - buildEventsListenerRegistryHolder.listenerRegistry.onTaskCompletion(it) - } - } + BuildMetricsService.registerIfAbsent(project) } private fun addKotlinCompilerConfiguration(project: Project) { diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt index 0ca7694660e..1f01c34f164 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsService.kt @@ -7,14 +7,16 @@ package org.jetbrains.kotlin.gradle.report import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.invocation.Gradle import org.gradle.api.logging.Logging +import org.gradle.api.provider.ListProperty import org.gradle.api.provider.Property import org.gradle.api.provider.Provider import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters import org.gradle.api.tasks.Internal import org.gradle.tooling.events.FailureResult -import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.task.TaskExecutionResult import org.gradle.tooling.events.task.TaskFailureResult @@ -24,13 +26,22 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetrics import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.statistic.HttpReportService +import org.jetbrains.kotlin.build.report.statistic.HttpReportServiceImpl +import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder +import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults +import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters +import org.jetbrains.kotlin.build.report.statistic.StatTag import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService +import org.jetbrains.kotlin.gradle.report.BuildReportsService.Companion.getStartParameters import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.gradle.tasks.withType import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject -import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics +import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable +import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics +import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentLinkedQueue import org.jetbrains.kotlin.gradle.dsl.KotlinVersion @@ -40,13 +51,27 @@ internal interface UsesBuildMetricsService : Task { val buildMetricsService: Property } -abstract class BuildMetricsService : BuildService, OperationCompletionListener { +abstract class BuildMetricsService : BuildService, AutoCloseable { + + //Part of BuildReportService + interface Parameters : BuildServiceParameters { + val startParameters: Property + val reportingSettings: Property + val httpService: Property + + val projectDir: DirectoryProperty + val label: Property + val projectName: Property + val kotlinVersion: Property + val buildConfigurationTags: ListProperty + } private val log = Logging.getLogger(this.javaClass) + private val buildReportService = BuildReportsService() // Tasks and transforms' records - internal val buildOperationRecords = ConcurrentLinkedQueue() - internal val failureMessages = ConcurrentLinkedQueue() + private val buildOperationRecords = ConcurrentLinkedQueue() + private val failureMessages = ConcurrentLinkedQueue() // Info for tasks only private val taskPathToMetricsReporter = ConcurrentHashMap() @@ -76,69 +101,87 @@ abstract class BuildMetricsService : BuildService, failureMessage?.let { failureMessages.add(it) } } - override fun onFinish(event: FinishEvent?) { - if (event is TaskFinishEvent) { - val result = event.result - val taskPath = event.descriptor.taskPath - val totalTimeMs = result.endTime - result.startTime + private fun updateBuildOperationRecord(event: TaskFinishEvent): TaskRecord { + val result = event.result + val taskPath = event.descriptor.taskPath + val totalTimeMs = result.endTime - result.startTime - val buildMetrics = BuildMetrics() - buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, totalTimeMs) - taskPathToMetricsReporter[taskPath]?.let { - buildMetrics.addAll(it.getMetrics()) - } - val taskExecutionResult = TaskExecutionResults[taskPath] - taskExecutionResult?.buildMetrics?.also { - buildMetrics.addAll(it) + val buildMetrics = BuildMetrics() + buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, totalTimeMs) + taskPathToMetricsReporter[taskPath]?.let { + buildMetrics.addAll(it.getMetrics()) + } + val taskExecutionResult = TaskExecutionResults[taskPath] + taskExecutionResult?.buildMetrics?.also { + buildMetrics.addAll(it) - KotlinBuildStatsService.applyIfInitialised { collector -> - collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs) - collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult) - val metricsMap = buildMetrics.buildPerformanceMetrics.asMap() + KotlinBuildStatsService.applyIfInitialised { collector -> + collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs) + collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult) + val metricsMap = buildMetrics.buildPerformanceMetrics.asMap() - val linesOfCode = metricsMap[BuildPerformanceMetric.ANALYZED_LINES_NUMBER] - if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) { - collector.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode) - collector.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode) - metricsMap[BuildPerformanceMetric.ANALYSIS_LPS]?.also { - collector.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, it, null, linesOfCode) - } - metricsMap[BuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value -> - collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode) - } + val linesOfCode = metricsMap[BuildPerformanceMetric.ANALYZED_LINES_NUMBER] + if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) { + collector.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode) + collector.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode) + metricsMap[BuildPerformanceMetric.ANALYSIS_LPS]?.also { value -> + collector.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, value, null, linesOfCode) + } + metricsMap[BuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value -> + collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode) } - collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1) - collector.report( - NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT, - if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0 - ) } - } - - buildOperationRecords.add( - TaskRecord( - path = taskPath, - classFqName = taskPathToTaskClass[taskPath] ?: "unknown", - startTimeMs = result.startTime, - totalTimeMs = totalTimeMs, - buildMetrics = buildMetrics, - didWork = result is TaskExecutionResult, - skipMessage = (result as? TaskSkippedResult)?.skipMessage, - icLogLines = taskExecutionResult?.icLogLines ?: emptyList(), - kotlinLanguageVersion = taskExecutionResult?.taskInfo?.kotlinLanguageVersion + collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1) + collector.report( + NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT, + if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0 ) - ) - if (result is TaskFailureResult) { - failureMessages.addAll(result.failures.map { it.message }) } } + + val buildOperation = TaskRecord( + path = taskPath, + classFqName = taskPathToTaskClass[taskPath] ?: "unknown", + startTimeMs = result.startTime, + totalTimeMs = totalTimeMs, + buildMetrics = buildMetrics, + didWork = result is TaskExecutionResult, + skipMessage = (result as? TaskSkippedResult)?.skipMessage, + icLogLines = taskExecutionResult?.icLogLines ?: emptyList(), + changedFiles = taskExecutionResult?.taskInfo?.changedFiles, + compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments ?: emptyArray(), + kotlinLanguageVersion = taskExecutionResult?.taskInfo?.kotlinLanguageVersion, + statTags = taskExecutionResult?.taskInfo?.tags ?: emptySet() + ) + buildOperationRecords.add(buildOperation) + if (result is TaskFailureResult) { + failureMessages.addAll(result.failures.map { it.message }) + } + return buildOperation + } + + override fun close() { + buildReportService.close(buildOperationRecords, failureMessages.toList(), parameters.toBuildReportParameters()) } companion object { private val serviceClass = BuildMetricsService::class.java private val serviceName = "${serviceClass.name}_${serviceClass.classLoader.hashCode()}" - private fun registerIfAbsentImpl(project: Project): Provider? { + private fun Parameters.toBuildReportParameters() = BuildReportParameters( + startParameters = startParameters.get(), + reportingSettings = reportingSettings.get(), + httpService = httpService.orNull, + projectDir = projectDir.asFile.get(), + label = label.orNull, + projectName = projectName.get(), + kotlinVersion = kotlinVersion.get(), + additionalTags = HashSet(buildConfigurationTags.get()) + ) + + private fun registerIfAbsentImpl( + project: Project, + ): Provider? { // Return early if the service was already registered to avoid the overhead of reading the reporting settings below project.gradle.sharedServices.registrations.findByName(serviceName)?.let { @Suppress("UNCHECKED_CAST") @@ -151,7 +194,59 @@ abstract class BuildMetricsService : BuildService, return null } - return project.gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) {}!! + val kotlinVersion = project.getKotlinPluginVersion() + + return project.gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) { + it.parameters.label.set(reportingSettings.buildReportLabel) + it.parameters.projectName.set(project.rootProject.name) + it.parameters.kotlinVersion.set(kotlinVersion) + it.parameters.startParameters.set(getStartParameters(project)) + it.parameters.reportingSettings.set(reportingSettings) + reportingSettings.httpReportSettings?.let { httpSettings -> + it.parameters.httpService.set( + HttpReportServiceImpl( + httpSettings.url, + httpSettings.user, + httpSettings.password + ) + ) + } + it.parameters.projectDir.set(project.rootProject.layout.projectDirectory) + //init gradle tags for build scan and http reports + it.parameters.buildConfigurationTags.value(setupTags(project.gradle)) + }.also { + subscribeForTaskEvents(project, it) + } + + } + + private fun subscribeForTaskEvents(project: Project, buildMetricService: Provider) { + // BuildScanExtension cant be parameter nor BuildService's field + val buildScanExtension = project.rootProject.extensions.findByName("buildScan") + val buildScan = buildScanExtension?.let { BuildScanExtensionHolder(it) } + val buildReportService = buildMetricService.map { it.buildReportService } + BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(project.provider { + OperationCompletionListener { event -> + if (event is TaskFinishEvent) { + val buildOperation = buildMetricService.get().updateBuildOperationRecord(event) + val buildParameters = buildMetricService.get().parameters.toBuildReportParameters() + buildReportService.get().onFinish(event, buildOperation, buildParameters, buildScan) + } + } + }) + + val buildScanReportSettings = buildMetricService.get().parameters.reportingSettings.orNull?.buildScanReportSettings + if (buildScanReportSettings != null) { + buildScan?.also { + buildReportService.get().initBuildScanTags( + it, buildMetricService.get().parameters.label.orNull + ) + it.buildScan.buildFinished { + buildReportService.get().addCollectedTags(buildScan) + } + } + } + } fun registerIfAbsent(project: Project) = registerIfAbsentImpl(project)?.also { serviceProvider -> @@ -161,6 +256,17 @@ abstract class BuildMetricsService : BuildService, } } } + + private fun setupTags(gradle: Gradle): ArrayList { + val additionalTags = ArrayList() + if (isConfigurationCacheAvailable(gradle)) { + additionalTags.add(StatTag.CONFIGURATION_CACHE) + } + if (gradle.startParameter.isBuildCacheEnabled) { + additionalTags.add(StatTag.BUILD_CACHE) + } + return additionalTags + } } } @@ -174,7 +280,10 @@ internal class TaskRecord( override val didWork: Boolean, override val skipMessage: String?, override val icLogLines: List, - val kotlinLanguageVersion: KotlinVersion? + val kotlinLanguageVersion: KotlinVersion?, + val changedFiles: ChangedFiles? = null, + val compilerArguments: Array = emptyArray(), + val statTags: Set = emptySet(), ) : BuildOperationRecord { override val isFromKotlinPlugin: Boolean = classFqName.startsWith("org.jetbrains.kotlin") } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt index beb58a3328b..d944f85a1dd 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt @@ -6,33 +6,21 @@ package org.jetbrains.kotlin.gradle.report import org.gradle.api.Project -import org.gradle.api.Task -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.invocation.Gradle import org.gradle.api.logging.Logging -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.provider.Provider -import org.gradle.api.services.BuildService -import org.gradle.api.services.BuildServiceParameters -import org.gradle.api.tasks.Internal -import org.gradle.tooling.events.FinishEvent -import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.task.TaskFinishEvent import org.jetbrains.kotlin.build.report.metrics.ValueType -import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder -import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion -import org.jetbrains.kotlin.gradle.plugin.stat.BuildFinishStatisticsData -import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData -import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters -import org.jetbrains.kotlin.gradle.plugin.stat.StatTag +import org.jetbrains.kotlin.build.report.statistic.HttpReportService +import org.jetbrains.kotlin.build.report.statistic.file.FileReportService +import org.jetbrains.kotlin.build.report.statistic.formatSize +import org.jetbrains.kotlin.build.report.statistic.BuildFinishStatisticsData +import org.jetbrains.kotlin.build.report.statistic.CompileStatisticsData +import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters +import org.jetbrains.kotlin.build.report.statistic.StatTag import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData -import org.jetbrains.kotlin.gradle.tasks.withType -import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject -import org.jetbrains.kotlin.gradle.utils.formatSize -import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable +import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult import java.io.File +import java.lang.management.ManagementFactory import java.net.InetAddress import java.text.SimpleDateFormat import java.util.* @@ -41,18 +29,16 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import kotlin.system.measureTimeMillis -internal interface UsesBuildReportsService : Task { - @get:Internal - val buildReportsService: Property -} - -abstract class BuildReportsService : BuildService, AutoCloseable, OperationCompletionListener { +//Because of https://github.com/gradle/gradle/issues/23359 gradle issue, two build services interaction is not reliable at the end of the build +//Switch back to proper BuildService as soon as this issue is fixed +class BuildReportsService { private val log = Logging.getLogger(this.javaClass) + private val loggerAdapter = GradleLoggerAdapter(log) private val startTime = System.nanoTime() private val buildUuid = UUID.randomUUID().toString() - private var executorService: ExecutorService = Executors.newSingleThreadExecutor() + private val executorService: ExecutorService = Executors.newSingleThreadExecutor() private val tags = LinkedHashSet() private var customValues = 0 // doesn't need to be thread-safe @@ -61,33 +47,46 @@ abstract class BuildReportsService : BuildService - val reportingSettings: Property - var buildMetricsService: Provider - val httpService: Property - - val projectDir: DirectoryProperty - val label: Property - val projectName: Property - val kotlinVersion: Property - val additionalTags: ListProperty - } - - override fun close() { + fun close( + buildOperationRecords: Collection, + failureMessages: List, + parameters: BuildReportParameters + ) { val buildData = BuildExecutionData( - startParameters = parameters.startParameters.get(), - failureMessages = parameters.buildMetricsService.orNull?.failureMessages?.toList() ?: emptyList(), - buildOperationRecord = parameters.buildMetricsService.orNull?.buildOperationRecords?.sortedBy { it.startTimeMs } ?: emptyList() + startParameters = parameters.startParameters, + failureMessages = failureMessages, + buildOperationRecord = buildOperationRecords.sortedBy { it.startTimeMs } ) - val reportingSettings = parameters.reportingSettings.get() + val reportingSettings = parameters.reportingSettings reportingSettings.httpReportSettings?.also { - executorService.submit { reportBuildFinish() } // + executorService.submit { reportBuildFinish(parameters) } } reportingSettings.fileReportSettings?.also { - reportBuildStatInFile(it, buildData) + FileReportService.reportBuildStatInFile( + it.buildReportDir, + parameters.projectName, + it.includeMetricsInReport, + buildOperationRecords.mapNotNull { + prepareData( + taskResult = null, + it.path, + it.startTimeMs, + it.totalTimeMs + it.startTimeMs, + parameters.projectName, + buildUuid, + parameters.label, + parameters.kotlinVersion, + it, + onlyKotlinTask = false, + parameters.additionalTags + ) + }, + parameters.startParameters, + failureMessages.filter { it.isNotEmpty() }, + loggerAdapter + ) } reportingSettings.singleOutputFile?.also { singleOutputFile -> @@ -98,26 +97,20 @@ abstract class BuildReportsService : BuildService - if (event is TaskFinishEvent) { - val data = - prepareData( - event, - parameters.projectName.get(), - buildUuid, - parameters.label.orNull, - parameters.kotlinVersion.get(), - parameters.buildMetricsService.get().buildOperationRecords, - parameters.additionalTags.get() - ) - data?.also { - executorService.submit { - httpService.sendData(data, log) - } + private fun addHttpReport( + event: TaskFinishEvent, + buildOperationRecord: BuildOperationRecord, + parameters: BuildReportParameters + ) { + parameters.httpService?.also { httpService -> + val data = + prepareData( + event, + parameters.projectName, + buildUuid, + parameters.label, + parameters.kotlinVersion, + buildOperationRecord, + onlyKotlinTask = true, + parameters.additionalTags + ) + data?.also { + executorService.submit { + httpService.sendData(data, loggerAdapter) } } } } + private fun addBuildScanReport( + event: TaskFinishEvent, + buildOperationRecord: BuildOperationRecord, + parameters: BuildReportParameters, + buildScanExtension: BuildScanExtensionHolder + ) { + val buildScanSettings = parameters.reportingSettings.buildScanReportSettings ?: return - private fun addBuildScanReport(event: FinishEvent?, buildScan: BuildScanExtensionHolder) { - val buildScanSettings = parameters.reportingSettings.orNull?.buildScanReportSettings - if (buildScanSettings != null && buildScan.buildScan != null) { - if (event is TaskFinishEvent) { - val (collectDataDuration, compileStatData) = measureTimeMillisWithResult { - prepareData( - event, parameters.projectName.get(), buildUuid, parameters.label.orNull, - parameters.kotlinVersion.get(), - parameters.buildMetricsService.get().buildOperationRecords, - metricsToShow = buildScanSettings.metrics - ) - } - log.debug("Collect data takes $collectDataDuration: $compileStatData") + val (collectDataDuration, compileStatData) = measureTimeMillisWithResult { + prepareData( + event, + parameters.projectName, buildUuid, parameters.label, + parameters.kotlinVersion, + buildOperationRecord, + metricsToShow = buildScanSettings.metrics + ) + } + log.debug("Collect data takes $collectDataDuration: $compileStatData") - compileStatData?.also { - addBuildScanReport(it, buildScanSettings.customValueLimit, buildScan) - } - } + compileStatData?.also { + addBuildScanReport(it, buildScanSettings.customValueLimit, buildScanExtension) } } private fun addBuildScanReport(data: CompileStatisticsData, customValuesLimit: Int, buildScan: BuildScanExtensionHolder) { val elapsedTime = measureTimeMillis { - data.tags.forEach { tags.add(it) } - buildScan.buildScan?.also { - if (customValues < customValuesLimit) { - readableString(data).forEach { - if (customValues < customValuesLimit) { - addBuildScanValue(buildScan, data, it) - } else { - log.debug( - "Can't add any more custom values into build scan." + - " Statistic data for ${data.taskName} was cut due to custom values limit." - ) - } + tags.addAll(data.tags) + if (customValues < customValuesLimit) { + readableString(data).forEach { + if (customValues < customValuesLimit) { + addBuildScanValue(buildScan, data, it) + } else { + log.debug( + "Can't add any more custom values into build scan." + + " Statistic data for ${data.taskName} was cut due to custom values limit." + ) } - } else { - log.debug("Can't add any more custom values into build scan.") } + } else { + log.debug("Can't add any more custom values into build scan.") } } @@ -228,7 +224,7 @@ abstract class BuildReportsService : BuildService - ): Provider? { - val serviceClass = BuildReportsService::class.java - val serviceName = "${serviceClass.name}_${serviceClass.classLoader.hashCode()}" - - val reportingSettings = reportingSettings(project) - if (reportingSettings.buildReportOutputs.isEmpty()) { - return null //no need to collect data - } - - val kotlinVersion = project.getKotlinPluginVersion() - val gradle = project.gradle - project.gradle.sharedServices.registrations.findByName(serviceName)?.let { - @Suppress("UNCHECKED_CAST") - return it.service as Provider - } - - return gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) { - it.parameters.label.set(reportingSettings.buildReportLabel) - it.parameters.projectName.set(project.rootProject.name) - it.parameters.kotlinVersion.set(kotlinVersion) - it.parameters.startParameters.set(getStartParameters(project)) - it.parameters.reportingSettings.set(reportingSettings) - reportingSettings.httpReportSettings?.let { httpSettings -> it.parameters.httpService.set(HttpReportServiceImpl(httpSettings)) } - it.parameters.buildMetricsService = buildMetricsService - it.parameters.projectDir.set(project.rootProject.layout.projectDirectory) - - //init gradle tags for build scan and http reports - it.parameters.additionalTags.value(setupTags(gradle)) - }.also { buildServiceProvider -> - if (reportingSettings.httpReportSettings != null) { - BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildServiceProvider) - } - - val buildScanExtension = project.rootProject.extensions.findByName("buildScan") - if (reportingSettings.buildScanReportSettings != null && buildScanExtension != null) { - val buildScan = BuildScanExtensionHolder(buildScanExtension) - buildServiceProvider.get().initBuildScanTags(buildScan) - BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(project.provider { - OperationCompletionListener { event -> - buildServiceProvider.get().addBuildScanReport(event, buildScan) - } - }) - buildScan.buildScan?.buildFinished { - buildServiceProvider.get().addCollectedTags(buildScan) - } - } - } - } - - fun registerIfAbsent(project: Project, buildMetricsService: Provider) = - registerIfAbsentImpl(project, buildMetricsService)?.also { serviceProvider -> - SingleActionPerProject.run(project, UsesBuildReportsService::class.java.name) { - project.tasks.withType().configureEach { task -> - task.usesService(serviceProvider) - } - } - } - - private fun setupTags(gradle: Gradle): ArrayList { - val additionalTags = ArrayList() - if (isConfigurationCacheAvailable(gradle)) { - additionalTags.add(StatTag.CONFIGURATION_CACHE) - } - if (gradle.startParameter.isBuildCacheEnabled) { - additionalTags.add(StatTag.BUILD_CACHE) - } - return additionalTags - } - val hostName: String? = try { InetAddress.getLocalHost().hostName } catch (_: Exception) { @@ -422,3 +350,15 @@ enum class TaskExecutionState { UP_TO_DATE ; } + +data class BuildReportParameters( + val startParameters: GradleBuildStartParameters, + val reportingSettings: ReportingSettings, + val httpService: HttpReportService?, + + val projectDir: File, + val label: String?, + val projectName: String, + val kotlinVersion: String, + val additionalTags: Set +) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildScanExtensionHolder.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildScanExtensionHolder.kt index f52f870989b..721e7ddb54b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildScanExtensionHolder.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildScanExtensionHolder.kt @@ -7,6 +7,6 @@ package org.jetbrains.kotlin.gradle.report import com.gradle.scan.plugin.BuildScanExtension -class BuildScanExtensionHolder(val buildScan: BuildScanExtension?) : java.io.Serializable { - constructor(extension: Any?) : this(extension as BuildScanExtension) +class BuildScanExtensionHolder(val buildScan: BuildScanExtension) : java.io.Serializable { + constructor(extension: Any) : this(extension as BuildScanExtension) } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/GradleLogger.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/GradleLogger.kt new file mode 100644 index 00000000000..99187e63f59 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/GradleLogger.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import org.gradle.api.logging.Logger + +class GradleLoggerAdapter(private val log: Logger) : org.jetbrains.kotlin.util.Logger { + override fun log(message: String) { + log.info(message) + } + + override fun warning(message: String) { + log.warn(message) + } + + override fun fatal(message: String): Nothing { + log.error(message) + kotlin.error(message) + } + + override fun error(message: String, throwable: Throwable?) { + throwable?.let { log.error(message, throwable) } ?: log.error(message) + } + + override fun lifecycle(message: String) { + log.lifecycle(message) + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt deleted file mode 100644 index 067afd22695..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.report - -import org.gradle.api.logging.Logger -import org.jetbrains.kotlin.build.report.metrics.* -import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData -import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord -import org.jetbrains.kotlin.gradle.utils.Printer -import org.jetbrains.kotlin.gradle.utils.asString -import org.jetbrains.kotlin.gradle.utils.formatSize -import org.jetbrains.kotlin.gradle.utils.formatTime -import java.io.File -import java.io.Serializable -import java.util.* -import kotlin.math.max - -internal class PlainTextBuildReportWriter( - private val outputFile: File, - private val printMetrics: Boolean -) : Serializable { - - private lateinit var p: Printer - - fun process(build: BuildExecutionData, log: Logger) { - val buildReportPath = outputFile.toPath().toUri().toString() - try { - outputFile.parentFile.mkdirs() - if (!(outputFile.parentFile.exists() && outputFile.parentFile.isDirectory)) { - log.error("Kotlin build report cannot be created: '$outputFile.parentFile' is a file or do not have permissions to create") - return - } - - outputFile.bufferedWriter().use { writer -> - p = Printer(writer) - printBuildReport(build) - } - - log.lifecycle("Kotlin build report is written to $buildReportPath") - } catch (e: Exception) { - log.error("Could not write Kotlin build report to $buildReportPath", e) - } - } - - private fun printBuildReport(build: BuildExecutionData) { - // NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms. - // Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound - // a bit unfamiliar. - // TODO: If it is confusing, consider renaming "tasks" to "build operations" in this class. - printBuildInfo(build) - if (printMetrics) { - printMetrics(build.aggregatedMetrics, aggregatedMetric = true) - p.println() - } - printTaskOverview(build) - printTasksLog(build) - } - - private fun printBuildInfo(build: BuildExecutionData) { - p.withIndent("Gradle start parameters:") { - build.startParameters.let { - p.println("tasks = ${it.tasks}") - p.println("excluded tasks = ${it.excludedTasks}") - p.println("current dir = ${it.currentDir}") - p.println("project properties args = ${it.projectProperties}") - p.println(" system properties args = ${it.systemProperties}") - } - } - p.println() - - if (build.failureMessages.isNotEmpty()) { - p.println("Build failed: ${build.failureMessages}") - p.println() - } - } - - private fun printMetrics(buildMetrics: BuildMetrics, aggregatedMetric: Boolean = false) { - printBuildTimes(buildMetrics.buildTimes) - if (aggregatedMetric) p.println() - - printBuildPerformanceMetrics(buildMetrics.buildPerformanceMetrics) - if (aggregatedMetric) p.println() - - printBuildAttributes(buildMetrics.buildAttributes) - - //TODO: KT-57310 Implement build GC metric in - if (!aggregatedMetric) { - printGcMetrics(buildMetrics.gcMetrics) - } - } - - private fun printBuildTimes(buildTimes: BuildTimes) { - val buildTimesMs = buildTimes.asMapMs() - - if (buildTimesMs.isEmpty()) return - - p.println("Time metrics:") - p.withIndent { - val visitedBuildTimes = HashSet() - fun printBuildTime(buildTime: BuildTime) { - if (!visitedBuildTimes.add(buildTime)) return - - val timeMs = buildTimesMs[buildTime] - if (timeMs != null) { - p.println("${buildTime.readableString}: ${formatTime(timeMs)}") - p.withIndent { - BuildTime.children[buildTime]?.forEach { printBuildTime(it) } - } - } else { - //Skip formatting if parent metric does not set - BuildTime.children[buildTime]?.forEach { printBuildTime(it) } - } - } - - for (buildTime in BuildTime.values()) { - if (buildTime.parent != null) continue - - printBuildTime(buildTime) - } - } - } - - private fun printBuildPerformanceMetrics(buildMetrics: BuildPerformanceMetrics) { - val allBuildMetrics = buildMetrics.asMap() - if (allBuildMetrics.isEmpty()) return - - p.withIndent("Size metrics:") { - for (metric in BuildPerformanceMetric.values()) { - allBuildMetrics[metric]?.let { printSizeMetric(metric, it) } - } - } - } - - private fun printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) { - fun BuildPerformanceMetric.numberOfAncestors(): Int { - var count = 0 - var parent: BuildPerformanceMetric? = parent - while (parent != null) { - count++ - parent = parent.parent - } - return count - } - - val indentLevel = sizeMetric.numberOfAncestors() - - repeat(indentLevel) { p.pushIndent() } - when (sizeMetric.type) { - ValueType.BYTES -> p.println("${sizeMetric.readableString}: ${formatSize(value)}") - ValueType.NUMBER -> p.println("${sizeMetric.readableString}: $value") - } - repeat(indentLevel) { p.popIndent() } - } - - private fun printBuildAttributes(buildAttributes: BuildAttributes) { - val allAttributes = buildAttributes.asMap() - if (allAttributes.isEmpty()) return - - p.withIndent("Build attributes:") { - val attributesByKind = allAttributes.entries.groupBy { it.key.kind }.toSortedMap() - for ((kind, attributesCounts) in attributesByKind) { - printMap(p, kind.name, attributesCounts.map { (k, v) -> k.readableString to v }.toMap()) - } - } - } - - private fun printGcMetrics(gcMetrics: GcMetrics) { - val allGcMetrics = gcMetrics.asMap() - if (allGcMetrics.isEmpty()) return - p.withIndent("GC metrics:") { - for (gcMetric in allGcMetrics) { - p.println("${gcMetric.key}:") - p.withIndent { - p.println("GC count: ${gcMetric.value.count}") - p.println("GC time: ${formatTime(gcMetric.value.time)}") - } - } - } - } - - private fun printTaskOverview(build: BuildExecutionData) { - var allTasksTimeMs = 0L - var kotlinTotalTimeMs = 0L - val kotlinTasks = ArrayList() - - for (task in build.buildOperationRecord) { - val taskTimeMs = task.totalTimeMs - allTasksTimeMs += taskTimeMs - - if (task.isFromKotlinPlugin) { - kotlinTotalTimeMs += taskTimeMs - kotlinTasks.add(task) - } - } - - if (kotlinTasks.isEmpty()) { - p.println("No Kotlin task was run") - return - } - - val ktTaskPercent = (kotlinTotalTimeMs.toDouble() / allTasksTimeMs * 100).asString(1) - p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)") - - val table = TextTable("Time", "% of Kotlin time", "Task") - for (task in kotlinTasks.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) { - val timeMs = task.totalTimeMs - val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1) - table.addRow(formatTime(timeMs), "$percent %", task.path) - } - table.printTo(p) - p.println() - } - - private fun printTasksLog(build: BuildExecutionData) { - for (task in build.buildOperationRecord.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) { - printTaskLog(task) - p.println() - } - } - - private fun printTaskLog(task: BuildOperationRecord) { - val skipMessage = task.skipMessage - if (skipMessage != null) { - p.println("Task '${task.path}' was skipped: $skipMessage") - } else { - p.println("Task '${task.path}' finished in ${formatTime(task.totalTimeMs)}") - } - - if (task.icLogLines.isNotEmpty()) { - p.withIndent("Compilation log for task '${task.path}':") { - task.icLogLines.forEach { p.println(it) } - } - } - - if (printMetrics) { - printMetrics(task.buildMetrics) - } - } -} - -private fun printMap(p: Printer, name: String, mapping: Map) { - if (mapping.isEmpty()) return - - if (mapping.size == 1) { - p.println("$name: ${mapping.keys.single()}") - return - } - - p.withIndent("$name:") { - val sortedEnumMap = mapping.toSortedMap() - for ((k, v) in sortedEnumMap) { - p.println("$k($v)") - } - } -} - -private class TextTable(vararg columnNames: String) { - private val rows = ArrayList>() - private val columnsCount = columnNames.size - private val maxLengths = IntArray(columnsCount) { columnNames[it].length } - - init { - rows.add(columnNames.toList()) - } - - fun addRow(vararg row: String) { - check(row.size == columnsCount) { "Row size ${row.size} differs from columns count $columnsCount" } - rows.add(row.toList()) - - for ((i, col) in row.withIndex()) { - maxLengths[i] = max(maxLengths[i], col.length) - } - } - - fun printTo(p: Printer) { - for (row in rows) { - val rowStr = row.withIndex().joinToString("|") { (i, col) -> col.padEnd(maxLengths[i], ' ') } - p.println(rowStr) - } - } -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt index 5632dab92e0..61c9d6ce964 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.gradle.report +import org.jetbrains.kotlin.build.report.FileReportSettings +import org.jetbrains.kotlin.build.report.HttpReportSettings import java.io.File import java.io.Serializable @@ -23,27 +25,6 @@ data class ReportingSettings( } } -data class FileReportSettings( - val buildReportDir: File, - val includeMetricsInReport: Boolean = false, -) : Serializable { - companion object { - const val serialVersionUID: Long = 0 - } -} - -data class HttpReportSettings( - val url: String, - val password: String?, - val user: String?, - val verboseEnvironment: Boolean, - val includeGitBranchName: Boolean -) : Serializable { - companion object { - const val serialVersionUID: Long = 0 - } -} - data class BuildScanSettings( val customValueLimit: Int, val metrics: Set? diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt index eeed4ccef49..208df137705 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.report import org.jetbrains.kotlin.build.report.metrics.BuildMetrics import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +import org.jetbrains.kotlin.build.report.statistic.StatTag import org.jetbrains.kotlin.incremental.ChangedFiles internal class TaskExecutionResult( @@ -19,6 +20,5 @@ internal class TaskExecutionInfo( val kotlinLanguageVersion: KotlinVersion? = null, val changedFiles: ChangedFiles? = null, val compilerArguments: Array = emptyArray(), - val withArtifactTransform: Boolean? = false, - val withAbiSnapshot: Boolean? = false + val tags: Set = emptySet(), ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt index b01fe0ecc7f..f17d823832d 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.gradle.report import org.gradle.api.Project +import org.jetbrains.kotlin.build.report.FileReportSettings +import org.jetbrains.kotlin.build.report.HttpReportSettings import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider @@ -14,7 +16,6 @@ import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLI import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly - private val availableMetrics = BuildTime.values().map { it.name } + BuildPerformanceMetric.values().map { it.name } internal fun reportingSettings(project: Project): ReportingSettings { diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt index 68a1a88750b..3ec2a18b7d8 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt @@ -6,7 +6,7 @@ package org.jetbrains.kotlin.gradle.report.data import org.jetbrains.kotlin.build.report.metrics.BuildMetrics -import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters +import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters class BuildExecutionData( val startParameters: GradleBuildStartParameters, diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt index da1ae9c4dd4..aca93f12076 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/reportDataUtil.kt @@ -10,20 +10,24 @@ import org.gradle.tooling.events.task.TaskFinishEvent import org.gradle.tooling.events.task.TaskSkippedResult import org.gradle.tooling.events.task.TaskSuccessResult import org.jetbrains.kotlin.build.report.metrics.* -import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults -import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData -import org.jetbrains.kotlin.gradle.plugin.stat.StatTag +import org.jetbrains.kotlin.build.report.statistic.CompileStatisticsData +import org.jetbrains.kotlin.build.report.statistic.StatTag import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.incremental.ChangedFiles -import org.jetbrains.kotlin.utils.addToStdlib.ifTrue -import java.lang.management.ManagementFactory -import java.util.ArrayList import java.util.concurrent.TimeUnit import org.jetbrains.kotlin.gradle.dsl.KotlinVersion -private fun availableForStat(taskPath: String): Boolean { - return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null) +internal fun getTaskResult(event: TaskFinishEvent) = when (val result = event.result) { + is TaskSuccessResult -> when { + result.isFromCache -> TaskExecutionState.FROM_CACHE + result.isUpToDate -> TaskExecutionState.UP_TO_DATE + else -> TaskExecutionState.SUCCESS + } + + is TaskSkippedResult -> TaskExecutionState.SKIPPED + is TaskFailureResult -> TaskExecutionState.FAILED + else -> TaskExecutionState.UNKNOWN } internal fun prepareData( @@ -32,95 +36,112 @@ internal fun prepareData( uuid: String, label: String?, kotlinVersion: String, - buildOperationRecords: Collection, - additionalTags: List = emptyList(), + buildOperationRecord: BuildOperationRecord, + onlyKotlinTask: Boolean = true, + additionalTags: Set = emptySet(), metricsToShow: Set? = null ): CompileStatisticsData? { val result = event.result val taskPath = event.descriptor.taskPath - val durationMs = result.endTime - result.startTime - val taskResult = when (result) { - is TaskSuccessResult -> when { - result.isFromCache -> TaskExecutionState.FROM_CACHE - result.isUpToDate -> TaskExecutionState.UP_TO_DATE - else -> TaskExecutionState.SUCCESS - } + return prepareData(getTaskResult(event), taskPath, result.startTime, result.endTime - result.startTime, projectName, uuid, + label, kotlinVersion, buildOperationRecord, onlyKotlinTask, additionalTags, metricsToShow) +} - is TaskSkippedResult -> TaskExecutionState.SKIPPED - is TaskFailureResult -> TaskExecutionState.FAILED - else -> TaskExecutionState.UNKNOWN - } - - if (!availableForStat(taskPath)) { +internal fun prepareData( + taskResult: TaskExecutionState?, + taskPath: String, + startTime: Long, + finishTime: Long, + projectName: String, + uuid: String, + label: String?, + kotlinVersion: String, + buildOperationRecord: BuildOperationRecord, + onlyKotlinTask: Boolean = true, + additionalTags: Set = emptySet(), + metricsToShow: Set? = null +): CompileStatisticsData? { + if (onlyKotlinTask && !(buildOperationRecord is TaskRecord && buildOperationRecord.isFromKotlinPlugin)) { return null } - val taskExecutionResult = TaskExecutionResults[taskPath] - val buildMetrics = buildOperationRecords.firstOrNull { it.path == taskPath }?.buildMetrics + val buildMetrics = buildOperationRecord.buildMetrics - val performanceMetrics = collectBuildPerformanceMetrics(taskExecutionResult, buildMetrics) + val performanceMetrics = collectBuildPerformanceMetrics(buildMetrics) val buildTimesMetrics = collectBuildMetrics( - taskExecutionResult, buildMetrics, performanceMetrics, result.startTime, - System.currentTimeMillis() + buildMetrics, startTime, System.currentTimeMillis() ) - val buildAttributes = collectBuildAttributes(taskExecutionResult, buildMetrics) - val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) { - is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } - else -> emptyList() + val buildAttributes = collectBuildAttributes(buildMetrics) + val changes = if (buildOperationRecord is TaskRecord && buildOperationRecord.changedFiles is ChangedFiles.Known) { + buildOperationRecord.changedFiles.modified.map { it.absolutePath } + buildOperationRecord.changedFiles.removed.map { it.absolutePath } + } else { + emptyList() } + val kotlinLanguageVersion = if (buildOperationRecord is TaskRecord) buildOperationRecord.kotlinLanguageVersion else null + return CompileStatisticsData( - durationMs = durationMs, - taskResult = taskResult.name, + durationMs = buildOperationRecord.totalTimeMs, + taskResult = taskResult?.name, label = label, buildTimesMetrics = filterMetrics(metricsToShow, buildTimesMetrics), performanceMetrics = filterMetrics(metricsToShow, performanceMetrics), projectName = projectName, taskName = taskPath, changes = changes, - tags = collectTags(taskExecutionResult, buildMetrics, additionalTags), + tags = collectTags(buildOperationRecord, additionalTags), nonIncrementalAttributes = buildAttributes, hostName = BuildReportsService.hostName, kotlinVersion = kotlinVersion, - kotlinLanguageVersion = taskExecutionResult?.taskInfo?.kotlinLanguageVersion, + kotlinLanguageVersion = kotlinLanguageVersion?.version, buildUuid = uuid, - finishTime = System.currentTimeMillis(), - compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList(), - gcCountMetrics = buildMetrics?.gcMetrics?.asGcCountMap(), - gcTimeMetrics = buildMetrics?.gcMetrics?.asGcTimeMap() + compilerArguments = collectCompilerArguments(buildOperationRecord), + gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(), + gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(), + finishTime = finishTime, + startTimeMs = startTime, + fromKotlinPlugin = buildOperationRecord.isFromKotlinPlugin, + skipMessage = buildOperationRecord.skipMessage, + icLogLines = buildOperationRecord.icLogLines ) } +fun collectCompilerArguments(buildOperationRecord: BuildOperationRecord?): List { + return if (buildOperationRecord is TaskRecord) { + buildOperationRecord.compilerArguments.asList() + } else emptyList() +} + private fun > filterMetrics( expectedMetrics: Set?, buildTimesMetrics: Map ): Map = expectedMetrics?.let { buildTimesMetrics.filterKeys { metric -> it.contains(metric.name) } } ?: buildTimesMetrics -private fun collectBuildAttributes(taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics?): Set { - val attributes = HashSet() - buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys?.also { attributes.addAll(it) } - taskExecutionResult?.buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys?.also { attributes.addAll(it) } - return attributes +private fun collectBuildAttributes(buildMetrics: BuildMetrics?): Set { + return buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys ?: emptySet() } private fun collectBuildPerformanceMetrics( - taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics? ): Map { - val taskBuildPerformanceMetrics = HashMap() - taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.let { taskBuildPerformanceMetrics.putAll(it) } - buildMetrics?.buildPerformanceMetrics?.asMap()?.let { taskBuildPerformanceMetrics.putAll(it) } - return taskBuildPerformanceMetrics.filterValues { value -> value != 0L } + return buildMetrics?.buildPerformanceMetrics?.asMap() + ?.filterValues { value -> value != 0L } + ?.filterKeys { key -> + key !in listOf( + BuildPerformanceMetric.START_WORKER_EXECUTION, + BuildPerformanceMetric.CALL_WORKER, + BuildPerformanceMetric.CALL_KOTLIN_DAEMON, + BuildPerformanceMetric.START_KOTLIN_DAEMON_EXECUTION + ) + } + ?: emptyMap() } private fun collectBuildMetrics( - taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics?, - performanceMetrics: Map, gradleTaskStartTime: Long? = null, taskFinishEventTime: Long? = null, ): Map { - val taskBuildMetrics = HashMap() - taskExecutionResult?.buildMetrics?.buildTimes?.asMapMs()?.let { taskBuildMetrics.putAll(it) } - buildMetrics?.buildTimes?.asMapMs()?.let { taskBuildMetrics.putAll(it) } + val taskBuildMetrics = HashMap(buildMetrics?.buildTimes?.asMapMs()) + val performanceMetrics = buildMetrics?.buildPerformanceMetrics?.asMap() ?: emptyMap() gradleTaskStartTime?.let { startTime -> performanceMetrics[BuildPerformanceMetric.START_TASK_ACTION_EXECUTION]?.let { actionStartTime -> taskBuildMetrics.put(BuildTime.GRADLE_TASK_PREPARATION, actionStartTime - startTime) @@ -141,41 +162,34 @@ private fun collectBuildMetrics( } private fun collectTags( - taskExecutionResult: TaskExecutionResult?, - buildMetrics: BuildMetrics?, - additionalTags: List -): List { - val tags = collectTags(taskExecutionResult, additionalTags) - val nonIncrementalAttributes = collectBuildAttributes(taskExecutionResult, buildMetrics) + buildOperation: BuildOperationRecord?, + additionalTags: Set +): Set { + val tags = HashSet(additionalTags) + if (buildOperation is TaskRecord) { + tags.addAll(collectTaskRecordTags(buildOperation)) + } + + val nonIncrementalAttributes = collectBuildAttributes(buildOperation?.buildMetrics) if (nonIncrementalAttributes.isEmpty()) { tags.add(StatTag.INCREMENTAL) } else { tags.add(StatTag.NON_INCREMENTAL) } + return tags } -private fun collectTags( - taskExecutionResult: TaskExecutionResult?, - additionalTags: List, -): MutableList { - val tags = ArrayList(additionalTags) - val taskInfo = taskExecutionResult?.taskInfo +private fun collectTaskRecordTags( + taskRecord: TaskRecord?, +): Set { + val tags = HashSet() - taskInfo?.withAbiSnapshot?.ifTrue { - tags.add(StatTag.ABI_SNAPSHOT) - } - taskInfo?.withArtifactTransform?.ifTrue { - tags.add(StatTag.ARTIFACT_TRANSFORM) - } - taskInfo?.kotlinLanguageVersion?.also { + taskRecord?.kotlinLanguageVersion?.also { tags.add(getLanguageVersionTag(it)) } - val debugConfiguration = "-agentlib:" - if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) { - tags.add(StatTag.GRADLE_DEBUG) - } + taskRecord?.statTags?.let { tags.addAll(it) } return tags } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt index 5afd0e8af94..ab36be838c9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt @@ -36,7 +36,6 @@ import org.jetbrains.kotlin.gradle.internal.UsesClassLoadersCachingBuildService import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger import org.jetbrains.kotlin.gradle.logging.kotlinDebug -import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext.Companion.default import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_SUPPRESS_EXPERIMENTAL_IC_OPTIMIZATIONS_WARNING import org.jetbrains.kotlin.gradle.plugin.UsesBuildFinishedListenerService import org.jetbrains.kotlin.gradle.plugin.UsesVariantImplementationFactories @@ -57,7 +56,6 @@ abstract class AbstractKotlinCompile @Inject constr ) : AbstractKotlinCompileTool(objectFactory), CompileUsingKotlinDaemonWithNormalization, UsesBuildMetricsService, - UsesBuildReportsService, UsesIncrementalModuleInfoBuildService, UsesCompilerSystemPropertiesService, UsesVariantImplementationFactories, @@ -114,7 +112,7 @@ abstract class AbstractKotlinCompile @Inject constr @get:Internal internal abstract val suppressKotlinOptionsFreeArgsModificationWarning: Property - internal fun reportingSettings() = buildReportsService.orNull?.parameters?.reportingSettings?.orNull ?: ReportingSettings() + internal fun reportingSettings() = buildMetricsService.orNull?.parameters?.reportingSettings?.orNull ?: ReportingSettings() @get:Internal protected val multiModuleICSettings: MultiModuleICSettings diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt index ea8d6ac6f4d..91845275e75 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt @@ -25,7 +25,6 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.associateWithClosure import org.jetbrains.kotlin.gradle.plugin.mpp.internal import org.jetbrains.kotlin.gradle.plugin.sources.applyLanguageSettingsToCompilerOptions import org.jetbrains.kotlin.gradle.report.BuildMetricsService -import org.jetbrains.kotlin.gradle.report.BuildReportsService import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile import org.jetbrains.kotlin.gradle.tasks.KOTLIN_BUILD_DIR_NAME import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile @@ -64,7 +63,6 @@ internal abstract class AbstractKotlinCompileConfig task.buildMetricsService.value(metricsService).disallowChanges() - buildReportsService?.also { reportsService -> - task.buildReportsService.value(reportsService).disallowChanges() - } } task.systemPropertiesService.value(compilerSystemPropertiesService).disallowChanges() task.incrementalModuleInfoProvider.value(incrementalModuleInfoProvider).disallowChanges() diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt index 8cc0462ca15..ba4d9c7039e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt @@ -115,8 +115,9 @@ internal fun getAllDependencies(dependency: ResolvedDependencyResult): Set= 1 -> "${(sizeInBytes.toDouble() / gbSize).asString(1)} GB" - sizeInBytes / mbSize >= 1 -> "${(sizeInBytes.toDouble() / mbSize).asString(1)} MB" - sizeInBytes / kbSize >= 1 -> "${(sizeInBytes.toDouble() / kbSize).asString(1)} KB" - else -> "$sizeInBytes B" -} - -internal fun Double.asString(decPoints: Int): String = "%,.${decPoints}f".format(this) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/resourceUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/resourceUtils.kt index 21ae96117d9..4b92c84723b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/resourceUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/resourceUtils.kt @@ -42,7 +42,7 @@ fun Project.probeRemoteFileLength(url: String, probingTimeoutMs: Int = 0): Long? else { logger.kotlinDebug(::probeRemoteFileLength.name + "($url, $probingTimeoutMs): Failed to obtain content-length during the probing timeout.") @Suppress("UNCHECKED_CAST") - null as Long? + null } } finally { connection.disconnect() diff --git a/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/unitTests/ReportDataTest.kt b/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/unitTests/ReportDataTest.kt index 89bd86be495..9b2a57b4893 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/unitTests/ReportDataTest.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/unitTests/ReportDataTest.kt @@ -11,10 +11,7 @@ import org.gradle.tooling.events.task.TaskFinishEvent import org.gradle.tooling.events.task.TaskOperationDescriptor import org.gradle.tooling.events.task.TaskOperationResult import org.jetbrains.kotlin.build.report.metrics.* -import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults -import org.jetbrains.kotlin.gradle.plugin.stat.StatTag -import org.jetbrains.kotlin.gradle.report.TaskExecutionInfo -import org.jetbrains.kotlin.gradle.report.TaskExecutionResult +import org.jetbrains.kotlin.build.report.statistic.StatTag import org.jetbrains.kotlin.gradle.report.TaskRecord import org.jetbrains.kotlin.gradle.report.prepareData import org.junit.Ignore @@ -32,11 +29,6 @@ class ReportDataTest { @Test @Suppress("DEPRECATION") fun testTags() { - TaskExecutionResults[kotlinTaskPath] = TaskExecutionResult( - buildMetrics = BuildMetrics(buildAttributes = BuildAttributes()), - taskInfo = TaskExecutionInfo(kotlinLanguageVersion = KotlinVersion.KOTLIN_1_4), - icLogLines = emptyList(), - ) val buildOperationRecord = taskRecord(BuildMetrics(buildAttributes = BuildAttributes().also { it.add(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) })) val statisticData = prepareData( @@ -45,8 +37,9 @@ class ReportDataTest { uuid = "uuid", label = "label", kotlinVersion = "version", - buildOperationRecords = listOf(buildOperationRecord), - additionalTags = listOf(StatTag.KOTLIN_DEBUG) + onlyKotlinTask = true, + buildOperationRecord = buildOperationRecord, + additionalTags = setOf(StatTag.KOTLIN_DEBUG) ) assertNotNull(statisticData) @@ -57,41 +50,34 @@ class ReportDataTest { private fun taskRecord(buildMetrics: BuildMetrics) = TaskRecord( path = kotlinTaskPath, - classFqName = "class name", + classFqName = "org.jetbrains.kotlin.TestTask", startTimeMs = 10, totalTimeMs = 20, buildMetrics = buildMetrics, didWork = true, skipMessage = null, icLogLines = emptyList(), - kotlinLanguageVersion = KotlinVersion.KOTLIN_1_8 + kotlinLanguageVersion = KotlinVersion.KOTLIN_1_8, + changedFiles = null, + compilerArguments = emptyArray(), + statTags = emptySet() ) @Test fun testMetricFilter() { - TaskExecutionResults["testKotlin"] = TaskExecutionResult( - buildMetrics = BuildMetrics( - buildPerformanceMetrics = BuildPerformanceMetrics().also { - it.add(BuildPerformanceMetric.BUNDLE_SIZE) - it.add(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE) - }, - buildTimes = BuildTimes().also { - it.addTimeMs(BuildTime.RESTORE_OUTPUT_FROM_BACKUP, 10) - it.addTimeMs(BuildTime.IC_ANALYZE_JAR_FILES, 10) - } - ), - taskInfo = TaskExecutionInfo(), - icLogLines = emptyList() - ) val buildOperationRecord = taskRecord( BuildMetrics( buildPerformanceMetrics = BuildPerformanceMetrics().also { it.add(BuildPerformanceMetric.COMPILE_ITERATION) it.add(BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT) + it.add(BuildPerformanceMetric.BUNDLE_SIZE) + it.add(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE) }, buildTimes = BuildTimes().also { it.addTimeMs(BuildTime.STORE_BUILD_INFO, 20) it.addTimeMs(BuildTime.GRADLE_TASK_ACTION, 100) + it.addTimeMs(BuildTime.RESTORE_OUTPUT_FROM_BACKUP, 10) + it.addTimeMs(BuildTime.IC_ANALYZE_JAR_FILES, 10) } ) ) @@ -102,8 +88,9 @@ class ReportDataTest { uuid = "uuid", label = "label", kotlinVersion = "version", - buildOperationRecords = listOf(buildOperationRecord), - additionalTags = listOf(StatTag.KOTLIN_DEBUG), + buildOperationRecord = buildOperationRecord, + onlyKotlinTask = true, + additionalTags = setOf(StatTag.KOTLIN_DEBUG), metricsToShow = setOf( BuildPerformanceMetric.BUNDLE_SIZE.name,// from TaskExecutionResult BuildTime.GRADLE_TASK_ACTION.name,// from buildOperationRecord @@ -132,19 +119,11 @@ class ReportDataTest { val startWorker = 110L val finishGradleTask = System.nanoTime() - TaskExecutionResults["testKotlin"] = TaskExecutionResult( - buildMetrics = BuildMetrics( - buildPerformanceMetrics = BuildPerformanceMetrics().also { - it.add(BuildPerformanceMetric.FINISH_KOTLIN_DAEMON_EXECUTION, System.currentTimeMillis()) - it.add(BuildPerformanceMetric.START_WORKER_EXECUTION, TimeUnit.MILLISECONDS.toNanos(startWorker)) - } - ), - taskInfo = TaskExecutionInfo(), - icLogLines = emptyList() - ) val buildOperationRecord = taskRecord( BuildMetrics( buildPerformanceMetrics = BuildPerformanceMetrics().also { + it.add(BuildPerformanceMetric.FINISH_KOTLIN_DAEMON_EXECUTION, System.currentTimeMillis()) + it.add(BuildPerformanceMetric.START_WORKER_EXECUTION, TimeUnit.MILLISECONDS.toNanos(startWorker)) it.add(BuildPerformanceMetric.START_TASK_ACTION_EXECUTION, startTaskAction) it.add(BuildPerformanceMetric.CALL_WORKER, TimeUnit.MILLISECONDS.toNanos(callWorker)) } @@ -157,8 +136,9 @@ class ReportDataTest { uuid = "uuid", label = "label", kotlinVersion = "version", - buildOperationRecords = listOf(buildOperationRecord), - additionalTags = listOf(StatTag.KOTLIN_DEBUG), + buildOperationRecord = buildOperationRecord, + onlyKotlinTask = true, + additionalTags = setOf(StatTag.KOTLIN_DEBUG), ) assertNotNull(statisticData) assertEquals(startTaskAction - startGradleTask, statisticData.buildTimesMetrics[BuildTime.GRADLE_TASK_PREPARATION]) diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/CliLoggerAdapter.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/CliLoggerAdapter.kt index 9035d536e84..b05b0e9961e 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/CliLoggerAdapter.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/CliLoggerAdapter.kt @@ -20,13 +20,15 @@ internal class CliLoggerAdapter( override fun warning(message: String) = printlnIndented("Warning: $message", *CommonizerLogLevel.values()) - override fun error(message: String) = fatal(message) + override fun error(message: String, throwable: Throwable?) = fatal(message) override fun fatal(message: String): Nothing { printlnIndented("Error: $message\n", *CommonizerLogLevel.values()) exitProcess(1) } + override fun lifecycle(message: String) = log(message) + private fun printlnIndented(text: String, vararg levels: CommonizerLogLevel) { if (level in levels) { if (indent.isEmpty()) println(text) diff --git a/prepare/compiler/build.gradle.kts b/prepare/compiler/build.gradle.kts index 55602c6a25c..fe52c000cf4 100644 --- a/prepare/compiler/build.gradle.kts +++ b/prepare/compiler/build.gradle.kts @@ -214,6 +214,7 @@ dependencies { fatJarContents(commonDependency("org.lz4:lz4-java")) { isTransitive = false } fatJarContents(commonDependency("org.jetbrains.intellij.deps:asm-all")) { isTransitive = false } fatJarContents(commonDependency("com.google.guava:guava")) { isTransitive = false } + fatJarContents(commonDependency("com.google.code.gson:gson")) { isTransitive = false} fatJarContentsStripServices(commonDependency("com.fasterxml:aalto-xml")) { isTransitive = false } fatJarContents(commonDependency("org.codehaus.woodstox:stax2-api")) { isTransitive = false } diff --git a/settings.gradle b/settings.gradle index 01b1fb38de8..79b097bd9b7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -205,6 +205,7 @@ include ":kotlin-imports-dumper-compiler-plugin", ":kotlin-gradle-plugin-test-utils-embeddable", ":kotlin-gradle-plugin-integration-tests", ":kotlin-gradle-plugins-bom", + ":kotlin-build-statistic", ":gradle:android-test-fixes", ":gradle:gradle-warnings-detector", ":gradle:kotlin-compiler-args-properties", @@ -807,6 +808,7 @@ project(':kotlin-scripting-ide-services-unshaded').projectDir = "$rootDir/plugin project(':kotlin-scripting-ide-services-test').projectDir = "$rootDir/plugins/scripting/scripting-ide-services-test" as File project(':kotlin-scripting-ide-services').projectDir = "$rootDir/plugins/scripting/scripting-ide-services-embeddable" as File project(':kotlin-scripting-ide-common').projectDir = "$rootDir/plugins/scripting/scripting-ide-common" as File +project(':kotlin-build-statistic').projectDir ="$rootDir/kotlin-build-statistic" as File // Uncomment to use locally built protobuf-relocated // includeBuild("dependencies/protobuf")