diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index 5db3c6992e0..cf30c5bdcc4 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -9,16 +9,16 @@ import java.io.Serializable @Suppress("Reformat") -enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String) : Serializable { - CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory"), - LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size"), - SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size"), +enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String, val type: BuildMetricType) : Serializable { + CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory", type = BuildMetricType.FILE_SIZE), + LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size", type = BuildMetricType.FILE_SIZE), + SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size", type = BuildMetricType.FILE_SIZE), - COMPILE_ITERATION(parent = null, "Total compiler iteration"), + COMPILE_ITERATION(parent = null, "Total compiler iteration", type = BuildMetricType.NUMBER), // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature - ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)"), - SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot"), + ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)", type = BuildMetricType.FILE_SIZE), + SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot", type = BuildMetricType.FILE_SIZE), ; companion object { @@ -28,4 +28,10 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va values().filter { it.parent != null }.groupBy { it.parent } } } +} + +enum class BuildMetricType { + TIME_IN_MS, + FILE_SIZE, + NUMBER } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index f39f32493fe..07941db2771 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -9,25 +9,25 @@ import java.io.Serializable @Suppress("Reformat") enum class BuildTime(val parent: BuildTime? = null, val readableString: String) : Serializable { - GRADLE_TASK_ACTION(readableString = "Task execution"), - GRADLE_TASK(readableString = "Total time"), - CLEAR_OUTPUT(GRADLE_TASK, "Clear output"), - BACKUP_OUTPUT(GRADLE_TASK, "Backup output"), - RESTORE_OUTPUT_FROM_BACKUP(GRADLE_TASK, "Restore output"), - CONNECT_TO_DAEMON(GRADLE_TASK, "Connect to Kotlin daemon"), - CLEAR_JAR_CACHE(GRADLE_TASK, "Clear jar cache"), - CALCULATE_OUTPUT_SIZE(GRADLE_TASK, "Calculate output size"), - RUN_COMPILER(GRADLE_TASK, "Run compiler"), - NON_INCREMENTAL_COMPILATION_IN_PROCESS(RUN_COMPILER, "Inprocess compilation"), - NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILER, "Out of process compilation"), - NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILER, "Non incremental compilation"), - INCREMENTAL_COMPILATION(RUN_COMPILER, "Incremental compilation"), - STORE_BUILD_INFO(INCREMENTAL_COMPILATION, "Store build info"), - JAR_SNAPSHOT(INCREMENTAL_COMPILATION, "ABI JAR Snapshot support"), + GRADLE_TASK(readableString = "Total Gradle task time"), + GRADLE_TASK_ACTION(readableString = "Task action"), + CLEAR_OUTPUT(GRADLE_TASK_ACTION, "Clear output"), + BACKUP_OUTPUT(GRADLE_TASK_ACTION, "Backup output"), + RESTORE_OUTPUT_FROM_BACKUP(GRADLE_TASK_ACTION, "Restore output"), + CONNECT_TO_DAEMON(GRADLE_TASK_ACTION, "Connect to Kotlin daemon"), + CLEAR_JAR_CACHE(GRADLE_TASK_ACTION, "Clear jar cache"), + CALCULATE_OUTPUT_SIZE(GRADLE_TASK_ACTION, "Calculate output size"), + RUN_COMPILATION(GRADLE_TASK_ACTION, "Run compilation"), + NON_INCREMENTAL_COMPILATION_IN_PROCESS(RUN_COMPILATION, "Non incremental inprocess compilation"), + NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILATION, "Non incremental out of process compilation"), + NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILATION, "Non incremental compilation in daemon"), + INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILATION, "Incremental compilation in daemon"), + STORE_BUILD_INFO(INCREMENTAL_COMPILATION_DAEMON, "Store build info"), + JAR_SNAPSHOT(INCREMENTAL_COMPILATION_DAEMON, "ABI JAR Snapshot support"), SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT, "Set up ABI snapshot"), IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT, "Analyze jar files"), - IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"), - COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"), + IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION_DAEMON, "Calculate initial dirty sources set"), //TODO + COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute task classpath changes"), LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"), REMOVE_DUPLICATE_CLASSES(LOAD_CURRENT_CLASSPATH_SNAPSHOT, "Remove duplicate classes"), SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"), @@ -46,19 +46,18 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) IC_ANALYZE_CHANGES_IN_JAVA_SOURCES(IC_CALCULATE_INITIAL_DIRTY_SET, "Analyze Java file changes"), IC_ANALYZE_CHANGES_IN_ANDROID_LAYOUTS(IC_CALCULATE_INITIAL_DIRTY_SET, "Analyze Android layouts"), IC_DETECT_REMOVED_CLASSES(IC_CALCULATE_INITIAL_DIRTY_SET, "Detect removed classes"), - CLEAR_OUTPUT_ON_REBUILD(INCREMENTAL_COMPILATION, "Clear outputs on rebuild"), - IC_UPDATE_CACHES(INCREMENTAL_COMPILATION, "Update caches"), - INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"), - NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"), - IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"), - SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Shrink and save current classpath snapshot after compilation"), + CLEAR_OUTPUT_ON_REBUILD(INCREMENTAL_COMPILATION_DAEMON, "Clear outputs on rebuild"), + IC_UPDATE_CACHES(INCREMENTAL_COMPILATION_DAEMON, "Update caches"), + COMPILATION_ROUND(INCREMENTAL_COMPILATION_DAEMON, "Sources compilation round"), + IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION_DAEMON, "Write history file"), + SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION_DAEMON, "Shrink and save current classpath snapshot after compilation"), LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load shrunk previous classpath snapshot after compilation"), LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load current classpath snapshot after compilation"), SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot after compilation"), SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot after compilation"), COMPILER_PERFORMANCE(readableString = "Compiler time"), COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"), - CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"), + CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analysis"), CODE_GENERATION(COMPILER_PERFORMANCE, "Compiler code generation"), ; diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt index 35843c64195..531955eed36 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt @@ -39,12 +39,6 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM("kotlin.incremental.useClasspathSnapshot"), KOTLIN_COLORS_ENABLED_PROPERTY("kotlin.colors.enabled"), - KOTLIN_STAT_ENABLED_PROPERTY("kotlin.plugin.stat.enabled"), - KOTLIN_STAT_ENDPOINT_PROPERTY("kotlin.plugin.stat.endpoint"), - KOTLIN_STAT_USER_PROPERTY("kotlin.plugin.stat.user"), - KOTLIN_STAT_PASSWORD_PROPERTY("kotlin.plugin.stat.password"), - KOTLIN_STAT_LABEl_PROPERTY("kotlin.plugin.stat.label"), - OS_NAME("os.name", alwaysDirectAccess = true), TMP_DIR("java.io.tmpdir"), USER_HOME("user.home", alwaysDirectAccess = true), diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 405dc5e8bcd..2b959e86cfe 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -72,7 +72,7 @@ abstract class IncrementalCompilerRunner< // otherwise we track source files changes ourselves. providedChangedFiles: ChangedFiles?, projectDir: File? = null - ): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION) { + ): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION_DAEMON) { compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir) } @@ -321,16 +321,17 @@ abstract class IncrementalCompilerRunner< abiSnapshot: AbiSnapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot: Map = HashMap() ): ExitCode { + if (compilationMode is CompilationMode.Rebuild) { + reporter.report { "Non-incremental compilation will be performed: ${compilationMode.reason}" } + } + preBuildHook(args, compilationMode) - val buildTimeMode: BuildTime val dirtySources = when (compilationMode) { is CompilationMode.Incremental -> { - buildTimeMode = BuildTime.INCREMENTAL_ITERATION compilationMode.dirtyFiles.toMutableList() } is CompilationMode.Rebuild -> { - buildTimeMode = BuildTime.NON_INCREMENTAL_ITERATION reporter.addAttribute(compilationMode.reason) allKotlinSources.toMutableList() } @@ -368,7 +369,7 @@ abstract class IncrementalCompilerRunner< val bufferingMessageCollector = BufferingMessageCollector() val messageCollectorAdapter = MessageCollectorToOutputItemsCollectorAdapter(bufferingMessageCollector, outputItemsCollector) - exitCode = reporter.measure(buildTimeMode) { + exitCode = reporter.measure(BuildTime.COMPILATION_ROUND) { runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter) } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt index b80094cb950..da989e65afa 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt @@ -946,14 +946,15 @@ class KotlinGradleIT : BaseGradleIT() { //Should contains build metrics for all compile kotlin tasks assertTrue { report.contains("Time metrics:") } - assertTrue { report.contains("RUN_COMPILER:") } - assertTrue { report.contains("INCREMENTAL_COMPILATION:") } + assertTrue { report.contains("Run compilation:") } + assertTrue { report.contains("Incremental compilation in daemon:") } assertTrue { report.contains("Build performance metrics:") } - assertTrue { report.contains("OUTPUT_SIZE:") } - assertTrue { report.contains("SNAPSHOT_SIZE:") } + assertTrue { report.contains("Total size of the cache directory:") } + assertTrue { report.contains("Total compiler iteration:") } + assertTrue { report.contains("ABI snapshot size:") } + //for non-incremental builds assertTrue { report.contains("Build attributes:") } assertTrue { report.contains("REBUILD_REASON:") } - assertTrue { report.contains("COMPILE_ITERATION:") } } @Test @@ -967,9 +968,9 @@ class KotlinGradleIT : BaseGradleIT() { assertNotNull(reports) assertEquals(1, reports.size) val report = reports[0].readText() - assertTrue { report.contains("CODE_ANALYSIS:") } - assertTrue { report.contains("CODE_GENERATION:") } - assertTrue { report.contains("COMPILER_INITIALIZATION:") } + assertTrue { report.contains("Compiler code analysis:") } + assertTrue { report.contains("Compiler code generation:") } + assertTrue { report.contains("Compiler initialization time:") } } @Test diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 831b46345a2..04d2cd3cca7 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -7,18 +7,14 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.logging.Logger import org.jetbrains.kotlin.build.report.metrics.* -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.messages.MessageCollector -import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.daemon.common.* import org.jetbrains.kotlin.gradle.logging.* import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.report.* -import org.jetbrains.kotlin.gradle.report.TaskExecutionProperties.ABI_SNAPSHOT import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError @@ -131,17 +127,9 @@ internal class GradleKotlinCompilerWork @Inject constructor( throwGradleExceptionIfError(exitCode, executionStrategy) } finally { - val properties = ArrayList() - COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient()?.let { - if (it) properties.add(ABI_SNAPSHOT) - } - CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.value.toBooleanLenient()?.let { - if (it) properties.add(TaskExecutionProperties.ARTIFACT_TRANSFORM) - } - val taskInfo = TaskExecutionInfo( changedFiles = incrementalCompilationEnvironment?.changedFiles, - properties = properties + compilerArguments = compilerArgs ) val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines, taskInfo = taskInfo) TaskExecutionResults[taskPath] = result @@ -300,7 +288,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( log.info("Options for KOTLIN DAEMON: $compilationOptions") val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(log, bufferingMessageCollector) val compilationResults = GradleCompilationResults(log, projectRootFile) - return metrics.measure(BuildTime.RUN_COMPILER) { + return metrics.measure(BuildTime.RUN_COMPILATION) { daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults) }.also { metrics.addMetrics(compilationResults.buildMetrics) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt index 64a1b87ae68..0922f578e43 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt @@ -14,9 +14,7 @@ import org.gradle.api.services.BuildServiceParameters import org.jetbrains.kotlin.gradle.logging.kotlinDebug import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers -import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics -import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener -import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToBuildScan +import org.jetbrains.kotlin.gradle.plugin.statistics.BuildScanStatisticsListener import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.report.ReportingSettings import org.jetbrains.kotlin.gradle.report.reportingSettings @@ -49,33 +47,35 @@ internal abstract class KotlinGradleBuildServices : BuildService = project.gradle.sharedServices.registerIfAbsent( - "kotlin-build-service-${KotlinGradleBuildServices::class.java.canonicalName}_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}", - KotlinGradleBuildServices::class.java - ) { service -> - service.parameters.rootDir = project.rootProject.rootDir - service.parameters.buildDir = project.rootProject.buildDir + fun registerIfAbsent(project: Project, kotlinVersion: String): Provider = + project.gradle.sharedServices.registerIfAbsent( + "kotlin-build-service-${KotlinGradleBuildServices::class.java.canonicalName}_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}", + KotlinGradleBuildServices::class.java + ) { service -> + service.parameters.rootDir = project.rootProject.rootDir + service.parameters.buildDir = project.rootProject.buildDir - val reportingSettings = reportingSettings(project.rootProject) - addListeners(project, reportingSettings, kotlinVersion) - } + val reportingSettings = reportingSettings(project.rootProject) + addListeners(project, reportingSettings, kotlinVersion) + } fun addListeners(project: Project, reportingSettings: ReportingSettings, kotlinVersion: String) { - val listeners = project.rootProject.objects.listProperty(ReportStatistics::class.java) - .value(listOf()) - project.rootProject.extensions.findByName("buildScan") ?.also { if (reportingSettings.buildReportOutputs.contains(BuildReportType.BUILD_SCAN)) { - listeners.add(ReportStatisticsToBuildScan(it as BuildScanExtension)) + val listenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project) + listenerRegistryHolder.listenerRegistry.onTaskCompletion( + project.provider { + BuildScanStatisticsListener( + it as BuildScanExtension, + project.rootProject.name, + reportingSettings.buildReportLabel, + kotlinVersion + ) + } + ) } } - - if (listeners.get().isNotEmpty()) { - val listenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project) - val statListener = KotlinBuildStatListener(project.rootProject.name, reportingSettings.buildReportLabel, kotlinVersion, listeners.get()) - listenerRegistryHolder.listenerRegistry.onTaskCompletion(project.provider { statListener }) - } } private val multipleProjectsHolder = KotlinPluginInMultipleProjectsHolder( diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt similarity index 51% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt rename to libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt index 60765db1995..396c19b3c0e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt @@ -7,15 +7,24 @@ package org.jetbrains.kotlin.gradle.plugin.statistics import com.gradle.scan.plugin.BuildScanExtension import org.gradle.api.logging.Logging -import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData -import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics +import org.gradle.tooling.events.FinishEvent +import org.gradle.tooling.events.OperationCompletionListener +import org.gradle.tooling.events.task.TaskFinishEvent +import org.jetbrains.kotlin.build.report.metrics.BuildMetricType +import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData import org.jetbrains.kotlin.konan.file.File -import java.util.function.Consumer +import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult +import java.util.* +import kotlin.collections.ArrayList +import kotlin.collections.LinkedHashSet import kotlin.system.measureTimeMillis -class ReportStatisticsToBuildScan( - private val buildScan: BuildScanExtension -) : ReportStatistics { +class BuildScanStatisticsListener( + private val buildScan: BuildScanExtension, + val projectName: String, + val label: String?, + val kotlinVersion: String, +) : OperationCompletionListener, AutoCloseable { companion object { const val kbSize = 1024 const val mbSize = kbSize * kbSize @@ -25,8 +34,28 @@ class ReportStatisticsToBuildScan( private val tags = LinkedHashSet() private val log = Logging.getLogger(this.javaClass) + private val buildUuid: String = UUID.randomUUID().toString() - override fun report(data: CompileStatData) { + override fun onFinish(event: FinishEvent?) { + if (event is TaskFinishEvent) { + val (collectDataDuration, compileStatData) = measureTimeMillisWithResult { + KotlinBuildStatListener.prepareData(event, projectName, buildUuid, label, kotlinVersion) + } + log.debug("Collect data takes $collectDataDuration: $compileStatData") + + compileStatData?.also { + val reportDataDuration = measureTimeMillis { + report(it) + } + log.debug("Report data takes $reportDataDuration: $compileStatData") + } + } + } + + override fun close() { + } + + fun report(data: CompileStatisticsData) { val elapsedTime = measureTimeMillis { readableString(data).forEach { buildScan.value(data.taskName, it) } @@ -41,19 +70,23 @@ class ReportStatisticsToBuildScan( log.debug("Report statistic to build scan takes $elapsedTime ms") } - private fun readableString(data: CompileStatData): List { - val nonIncrementalReasons = data.nonIncrementalAttributes.filterValues { it > 0 }.keys + private fun readableString(data: CompileStatisticsData): List { val readableString = StringBuilder() - if (nonIncrementalReasons.isEmpty()) { + if (data.nonIncrementalAttributes.isEmpty()) { readableString.append("Incremental build; ") + data.changes.joinTo(readableString, prefix = "Changes: [", postfix = "]; ") { it.substringAfterLast(File.separator) } } else { - nonIncrementalReasons.joinTo(readableString, prefix = "Non incremental build because: [", postfix = "]; ") { it.readableString } + data.nonIncrementalAttributes.joinTo(readableString, prefix = "Non incremental build because: [", postfix = "]; ") { it.readableString } } - data.changes.joinTo(readableString, prefix = "Changes: [", postfix = "]; ") { it.substringAfterLast(File.separator) } val timeData = - data.buildTimesMs.map { (key, value) -> "${key.readableString}: ${value}ms" } //sometimes it is better to have separate variable to be able debug - val perfData = data.perfData.map { (key, value) -> "${key.readableString}: ${readableFileLength(value)}" } + data.buildTimesMetrics.map { (key, value) -> "${key.readableString}: ${value}ms" } //sometimes it is better to have separate variable to be able debug + val perfData = data.performanceMetrics.map { (key, value) -> + when (key.type) { + BuildMetricType.FILE_SIZE -> "${key.readableString}: ${readableFileLength(value)}" + else -> "${key.readableString}: $value}" + } + } timeData.union(perfData).joinTo(readableString, ",", "Performance: [", "]") return splitStringIfNeed(readableString.toString(), lengthLimit) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatistics.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt similarity index 54% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatistics.kt rename to libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt index 1b2627925e7..f2cf7c6f8ed 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatistics.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/CompileStatisticsData.kt @@ -9,9 +9,11 @@ import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.BuildTime import java.text.SimpleDateFormat +import java.util.* -data class CompileStatData( - val version: Int = 1, +//Sensitive data. This object is used directly for statistic via http +data class CompileStatisticsData( + val version: Int = 2, val projectName: String?, val label: String?, val taskName: String?, @@ -20,21 +22,28 @@ data class CompileStatData( val tags: List, val changes: List, val buildUuid: String = "Unset", - val kotlinVersion: String = "0.0.0", + val kotlinVersion: String, val hostName: String? = "Unset", - val timeInMillis: Long, - val timestamp: String = formatter.format(timeInMillis), - val nonIncrementalAttributes: Map, - val buildTimesMs: Map, - val perfData: Map + val finishTime: Long, + val timestamp: String = formatter.format(finishTime), + val compilerArguments: List, + val nonIncrementalAttributes: Set, + //TODO think about it,time in milliseconds + val buildTimesMetrics: Map, + val performanceMetrics: Map ) { companion object { - private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") + private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC")} } } - -interface ReportStatistics { - fun report(data: CompileStatData) +enum class StatTag { + ABI_SNAPSHOT, + ARTIFACT_TRANSFORM, + INCREMENTAL, + NON_INCREMENTAL, + GRADLE_DEBUG, + KOTLIN_DEBUG } + diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt index e71563610c4..ca0cb2b7e86 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt @@ -5,21 +5,20 @@ package org.jetbrains.kotlin.gradle.plugin.statistics -import org.gradle.api.logging.Logging -import org.gradle.tooling.events.FinishEvent -import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.task.TaskFailureResult import org.gradle.tooling.events.task.TaskFinishEvent import org.gradle.tooling.events.task.TaskSkippedResult import org.gradle.tooling.events.task.TaskSuccessResult +import org.jetbrains.kotlin.cli.common.CompilerSystemProperties +import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults -import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData -import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics +import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData +import org.jetbrains.kotlin.gradle.plugin.stat.StatTag +import org.jetbrains.kotlin.gradle.report.TaskExecutionResult import org.jetbrains.kotlin.incremental.ChangedFiles -import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult +import java.lang.management.ManagementFactory import java.net.InetAddress import java.util.* -import kotlin.system.measureTimeMillis enum class TaskExecutionState { SKIPPED, @@ -31,16 +30,7 @@ enum class TaskExecutionState { ; } -class KotlinBuildStatListener( - val projectName: String, - val label: String?, - val kotlinVersion: String, - val reportStatistics: List -) : OperationCompletionListener, AutoCloseable { - - private val log = Logging.getLogger(this.javaClass) - val buildUuid: String = UUID.randomUUID().toString() - +class KotlinBuildStatListener { companion object { val hostName: String? = try { InetAddress.getLocalHost().hostName @@ -59,7 +49,7 @@ class KotlinBuildStatListener( uuid: String, label: String?, kotlinVersion: String - ): CompileStatData? { + ): CompileStatisticsData? { val result = event.result val taskPath = event.descriptor.taskPath val durationMs = result.endTime - result.startTime @@ -89,33 +79,50 @@ class KotlinBuildStatListener( else -> emptyList() } - return CompileStatData( - durationMs = durationMs, taskResult = taskResult.name, label = label, - buildTimesMs = buildTimesMs, perfData = perfData, projectName = projectName, taskName = taskPath, changes = changes, - tags = taskExecutionResult?.taskInfo?.properties?.map { it.name } ?: emptyList(), - nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap(), - hostName = hostName, kotlinVersion = kotlinVersion, buildUuid = uuid, timeInMillis = System.currentTimeMillis() + return CompileStatisticsData( + durationMs = durationMs, + taskResult = taskResult.name, + label = label, + buildTimesMetrics = buildTimesMs, + performanceMetrics = perfData, + projectName = projectName, + taskName = taskPath, + changes = changes, + tags = parseTags(taskExecutionResult).map { it.name }, + nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys ?: emptySet(), + hostName = hostName, + kotlinVersion = kotlinVersion, + buildUuid = uuid, + finishTime = System.currentTimeMillis(), + compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList() ) } - } - - override fun onFinish(event: FinishEvent?) { - if (event is TaskFinishEvent) { - val (collectDataDuration, compileStatData) = measureTimeMillisWithResult { - prepareData(event, projectName, buildUuid, label, kotlinVersion) + private fun parseTags(taskExecutionResult: TaskExecutionResult?): List { + val tags = ArrayList() + CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient()?.let { + if (it) tags.add(StatTag.ABI_SNAPSHOT) } - log.debug("Collect data takes $collectDataDuration: $compileStatData") - - val reportDataDuration = measureTimeMillis { - compileStatData?.also { data -> reportStatistics.forEach { it.report(data) }} + CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.value.toBooleanLenient()?.let { + if (it) tags.add(StatTag.ARTIFACT_TRANSFORM) } - log.debug("Report data takes $reportDataDuration: $compileStatData") + val nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap() + + if (nonIncrementalAttributes.isEmpty()) { + tags.add(StatTag.INCREMENTAL) + } else { + tags.add(StatTag.NON_INCREMENTAL) + } + + val debugConfiguration = "-agentlib:" + if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) { + tags.add(StatTag.GRADLE_DEBUG) + } + return tags } - } - override fun close() { } } + diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt index ee60373d445..d408018d711 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/HttpReportService.kt @@ -14,7 +14,7 @@ import org.gradle.api.services.BuildServiceParameters import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.task.TaskFinishEvent -import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData +import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener.Companion.prepareData import java.net.HttpURLConnection import java.net.URL @@ -72,7 +72,7 @@ abstract class HttpReportService : BuildService, } - fun report(data: CompileStatData) { + fun report(data: CompileStatisticsData) { val elapsedTime = measureTimeMillis { val connection = URL(parameters.httpSettings.url).openConnection() as HttpURLConnection @@ -97,7 +97,7 @@ abstract class HttpReportService : BuildService, connection.disconnect() } } - log.debug("Report statistic to elastic search takes $elapsedTime ms") + log.debug("Report statistic by http takes $elapsedTime ms") } private fun checkResponseAndLog(connection: HttpURLConnection) { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt index e440539fa6a..710c8baa409 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt @@ -98,7 +98,7 @@ internal class PlainTextBuildReportWriter( val timeMs = buildTimesMs[buildTime] if (timeMs != null) { - p.println("${buildTime.name}: ${formatTime(timeMs)}") + p.println("${buildTime.readableString}: ${formatTime(timeMs)}") p.withIndent { BuildTime.children[buildTime]?.forEach { printBuildTime(it) } } @@ -123,7 +123,7 @@ internal class PlainTextBuildReportWriter( p.withIndent("Build performance metrics:") { for (metric in BuildPerformanceMetric.values()) { - allBuildMetrics[metric]?.let { p.println("${metric.name}: $it") } + allBuildMetrics[metric]?.let { p.println("${metric.readableString}: $it") } } } p.println() @@ -136,7 +136,7 @@ internal class PlainTextBuildReportWriter( p.withIndent("Build attributes:") { val attributesByKind = allAttributes.entries.groupBy { it.key.kind }.toSortedMap() for ((kind, attributesCounts) in attributesByKind) { - printMap(p, kind.name, attributesCounts.map { (k, v) -> k.name to v }.toMap()) + printMap(p, kind.name, attributesCounts.map { (k, v) -> k.readableString to v }.toMap()) } } p.println() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt index cad5e296301..0e6651d2b94 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt @@ -15,12 +15,6 @@ internal class TaskExecutionResult( ) internal class TaskExecutionInfo( - val properties: List = emptyList(), - val changedFiles: ChangedFiles? = null + val changedFiles: ChangedFiles? = null, + val compilerArguments: Array = emptyArray() ) - -internal enum class TaskExecutionProperties { - ABI_SNAPSHOT, - ARTIFACT_TRANSFORM - ; -}