From 5f1cf34c79d7863009b85c7e8215b349c82a846e Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Thu, 17 Feb 2022 11:22:00 +0000 Subject: [PATCH] KT-45777: Take coarse-grained snapshots of external libraries to reduce the size of the snapshots. - Track metrics for Gradle classpath snapshot artifact transform - Format size metrics so it's more readable ^KT-45777 In Progress --- .../report/metrics/BuildPerformanceMetric.kt | 27 +-- .../kotlin/build/report/metrics/BuildTime.kt | 20 ++- .../classpathDiff/ClasspathSnapshot.kt | 14 +- .../ClasspathSnapshotShrinker.kt | 19 +- .../classpathDiff/ClasspathSnapshotter.kt | 51 ++++-- .../build/metrics/GradleBuildMetricsData.kt | 5 +- .../jetbrains/kotlin/gradle/BuildReportsIT.kt | 2 +- .../ClasspathEntrySnapshotTransform.kt | 81 ++++++++- .../gradle/plugin/KotlinPluginWrapper.kt | 3 +- .../statistics/BuildScanStatisticsListener.kt | 18 +- .../report/BuildMetricsReporterService.kt | 164 +++++++++++------- .../report/KotlinBuildReporterHandler.kt | 14 +- .../kotlin/gradle/report/MetricsWriter.kt | 12 +- .../report/PlainTextBuildReportWriter.kt | 69 +++++--- .../kotlin/gradle/report/ReportingSettings.kt | 1 - .../gradle/report/data/BuildExecutionData.kt | 4 +- ...ecutionData.kt => BuildOperationRecord.kt} | 21 +-- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 7 +- 18 files changed, 353 insertions(+), 179 deletions(-) rename libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/{TaskExecutionData.kt => BuildOperationRecord.kt} (65%) diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index cf30c5bdcc4..94e079dea08 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -7,18 +7,22 @@ package org.jetbrains.kotlin.build.report.metrics import java.io.Serializable - @Suppress("Reformat") -enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String, val type: BuildMetricType) : Serializable { - CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory", type = BuildMetricType.FILE_SIZE), - LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size", type = BuildMetricType.FILE_SIZE), - SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size", type = BuildMetricType.FILE_SIZE), +enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String, val type: SizeMetricType) : Serializable { + CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory", type = SizeMetricType.BYTES), + LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size", type = SizeMetricType.BYTES), + SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size", type = SizeMetricType.BYTES), - COMPILE_ITERATION(parent = null, "Total compiler iteration", type = BuildMetricType.NUMBER), + COMPILE_ITERATION(parent = null, "Total compiler iteration", type = SizeMetricType.NUMBER), // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature - ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)", type = BuildMetricType.FILE_SIZE), - SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot", type = BuildMetricType.FILE_SIZE), + CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT(parent = null, "Number of times 'ClasspathEntrySnapshotTransform' ran", type = SizeMetricType.NUMBER), + JAR_CLASSPATH_ENTRY_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of classpath entry (if it's a jar)", type = SizeMetricType.BYTES), + CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of classpath entry (directory or jar)'s snapshot", type = SizeMetricType.BYTES), + SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT(parent = null, "Number of times classpath snapshot is shrunk and saved after compilation", type = SizeMetricType.NUMBER), + CLASSPATH_ENTRY_COUNT(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Number of classpath entries", type = SizeMetricType.NUMBER), + CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of classpath snapshot", type = SizeMetricType.BYTES), + SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of shrunk classpath snapshot", type = SizeMetricType.BYTES), ; companion object { @@ -30,8 +34,7 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va } } -enum class BuildMetricType { - TIME_IN_MS, - FILE_SIZE, +enum class SizeMetricType { + BYTES, NUMBER -} \ No newline at end of file +} diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index 07941db2771..38cdf918892 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -27,7 +27,7 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT, "Set up ABI snapshot"), IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT, "Analyze jar files"), IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION_DAEMON, "Calculate initial dirty sources set"), //TODO - COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute task classpath changes"), + COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"), LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"), REMOVE_DUPLICATE_CLASSES(LOAD_CURRENT_CLASSPATH_SNAPSHOT, "Remove duplicate classes"), SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"), @@ -51,14 +51,24 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) COMPILATION_ROUND(INCREMENTAL_COMPILATION_DAEMON, "Sources compilation round"), IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION_DAEMON, "Write history file"), SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION_DAEMON, "Shrink and save current classpath snapshot after compilation"), - LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load shrunk previous classpath snapshot after compilation"), - LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load current classpath snapshot after compilation"), - SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot after compilation"), - SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot after compilation"), + INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Load shrunk previous classpath snapshot"), + INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Load current classpath snapshot"), + INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Shrink current classpath snapshot"), + NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Non-incremental shrinking - Load current classpath snapshot"), + NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Non-incremental shrinking - Shrink current classpath snapshot"), + SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk current classpath snapshot"), COMPILER_PERFORMANCE(readableString = "Compiler time"), COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"), CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analysis"), CODE_GENERATION(COMPILER_PERFORMANCE, "Compiler code generation"), + CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM(parent = null, "Classpath entry snapshot transform"), + LOAD_CLASSES(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM, "Load classes"), + SNAPSHOT_CLASSES(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM, "Snapshot classes"), + READ_CLASSES_BASIC_INFO(parent = SNAPSHOT_CLASSES, "Read basic information about classes"), + FIND_INACCESSIBLE_CLASSES(parent = SNAPSHOT_CLASSES, "Find inaccessible classes"), + SNAPSHOT_KOTLIN_CLASSES(parent = SNAPSHOT_CLASSES, "Snapshot Kotlin classes"), + SNAPSHOT_JAVA_CLASSES(parent = SNAPSHOT_CLASSES, "Snapshot Java classes"), + SAVE_CLASSPATH_ENTRY_SNAPSHOT(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM, "Save classpath entry snapshot") ; companion object { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt index 59a05789422..c4818c4668b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.incremental.classpathDiff import org.jetbrains.kotlin.incremental.KotlinClassInfo +import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_LEVEL import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_MEMBER_LEVEL import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader.Kind.* import org.jetbrains.kotlin.name.ClassId @@ -141,7 +142,18 @@ class JavaElementSnapshotForTests( */ object InaccessibleClassSnapshot : ClassSnapshot() -/** The granularity of a [ClassSnapshot]. */ +/** + * The granularity of a [ClassSnapshot]. + * + * There are currently two granularity levels: + * - [CLASS_LEVEL]) (coarse-grained): The size of the snapshot will be smaller, but we will have coarse-grained classpath changes, which + * means more source files will be recompiled. + * - [CLASS_MEMBER_LEVEL] (fine-grained): The size of the snapshot will be larger, but we will have fine-grained classpath changes, which + * means fewer source files will be recompiled. + * + * Therefore, [CLASS_LEVEL] is typically suitable for classes that are infrequently changed (e.g., external libraries), whereas + * [CLASS_MEMBER_LEVEL] is suitable for classes that are frequently changed (e.g., classes produced by the current project). + */ enum class ClassSnapshotGranularity { /** diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt index acae8cae45b..70476c40b9e 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt @@ -207,12 +207,12 @@ internal fun shrinkAndSaveClasspathSnapshot( ShrinkMode.NoChanges } else { val shrunkPreviousClasspathAgainstPreviousLookups = - metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + metrics.measure(BuildTime.INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) { ListExternalizer(AccessibleClassSnapshotExternalizer) .loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile) } ShrinkMode.Incremental( - currentClasspathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + currentClasspathSnapshot = metrics.measure(BuildTime.INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) { CachedClasspathSnapshotSerializer .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) .removeDuplicateAndInaccessibleClasses() @@ -243,7 +243,7 @@ internal fun shrinkAndSaveClasspathSnapshot( // shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups } - is ShrinkMode.Incremental -> metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + is ShrinkMode.Incremental -> metrics.measure(BuildTime.INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { val shrunkClasses = shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups.mapTo(mutableSetOf()) { it.classId } val notYetShrunkClasses = shrinkMode.currentClasspathSnapshot.filter { it.classId !in shrunkClasses } val addedLookupSymbols = shrinkMode.addedLookupSymbols @@ -252,12 +252,12 @@ internal fun shrinkAndSaveClasspathSnapshot( shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups } is ShrinkMode.NonIncremental -> { - val classpathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + val classpathSnapshot = metrics.measure(BuildTime.NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) { CachedClasspathSnapshotSerializer .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) .removeDuplicateAndInaccessibleClasses() } - metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + metrics.measure(BuildTime.NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { shrinkClasspath(classpathSnapshot, lookupStorage) } } @@ -269,7 +269,7 @@ internal fun shrinkAndSaveClasspathSnapshot( "File '${classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.path}' does not exist" } } else { - metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT) { ListExternalizer(AccessibleClassSnapshotExternalizer).saveToFile( classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, shrunkCurrentClasspath!! @@ -277,8 +277,13 @@ internal fun shrinkAndSaveClasspathSnapshot( } } + metrics.addMetric(BuildPerformanceMetric.SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, 1) metrics.addMetric( - BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE, + BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT, + classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.size.toLong() + ) + metrics.addMetric( + BuildPerformanceMetric.CLASSPATH_SNAPSHOT_SIZE, classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() } ) metrics.addMetric( diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt index 202cfa97f9f..144f7d34f71 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt @@ -5,6 +5,10 @@ package org.jetbrains.kotlin.incremental.classpathDiff +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.incremental.ChangesCollector.Companion.getNonPrivateMemberNames import org.jetbrains.kotlin.incremental.KotlinClassInfo import org.jetbrains.kotlin.incremental.PackagePartProtoData @@ -26,14 +30,22 @@ object ClasspathEntrySnapshotter { && !unixStyleRelativePath.equals("module-info.class", ignoreCase = true) } - fun snapshot(classpathEntry: File): ClasspathEntrySnapshot { - val classes = DirectoryOrJarContentsReader - .read(classpathEntry, DEFAULT_CLASS_FILTER) - .map { (unixStyleRelativePath, contents) -> - ClassFileWithContents(ClassFile(classpathEntry, unixStyleRelativePath), contents) - } + fun snapshot( + classpathEntry: File, + granularity: ClassSnapshotGranularity, + metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter + ): ClasspathEntrySnapshot { + val classes = metrics.measure(BuildTime.LOAD_CLASSES) { + DirectoryOrJarContentsReader + .read(classpathEntry, DEFAULT_CLASS_FILTER) + .map { (unixStyleRelativePath, contents) -> + ClassFileWithContents(ClassFile(classpathEntry, unixStyleRelativePath), contents) + } + } - val snapshots = ClassSnapshotter.snapshot(classes) + val snapshots = metrics.measure(BuildTime.SNAPSHOT_CLASSES) { + ClassSnapshotter.snapshot(classes, granularity, metrics = metrics) + } val relativePathsToSnapshotsMap = classes.map { it.classFile.unixStyleRelativePath }.zip(snapshots).toMap(LinkedHashMap()) return ClasspathEntrySnapshot(relativePathsToSnapshotsMap) @@ -47,17 +59,24 @@ object ClassSnapshotter { fun snapshot( classes: List, granularity: ClassSnapshotGranularity = CLASS_MEMBER_LEVEL, - includeDebugInfoInJavaSnapshot: Boolean = false + includeDebugInfoInJavaSnapshot: Boolean = false, + metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter ): List { - // Find inaccessible classes first - val classesInfo: List = classes.map { it.classInfo } - val inaccessibleClassesInfo: Set = getInaccessibleClasses(classesInfo).toSet() - + val classesInfo: List = metrics.measure(BuildTime.READ_CLASSES_BASIC_INFO) { + classes.map { it.classInfo } + } + val inaccessibleClassesInfo: Set = metrics.measure(BuildTime.FIND_INACCESSIBLE_CLASSES) { + findInaccessibleClasses(classesInfo) + } return classes.map { when { it.classInfo in inaccessibleClassesInfo -> InaccessibleClassSnapshot - it.classInfo.isKotlinClass -> snapshotKotlinClass(it, granularity) - else -> JavaClassSnapshotter.snapshot(it, granularity, includeDebugInfoInJavaSnapshot) + it.classInfo.isKotlinClass -> metrics.measure(BuildTime.SNAPSHOT_KOTLIN_CLASSES) { + snapshotKotlinClass(it, granularity) + } + else -> metrics.measure(BuildTime.SNAPSHOT_JAVA_CLASSES) { + JavaClassSnapshotter.snapshot(it, granularity, includeDebugInfoInJavaSnapshot) + } } } } @@ -96,7 +115,7 @@ object ClassSnapshotter { * NOTE: If we do not have enough info to determine whether a class is inaccessible, we will assume that the class is accessible to be * safe. */ - private fun getInaccessibleClasses(classesInfo: List): List { + private fun findInaccessibleClasses(classesInfo: List): Set { fun BasicClassInfo.isInaccessible(): Boolean { return if (this.isKotlinClass) { when (this.kotlinClassHeader!!.kind) { @@ -130,7 +149,7 @@ object ClassSnapshotter { } } - return classesInfo.filter { it.isTransitivelyInaccessible() } + return classesInfo.filterTo(mutableSetOf()) { it.isTransitivelyInaccessible() } } } diff --git a/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt b/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt index 54be62bfa3a..4553927f7e2 100644 --- a/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt +++ b/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt @@ -10,14 +10,15 @@ import java.io.Serializable class GradleBuildMetricsData : Serializable { val parentMetric: MutableMap = LinkedHashMap() val buildAttributeKind: MutableMap = LinkedHashMap() - val taskData: MutableMap = LinkedHashMap() + val buildOperationData: MutableMap = LinkedHashMap() companion object { const val serialVersionUID = 0L } } -data class TaskData( +/** Data for a build operation (e.g., task or transform). */ +data class BuildOperationData( val path: String, val typeFqName: String, val buildTimesMs: Map, diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt index cd89d0965ae..4e37cdb9287 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt @@ -64,7 +64,7 @@ class BuildReportsIT : KGPBaseTest() { assertTrue { report.contains("Time metrics:") } assertTrue { report.contains("Run compilation:") } assertTrue { report.contains("Incremental compilation in daemon:") } - assertTrue { report.contains("Build performance metrics:") } + assertTrue { report.contains("Size metrics:") } assertTrue { report.contains("Total size of the cache directory:") } assertTrue { report.contains("Total compiler iteration:") } assertTrue { report.contains("ABI snapshot size:") } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt index af46261ef13..b7867b215fd 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt @@ -6,26 +6,97 @@ package org.jetbrains.kotlin.gradle.internal.transforms import org.gradle.api.artifacts.transform.* +import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.FileSystemLocation +import org.gradle.api.provider.Property import org.gradle.api.provider.Provider import org.gradle.api.tasks.Classpath +import org.gradle.api.tasks.Internal +import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.gradle.report.BuildMetricsReporterService +import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity +import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_LEVEL +import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_MEMBER_LEVEL import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotExternalizer import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter import org.jetbrains.kotlin.incremental.storage.saveToFile +import java.io.File /** Transform to create a snapshot of a classpath entry (directory or jar). */ @CacheableTransform -abstract class ClasspathEntrySnapshotTransform : TransformAction { +abstract class ClasspathEntrySnapshotTransform : TransformAction { + + abstract class Parameters : TransformParameters { + @get:Internal + abstract val gradleUserHomeDir: DirectoryProperty + + @get:Internal + abstract val buildMetricsReporterService: Property + } @get:Classpath @get:InputArtifact abstract val inputArtifact: Provider override fun transform(outputs: TransformOutputs) { - val classpathEntry = inputArtifact.get().asFile - val snapshotFile = outputs.file(classpathEntry.name.replace('.', '_') + "-snapshot.bin") + val classpathEntryInputDirOrJar = inputArtifact.get().asFile + val snapshotOutputFile = outputs.file(classpathEntryInputDirOrJar.name.replace('.', '_') + "-snapshot.bin") - val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntry) - ClasspathEntrySnapshotExternalizer.saveToFile(snapshotFile, snapshot) + val granularity = getClassSnapshotGranularity(classpathEntryInputDirOrJar, parameters.gradleUserHomeDir.get().asFile) + + val buildMetricsReporterService = parameters.buildMetricsReporterService.orNull + val metricsReporter = buildMetricsReporterService?.let { BuildMetricsReporterImpl() } ?: DoNothingBuildMetricsReporter + + val startTimeMs = System.currentTimeMillis() + var failureMessage: String? = null + try { + doTransform(classpathEntryInputDirOrJar, snapshotOutputFile, granularity, metricsReporter) + } catch (e: Throwable) { + failureMessage = e.message + throw e + } finally { + buildMetricsReporterService?.addTransformMetrics( + transformPath = "${ClasspathEntrySnapshotTransform::class.simpleName} for ${classpathEntryInputDirOrJar.path}", + transformClass = ClasspathEntrySnapshotTransform::class.java, + isKotlinTransform = true, + startTimeMs = startTimeMs, + totalTimeMs = System.currentTimeMillis() - startTimeMs, + buildMetrics = metricsReporter.getMetrics(), + failureMessage = failureMessage + ) + } + } + + /** + * Determines the [ClassSnapshotGranularity] when taking a snapshot of the given [classpathEntryDirOrJar]. + * + * As mentioned in [ClassSnapshotGranularity]'s kdoc, we will take [CLASS_LEVEL] snapshots for classes that are infrequently changed + * (e.g., external libraries which are typically stored/transformed inside the Gradle user home, or a few hard-coded cases), and take + * [CLASS_MEMBER_LEVEL] snapshots for the others. + */ + private fun getClassSnapshotGranularity(classpathEntryDirOrJar: File, gradleUserHomeDir: File): ClassSnapshotGranularity { + return if ( + classpathEntryDirOrJar.startsWith(gradleUserHomeDir) || + classpathEntryDirOrJar.name == "android.jar" + ) CLASS_LEVEL + else CLASS_MEMBER_LEVEL + } + + private fun doTransform( + classpathEntryInputDirOrJar: File, snapshotOutputFile: File, + granularity: ClassSnapshotGranularity, metrics: BuildMetricsReporter + ) { + metrics.measure(BuildTime.CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM) { + val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntryInputDirOrJar, granularity, metrics) + metrics.measure(BuildTime.SAVE_CLASSPATH_ENTRY_SNAPSHOT) { + ClasspathEntrySnapshotExternalizer.saveToFile(snapshotOutputFile, snapshot) + } + + metrics.addMetric(BuildPerformanceMetric.CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, 1) + if (classpathEntryInputDirOrJar.extension.equals("jar", ignoreCase = true)) { + metrics.addMetric(BuildPerformanceMetric.JAR_CLASSPATH_ENTRY_SIZE, classpathEntryInputDirOrJar.length()) + } + metrics.addMetric(BuildPerformanceMetric.CLASSPATH_ENTRY_SNAPSHOT_SIZE, snapshotOutputFile.length()) + } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt index c390eb394c0..97779937375 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt @@ -85,8 +85,7 @@ abstract class KotlinBasePluginWrapper : Plugin { KotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, kotlinPluginVersion) - val buildMetricReporter = - BuildMetricsReporterService.registerIfAbsent(project, BuildMetricsReporterService.getStartParameters(project)) + val buildMetricReporter = BuildMetricsReporterService.registerIfAbsent(project) buildMetricReporter?.also { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(it) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt index 396c19b3c0e..e59f42329fb 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt @@ -10,13 +10,12 @@ import org.gradle.api.logging.Logging import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.task.TaskFinishEvent -import org.jetbrains.kotlin.build.report.metrics.BuildMetricType +import org.jetbrains.kotlin.build.report.metrics.SizeMetricType import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData +import org.jetbrains.kotlin.gradle.report.formatSize import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult import java.util.* -import kotlin.collections.ArrayList -import kotlin.collections.LinkedHashSet import kotlin.system.measureTimeMillis class BuildScanStatisticsListener( @@ -26,9 +25,6 @@ class BuildScanStatisticsListener( val kotlinVersion: String, ) : OperationCompletionListener, AutoCloseable { companion object { - const val kbSize = 1024 - const val mbSize = kbSize * kbSize - const val gbSize = kbSize * mbSize const val lengthLimit = 100_000 } @@ -83,7 +79,7 @@ class BuildScanStatisticsListener( data.buildTimesMetrics.map { (key, value) -> "${key.readableString}: ${value}ms" } //sometimes it is better to have separate variable to be able debug val perfData = data.performanceMetrics.map { (key, value) -> when (key.type) { - BuildMetricType.FILE_SIZE -> "${key.readableString}: ${readableFileLength(value)}" + SizeMetricType.BYTES -> "${key.readableString}: ${formatSize(value)}" else -> "${key.readableString}: $value}" } } @@ -111,12 +107,4 @@ class BuildScanStatisticsListener( splattedString.add(tempStr) return splattedString } - - private fun readableFileLength(length: Long): String = - when { - length / gbSize > 0 -> "${length / gbSize} GB" - length / mbSize > 0 -> "${length / mbSize} MB" - length / kbSize > 0 -> "${length / kbSize} KB" - else -> "$length B" - } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt index a972e40c9a7..936ee60d703 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt @@ -19,11 +19,10 @@ import org.gradle.tooling.events.task.TaskSkippedResult import org.jetbrains.kotlin.build.report.metrics.BuildMetrics import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.BuildTime -import org.jetbrains.kotlin.gradle.plugin.KotlinGradleBuildServices import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor -import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData +import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentLinkedQueue @@ -38,42 +37,67 @@ abstract class BuildMetricsReporterService : BuildService() - private val taskRecords = ConcurrentHashMap() - private val failureMessages = ConcurrentLinkedQueue() - private val taskClass = ConcurrentHashMap() + // Tasks and transforms' records + private val buildOperationRecords = ConcurrentLinkedQueue() + private val failureMessages = ConcurrentLinkedQueue() - open fun add(taskPath: String, type: String, metrics: BuildMetricsReporter) { - if (buildMetricsMap.containsKey(taskPath)) { - log.warn("Duplicate path $taskPath") + // Info for tasks only + private val taskPathToMetricsReporter = ConcurrentHashMap() + private val taskPathToTaskClass = ConcurrentHashMap() + + open fun addTask(taskPath: String, taskClass: Class<*>, metricsReporter: BuildMetricsReporter) { + taskPathToMetricsReporter.put(taskPath, metricsReporter).also { + if (it != null) log.warn("Duplicate task path: $taskPath") // Should never happen but log it just in case } - buildMetricsMap[taskPath] = metrics - taskClass[taskPath] = type + taskPathToTaskClass.put(taskPath, taskClass.name).also { + if (it != null) log.warn("Duplicate task path: $taskPath") // Should never happen but log it just in case + } + } + + open fun addTransformMetrics( + transformPath: String, + transformClass: Class<*>, + isKotlinTransform: Boolean, + startTimeMs: Long, + totalTimeMs: Long, + buildMetrics: BuildMetrics, + failureMessage: String? + ) { + buildOperationRecords.add( + TransformRecord(transformPath, transformClass.name, isKotlinTransform, startTimeMs, totalTimeMs, buildMetrics) + ) + failureMessage?.let { failureMessages.add(it) } } override fun onFinish(event: FinishEvent?) { if (event is TaskFinishEvent) { val result = event.result val taskPath = event.descriptor.taskPath - val startMs = event.result.startTime - val finishMs = event.result.endTime + val totalTimeMs = result.endTime - result.startTime + + val buildMetrics = BuildMetrics() + buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, totalTimeMs) + taskPathToMetricsReporter[taskPath]?.let { + buildMetrics.addAll(it.getMetrics()) + } + val taskExecutionResult = TaskExecutionResults[taskPath] + taskExecutionResult?.buildMetrics?.also { buildMetrics.addAll(it) } + + buildOperationRecords.add( + TaskRecord( + path = taskPath, + classFqName = taskPathToTaskClass[taskPath] ?: "unknown", + startTimeMs = result.startTime, + totalTimeMs = totalTimeMs, + buildMetrics = buildMetrics, + didWork = result is TaskExecutionResult, + skipMessage = (result as? TaskSkippedResult)?.skipMessage, + icLogLines = taskExecutionResult?.icLogLines ?: emptyList() + ) + ) if (result is TaskFailureResult) { failureMessages.addAll(result.failures.map { it.message }) } - val skipMessage = when (result) { - is TaskSkippedResult -> result.skipMessage - else -> null - } - val didWork = result is TaskExecutionResult - val buildMetrics = BuildMetrics() - buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, finishMs - startMs) - - buildMetricsMap[taskPath]?.also { buildMetrics.addAll(it.getMetrics()) } - val taskExecutionResult = TaskExecutionResults[taskPath] - val icLogLines = taskExecutionResult?.icLogLines ?: emptyList() - taskExecutionResult?.buildMetrics?.also { buildMetrics.addAll(it) } - taskRecords[taskPath] = TaskRecord(taskPath, startMs, finishMs, skipMessage, didWork, icLogLines, buildMetrics, taskClass[taskPath] ?: "unknown") } } @@ -81,19 +105,16 @@ abstract class BuildMetricsReporterService : BuildService() - startParameters.add("tasks = ${it.taskRequests.joinToString { it.args.toString() }}") + startParameters.add("tasks = ${it.taskRequests.joinToString { task -> task.args.toString() }}") startParameters.add("excluded tasks = ${it.excludedTaskNames}") startParameters.add("current dir = ${it.currentDir}") startParameters.add("project properties args = ${it.projectProperties}") @@ -101,33 +122,34 @@ abstract class BuildMetricsReporterService : BuildService): Provider? { - val rootProject = project.gradle.rootProject - val reportingSettings = reportingSettings(rootProject) + fun registerIfAbsent(project: Project): Provider? { + val serviceClass = BuildMetricsReporterService::class.java + val serviceName = "${serviceClass.name}_${serviceClass.classLoader.hashCode()}" - val buildDataProcessors = ArrayList() - reportingSettings.fileReportSettings?.let { - buildDataProcessors.add( - PlainTextBuildReportWriterDataProcessor( - it, - rootProject.name - ) - ) + // Return early if the service was already registered to avoid the overhead of reading the reporting settings below + project.gradle.sharedServices.registrations.findByName(serviceName)?.let { + @Suppress("UNCHECKED_CAST") + return it.service as Provider } - if (reportingSettings.metricsOutputFile != null) { - buildDataProcessors.add(MetricsWriter(reportingSettings.metricsOutputFile.absoluteFile)) - } - - if (reportingSettings.buildReportOutputs.isEmpty() && buildDataProcessors.isEmpty()) { + val reportingSettings = reportingSettings(project.rootProject) + if (reportingSettings.buildReportOutputs.isEmpty() + && reportingSettings.fileReportSettings == null + && reportingSettings.metricsOutputFile == null + ) { return null } - return project.gradle.sharedServices.registerIfAbsent( - "build_metric_service_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}", - BuildMetricsReporterService::class.java - ) { - it.parameters.startParameters = startParameters + return project.gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) { + val buildDataProcessors = mutableListOf() + reportingSettings.fileReportSettings?.let { fileReportSettings -> + buildDataProcessors.add(PlainTextBuildReportWriterDataProcessor(fileReportSettings, project.rootProject.name)) + } + reportingSettings.metricsOutputFile?.let { metricsOutputFile -> + buildDataProcessors.add(MetricsWriter(metricsOutputFile.absoluteFile)) + } + + it.parameters.startParameters = getStartParameters(project) it.parameters.buildDataProcessors = buildDataProcessors it.parameters.reportingSettings = reportingSettings }!! @@ -138,19 +160,27 @@ abstract class BuildMetricsReporterService : BuildService, + override val path: String, + override val classFqName: String, + override val startTimeMs: Long, + override val totalTimeMs: Long, override val buildMetrics: BuildMetrics, - override val type: String -) : TaskExecutionData { - override val isKotlinTask by lazy { - type.startsWith("org.jetbrains.kotlin") - } + override val didWork: Boolean, + override val skipMessage: String?, + override val icLogLines: List +) : BuildOperationRecord { + override val isFromKotlinPlugin: Boolean = classFqName.startsWith("org.jetbrains.kotlin") +} - override val totalTimeMs: Long - get() = (endMs - startMs) -} \ No newline at end of file +private class TransformRecord( + override val path: String, + override val classFqName: String, + override val isFromKotlinPlugin: Boolean, + override val startTimeMs: Long, + override val totalTimeMs: Long, + override val buildMetrics: BuildMetrics +) : BuildOperationRecord { + override val didWork: Boolean = true + override val skipMessage: String? = null + override val icLogLines: List = emptyList() +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterHandler.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterHandler.kt index bdd9b460cae..d87f5690e12 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterHandler.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterHandler.kt @@ -106,5 +106,15 @@ internal fun formatTime(ms: Long): String { return seconds.asString(2) + " s" } -internal fun Double.asString(decPoints: Int): String = - String.format("%.${decPoints}f", this) +private const val kbSize = 1024 +private const val mbSize = kbSize * 1024 +private const val gbSize = mbSize * 1024 + +internal fun formatSize(sizeInBytes: Long): String = when { + sizeInBytes / gbSize >= 1 -> "${(sizeInBytes.toDouble() / gbSize).asString(1)} GB" + sizeInBytes / mbSize >= 1 -> "${(sizeInBytes.toDouble() / mbSize).asString(1)} MB" + sizeInBytes / kbSize >= 1 -> "${(sizeInBytes.toDouble() / kbSize).asString(1)} KB" + else -> "$sizeInBytes B" +} + +internal fun Double.asString(decPoints: Int): String = "%,.${decPoints}f".format(this) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt index 67114610830..04b83d60513 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt @@ -9,7 +9,7 @@ import org.gradle.api.logging.Logger import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData -import org.jetbrains.kotlin.gradle.internal.build.metrics.TaskData +import org.jetbrains.kotlin.gradle.internal.build.metrics.BuildOperationData import org.jetbrains.kotlin.gradle.logging.kotlinDebug import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor @@ -34,11 +34,11 @@ internal class MetricsWriter( buildMetricsData.buildAttributeKind[attr.name] = attr.kind.name } - for (data in build.taskExecutionData) { - buildMetricsData.taskData[data.taskPath] = - TaskData( - path = data.taskPath, - typeFqName = data.type, + for (data in build.buildOperationRecord) { + buildMetricsData.buildOperationData[data.path] = + BuildOperationData( + path = data.path, + typeFqName = data.classFqName, buildTimesMs = data.buildMetrics.buildTimes.asMapMs().mapKeys { it.key.name }, performanceMetrics = data.buildMetrics.buildPerformanceMetrics.asMap().mapKeys { it.key.name }, buildAttributes = data.buildMetrics.buildAttributes.asMap().mapKeys { it.key.name }, diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt index 710c8baa409..5c763efa659 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt @@ -9,7 +9,7 @@ import org.gradle.api.logging.Logger import org.jetbrains.kotlin.build.report.metrics.* import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor -import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData +import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.gradle.utils.Printer import java.io.File import java.io.Serializable @@ -59,8 +59,15 @@ internal class PlainTextBuildReportWriter( } private fun printBuildReport(build: BuildExecutionData) { + // NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms. + // Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound + // a bit unfamiliar. + // TODO: If it is confusing, consider renaming "tasks" to "build operations" in this class. printBuildInfo(build) - printMetrics(build.aggregatedMetrics) + if (printMetrics) { + printMetrics(build.aggregatedMetrics) + p.println() + } printTaskOverview(build) printTasksLog(build) } @@ -69,17 +76,15 @@ internal class PlainTextBuildReportWriter( p.withIndent("Gradle start parameters:") { build.startParameters.forEach { p.println(it) } } - p.println() + if (build.failureMessages.isNotEmpty()) { p.println("Build failed: ${build.failureMessages}") + p.println() } - p.println() } private fun printMetrics(buildMetrics: BuildMetrics) { - if (!printMetrics) return - printBuildTimes(buildMetrics.buildTimes) printBuildPerformanceMetrics(buildMetrics.buildPerformanceMetrics) printBuildAttributes(buildMetrics.buildAttributes) @@ -114,19 +119,38 @@ internal class PlainTextBuildReportWriter( printBuildTime(buildTime) } } - p.println() } private fun printBuildPerformanceMetrics(buildMetrics: BuildPerformanceMetrics) { val allBuildMetrics = buildMetrics.asMap() if (allBuildMetrics.isEmpty()) return - p.withIndent("Build performance metrics:") { + p.withIndent("Size metrics:") { for (metric in BuildPerformanceMetric.values()) { - allBuildMetrics[metric]?.let { p.println("${metric.readableString}: $it") } + allBuildMetrics[metric]?.let { printSizeMetric(metric, it) } } } - p.println() + } + + private fun printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) { + fun BuildPerformanceMetric.numberOfAncestors(): Int { + var count = 0 + var parent: BuildPerformanceMetric? = parent + while (parent != null) { + count++ + parent = parent.parent + } + return count + } + + val indentLevel = sizeMetric.numberOfAncestors() + + repeat(indentLevel) { p.pushIndent() } + when (sizeMetric.type) { + SizeMetricType.BYTES -> p.println("${sizeMetric.readableString}: ${formatSize(value)}") + SizeMetricType.NUMBER -> p.println("${sizeMetric.readableString}: $value") + } + repeat(indentLevel) { p.popIndent() } } private fun printBuildAttributes(buildAttributes: BuildAttributes) { @@ -139,19 +163,18 @@ internal class PlainTextBuildReportWriter( printMap(p, kind.name, attributesCounts.map { (k, v) -> k.readableString to v }.toMap()) } } - p.println() } private fun printTaskOverview(build: BuildExecutionData) { var allTasksTimeMs = 0L var kotlinTotalTimeMs = 0L - val kotlinTasks = ArrayList() + val kotlinTasks = ArrayList() - for (task in build.taskExecutionData) { + for (task in build.buildOperationRecord) { val taskTimeMs = task.totalTimeMs allTasksTimeMs += taskTimeMs - if (task.isKotlinTask) { + if (task.isFromKotlinPlugin) { kotlinTotalTimeMs += taskTimeMs kotlinTasks.add(task) } @@ -166,37 +189,39 @@ internal class PlainTextBuildReportWriter( p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)") val table = TextTable("Time", "% of Kotlin time", "Task") - for (task in kotlinTasks.sortedByDescending { it.totalTimeMs }) { + for (task in kotlinTasks.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) { val timeMs = task.totalTimeMs val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1) - table.addRow(formatTime(timeMs), "$percent %", task.taskPath) + table.addRow(formatTime(timeMs), "$percent %", task.path) } table.printTo(p) p.println() } private fun printTasksLog(build: BuildExecutionData) { - for (task in build.taskExecutionData) { + for (task in build.buildOperationRecord.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) { printTaskLog(task) p.println() } } - private fun printTaskLog(task: TaskExecutionData) { + private fun printTaskLog(task: BuildOperationRecord) { val skipMessage = task.skipMessage if (skipMessage != null) { - p.println("Task '${task.taskPath}' was skipped: $skipMessage") + p.println("Task '${task.path}' was skipped: $skipMessage") } else { - p.println("Task '${task.taskPath}' finished in ${formatTime(task.totalTimeMs)}") + p.println("Task '${task.path}' finished in ${formatTime(task.totalTimeMs)}") } if (task.icLogLines.isNotEmpty()) { - p.withIndent("Compilation log for task '${task.taskPath}':") { + p.withIndent("Compilation log for task '${task.path}':") { task.icLogLines.forEach { p.println(it) } } } - printMetrics(task.buildMetrics) + if (printMetrics) { + printMetrics(task.buildMetrics) + } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt index e2521ecf443..97fda8ef75a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt @@ -22,7 +22,6 @@ data class ReportingSettings( } data class FileReportSettings( - val metricsOutputFile: File? = null, val buildReportDir: File, val includeMetricsInReport: Boolean = false, ) : Serializable { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt index 8c0d08d4752..6687c961dd7 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt @@ -10,11 +10,11 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetrics class BuildExecutionData( val startParameters: Collection, val failureMessages: List, - val taskExecutionData: Collection + val buildOperationRecord: Collection ) { val aggregatedMetrics by lazy { BuildMetrics().also { acc -> - taskExecutionData.forEach { acc.addAll(it.buildMetrics) } + buildOperationRecord.forEach { acc.addAll(it.buildMetrics) } } } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/TaskExecutionData.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildOperationRecord.kt similarity index 65% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/TaskExecutionData.kt rename to libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildOperationRecord.kt index c98bdc40c31..d927aeb476c 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/TaskExecutionData.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildOperationRecord.kt @@ -5,20 +5,17 @@ package org.jetbrains.kotlin.gradle.report.data -import org.gradle.api.Task -import org.gradle.api.tasks.TaskState import org.jetbrains.kotlin.build.report.metrics.BuildMetrics -interface TaskExecutionData { - val taskPath: String - val startMs: Long - val endMs: Long +/** Data for a build operation (e.g., task or transform). */ +interface BuildOperationRecord { + val path: String + val classFqName: String + val isFromKotlinPlugin: Boolean + val startTimeMs: Long // Measured by System.currentTimeMillis() val totalTimeMs: Long - val skipMessage: String? - val didWork: Boolean - val icLogLines: List val buildMetrics: BuildMetrics - val isKotlinTask: Boolean - val type: String + val didWork: Boolean + val skipMessage: String? + val icLogLines: List } - diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index faf92b722cb..0bfdeddfad2 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -392,7 +392,7 @@ abstract class AbstractKotlinCompile : AbstractKotl executeImpl(inputChanges, outputsBackup) } - buildMetricsReporterService.orNull?.also { it.add(path, this::class.java.name, buildMetrics) } + buildMetricsReporterService.orNull?.also { it.addTask(path, this.javaClass, buildMetrics) } } protected open fun skipCondition(): Boolean = @@ -560,13 +560,18 @@ abstract class KotlinCompile @Inject constructor( } project.extensions.extraProperties[TRANSFORMS_REGISTERED] = true + val buildMetricsReporterService = BuildMetricsReporterService.registerIfAbsent(project) project.dependencies.registerTransform(ClasspathEntrySnapshotTransform::class.java) { it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, JAR_ARTIFACT_TYPE) it.to.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE) + it.parameters.gradleUserHomeDir.set(project.gradle.gradleUserHomeDir) + buildMetricsReporterService?.apply { it.parameters.buildMetricsReporterService.set(this) } } project.dependencies.registerTransform(ClasspathEntrySnapshotTransform::class.java) { it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_ARTIFACT_TYPE) it.to.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE) + it.parameters.gradleUserHomeDir.set(project.gradle.gradleUserHomeDir) + buildMetricsReporterService?.apply { it.parameters.buildMetricsReporterService.set(this) } } }