KT-45777: Take coarse-grained snapshots of external libraries
to reduce the size of the snapshots. - Track metrics for Gradle classpath snapshot artifact transform - Format size metrics so it's more readable ^KT-45777 In Progress
This commit is contained in:
committed by
Yahor Berdnikau
parent
54fb4d9b70
commit
5f1cf34c79
+15
-12
@@ -7,18 +7,22 @@ package org.jetbrains.kotlin.build.report.metrics
|
||||
|
||||
import java.io.Serializable
|
||||
|
||||
|
||||
@Suppress("Reformat")
|
||||
enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String, val type: BuildMetricType) : Serializable {
|
||||
CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory", type = BuildMetricType.FILE_SIZE),
|
||||
LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size", type = BuildMetricType.FILE_SIZE),
|
||||
SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size", type = BuildMetricType.FILE_SIZE),
|
||||
enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String, val type: SizeMetricType) : Serializable {
|
||||
CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory", type = SizeMetricType.BYTES),
|
||||
LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size", type = SizeMetricType.BYTES),
|
||||
SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size", type = SizeMetricType.BYTES),
|
||||
|
||||
COMPILE_ITERATION(parent = null, "Total compiler iteration", type = BuildMetricType.NUMBER),
|
||||
COMPILE_ITERATION(parent = null, "Total compiler iteration", type = SizeMetricType.NUMBER),
|
||||
|
||||
// Metrics for the `kotlin.incremental.useClasspathSnapshot` feature
|
||||
ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)", type = BuildMetricType.FILE_SIZE),
|
||||
SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot", type = BuildMetricType.FILE_SIZE),
|
||||
CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT(parent = null, "Number of times 'ClasspathEntrySnapshotTransform' ran", type = SizeMetricType.NUMBER),
|
||||
JAR_CLASSPATH_ENTRY_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of classpath entry (if it's a jar)", type = SizeMetricType.BYTES),
|
||||
CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of classpath entry (directory or jar)'s snapshot", type = SizeMetricType.BYTES),
|
||||
SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT(parent = null, "Number of times classpath snapshot is shrunk and saved after compilation", type = SizeMetricType.NUMBER),
|
||||
CLASSPATH_ENTRY_COUNT(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Number of classpath entries", type = SizeMetricType.NUMBER),
|
||||
CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of classpath snapshot", type = SizeMetricType.BYTES),
|
||||
SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of shrunk classpath snapshot", type = SizeMetricType.BYTES),
|
||||
;
|
||||
|
||||
companion object {
|
||||
@@ -30,8 +34,7 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va
|
||||
}
|
||||
}
|
||||
|
||||
enum class BuildMetricType {
|
||||
TIME_IN_MS,
|
||||
FILE_SIZE,
|
||||
enum class SizeMetricType {
|
||||
BYTES,
|
||||
NUMBER
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
|
||||
SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT, "Set up ABI snapshot"),
|
||||
IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT, "Analyze jar files"),
|
||||
IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION_DAEMON, "Calculate initial dirty sources set"), //TODO
|
||||
COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute task classpath changes"),
|
||||
COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"),
|
||||
LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"),
|
||||
REMOVE_DUPLICATE_CLASSES(LOAD_CURRENT_CLASSPATH_SNAPSHOT, "Remove duplicate classes"),
|
||||
SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"),
|
||||
@@ -51,14 +51,24 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
|
||||
COMPILATION_ROUND(INCREMENTAL_COMPILATION_DAEMON, "Sources compilation round"),
|
||||
IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION_DAEMON, "Write history file"),
|
||||
SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION_DAEMON, "Shrink and save current classpath snapshot after compilation"),
|
||||
LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load shrunk previous classpath snapshot after compilation"),
|
||||
LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load current classpath snapshot after compilation"),
|
||||
SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot after compilation"),
|
||||
SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot after compilation"),
|
||||
INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Load shrunk previous classpath snapshot"),
|
||||
INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Load current classpath snapshot"),
|
||||
INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Shrink current classpath snapshot"),
|
||||
NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Non-incremental shrinking - Load current classpath snapshot"),
|
||||
NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Non-incremental shrinking - Shrink current classpath snapshot"),
|
||||
SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk current classpath snapshot"),
|
||||
COMPILER_PERFORMANCE(readableString = "Compiler time"),
|
||||
COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"),
|
||||
CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analysis"),
|
||||
CODE_GENERATION(COMPILER_PERFORMANCE, "Compiler code generation"),
|
||||
CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM(parent = null, "Classpath entry snapshot transform"),
|
||||
LOAD_CLASSES(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM, "Load classes"),
|
||||
SNAPSHOT_CLASSES(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM, "Snapshot classes"),
|
||||
READ_CLASSES_BASIC_INFO(parent = SNAPSHOT_CLASSES, "Read basic information about classes"),
|
||||
FIND_INACCESSIBLE_CLASSES(parent = SNAPSHOT_CLASSES, "Find inaccessible classes"),
|
||||
SNAPSHOT_KOTLIN_CLASSES(parent = SNAPSHOT_CLASSES, "Snapshot Kotlin classes"),
|
||||
SNAPSHOT_JAVA_CLASSES(parent = SNAPSHOT_CLASSES, "Snapshot Java classes"),
|
||||
SAVE_CLASSPATH_ENTRY_SNAPSHOT(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM, "Save classpath entry snapshot")
|
||||
;
|
||||
|
||||
companion object {
|
||||
|
||||
+13
-1
@@ -6,6 +6,7 @@
|
||||
package org.jetbrains.kotlin.incremental.classpathDiff
|
||||
|
||||
import org.jetbrains.kotlin.incremental.KotlinClassInfo
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_LEVEL
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_MEMBER_LEVEL
|
||||
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader.Kind.*
|
||||
import org.jetbrains.kotlin.name.ClassId
|
||||
@@ -141,7 +142,18 @@ class JavaElementSnapshotForTests(
|
||||
*/
|
||||
object InaccessibleClassSnapshot : ClassSnapshot()
|
||||
|
||||
/** The granularity of a [ClassSnapshot]. */
|
||||
/**
|
||||
* The granularity of a [ClassSnapshot].
|
||||
*
|
||||
* There are currently two granularity levels:
|
||||
* - [CLASS_LEVEL]) (coarse-grained): The size of the snapshot will be smaller, but we will have coarse-grained classpath changes, which
|
||||
* means more source files will be recompiled.
|
||||
* - [CLASS_MEMBER_LEVEL] (fine-grained): The size of the snapshot will be larger, but we will have fine-grained classpath changes, which
|
||||
* means fewer source files will be recompiled.
|
||||
*
|
||||
* Therefore, [CLASS_LEVEL] is typically suitable for classes that are infrequently changed (e.g., external libraries), whereas
|
||||
* [CLASS_MEMBER_LEVEL] is suitable for classes that are frequently changed (e.g., classes produced by the current project).
|
||||
*/
|
||||
enum class ClassSnapshotGranularity {
|
||||
|
||||
/**
|
||||
|
||||
+12
-7
@@ -207,12 +207,12 @@ internal fun shrinkAndSaveClasspathSnapshot(
|
||||
ShrinkMode.NoChanges
|
||||
} else {
|
||||
val shrunkPreviousClasspathAgainstPreviousLookups =
|
||||
metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
|
||||
metrics.measure(BuildTime.INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) {
|
||||
ListExternalizer(AccessibleClassSnapshotExternalizer)
|
||||
.loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile)
|
||||
}
|
||||
ShrinkMode.Incremental(
|
||||
currentClasspathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
|
||||
currentClasspathSnapshot = metrics.measure(BuildTime.INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
|
||||
CachedClasspathSnapshotSerializer
|
||||
.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
|
||||
.removeDuplicateAndInaccessibleClasses()
|
||||
@@ -243,7 +243,7 @@ internal fun shrinkAndSaveClasspathSnapshot(
|
||||
// shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups
|
||||
shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups
|
||||
}
|
||||
is ShrinkMode.Incremental -> metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
|
||||
is ShrinkMode.Incremental -> metrics.measure(BuildTime.INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
|
||||
val shrunkClasses = shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups.mapTo(mutableSetOf()) { it.classId }
|
||||
val notYetShrunkClasses = shrinkMode.currentClasspathSnapshot.filter { it.classId !in shrunkClasses }
|
||||
val addedLookupSymbols = shrinkMode.addedLookupSymbols
|
||||
@@ -252,12 +252,12 @@ internal fun shrinkAndSaveClasspathSnapshot(
|
||||
shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups
|
||||
}
|
||||
is ShrinkMode.NonIncremental -> {
|
||||
val classpathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
|
||||
val classpathSnapshot = metrics.measure(BuildTime.NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
|
||||
CachedClasspathSnapshotSerializer
|
||||
.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
|
||||
.removeDuplicateAndInaccessibleClasses()
|
||||
}
|
||||
metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
|
||||
metrics.measure(BuildTime.NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
|
||||
shrinkClasspath(classpathSnapshot, lookupStorage)
|
||||
}
|
||||
}
|
||||
@@ -269,7 +269,7 @@ internal fun shrinkAndSaveClasspathSnapshot(
|
||||
"File '${classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.path}' does not exist"
|
||||
}
|
||||
} else {
|
||||
metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
|
||||
metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT) {
|
||||
ListExternalizer(AccessibleClassSnapshotExternalizer).saveToFile(
|
||||
classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
|
||||
shrunkCurrentClasspath!!
|
||||
@@ -277,8 +277,13 @@ internal fun shrinkAndSaveClasspathSnapshot(
|
||||
}
|
||||
}
|
||||
|
||||
metrics.addMetric(BuildPerformanceMetric.SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, 1)
|
||||
metrics.addMetric(
|
||||
BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE,
|
||||
BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT,
|
||||
classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.size.toLong()
|
||||
)
|
||||
metrics.addMetric(
|
||||
BuildPerformanceMetric.CLASSPATH_SNAPSHOT_SIZE,
|
||||
classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() }
|
||||
)
|
||||
metrics.addMetric(
|
||||
|
||||
+35
-16
@@ -5,6 +5,10 @@
|
||||
|
||||
package org.jetbrains.kotlin.incremental.classpathDiff
|
||||
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildTime
|
||||
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
|
||||
import org.jetbrains.kotlin.build.report.metrics.measure
|
||||
import org.jetbrains.kotlin.incremental.ChangesCollector.Companion.getNonPrivateMemberNames
|
||||
import org.jetbrains.kotlin.incremental.KotlinClassInfo
|
||||
import org.jetbrains.kotlin.incremental.PackagePartProtoData
|
||||
@@ -26,14 +30,22 @@ object ClasspathEntrySnapshotter {
|
||||
&& !unixStyleRelativePath.equals("module-info.class", ignoreCase = true)
|
||||
}
|
||||
|
||||
fun snapshot(classpathEntry: File): ClasspathEntrySnapshot {
|
||||
val classes = DirectoryOrJarContentsReader
|
||||
.read(classpathEntry, DEFAULT_CLASS_FILTER)
|
||||
.map { (unixStyleRelativePath, contents) ->
|
||||
ClassFileWithContents(ClassFile(classpathEntry, unixStyleRelativePath), contents)
|
||||
}
|
||||
fun snapshot(
|
||||
classpathEntry: File,
|
||||
granularity: ClassSnapshotGranularity,
|
||||
metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter
|
||||
): ClasspathEntrySnapshot {
|
||||
val classes = metrics.measure(BuildTime.LOAD_CLASSES) {
|
||||
DirectoryOrJarContentsReader
|
||||
.read(classpathEntry, DEFAULT_CLASS_FILTER)
|
||||
.map { (unixStyleRelativePath, contents) ->
|
||||
ClassFileWithContents(ClassFile(classpathEntry, unixStyleRelativePath), contents)
|
||||
}
|
||||
}
|
||||
|
||||
val snapshots = ClassSnapshotter.snapshot(classes)
|
||||
val snapshots = metrics.measure(BuildTime.SNAPSHOT_CLASSES) {
|
||||
ClassSnapshotter.snapshot(classes, granularity, metrics = metrics)
|
||||
}
|
||||
|
||||
val relativePathsToSnapshotsMap = classes.map { it.classFile.unixStyleRelativePath }.zip(snapshots).toMap(LinkedHashMap())
|
||||
return ClasspathEntrySnapshot(relativePathsToSnapshotsMap)
|
||||
@@ -47,17 +59,24 @@ object ClassSnapshotter {
|
||||
fun snapshot(
|
||||
classes: List<ClassFileWithContents>,
|
||||
granularity: ClassSnapshotGranularity = CLASS_MEMBER_LEVEL,
|
||||
includeDebugInfoInJavaSnapshot: Boolean = false
|
||||
includeDebugInfoInJavaSnapshot: Boolean = false,
|
||||
metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter
|
||||
): List<ClassSnapshot> {
|
||||
// Find inaccessible classes first
|
||||
val classesInfo: List<BasicClassInfo> = classes.map { it.classInfo }
|
||||
val inaccessibleClassesInfo: Set<BasicClassInfo> = getInaccessibleClasses(classesInfo).toSet()
|
||||
|
||||
val classesInfo: List<BasicClassInfo> = metrics.measure(BuildTime.READ_CLASSES_BASIC_INFO) {
|
||||
classes.map { it.classInfo }
|
||||
}
|
||||
val inaccessibleClassesInfo: Set<BasicClassInfo> = metrics.measure(BuildTime.FIND_INACCESSIBLE_CLASSES) {
|
||||
findInaccessibleClasses(classesInfo)
|
||||
}
|
||||
return classes.map {
|
||||
when {
|
||||
it.classInfo in inaccessibleClassesInfo -> InaccessibleClassSnapshot
|
||||
it.classInfo.isKotlinClass -> snapshotKotlinClass(it, granularity)
|
||||
else -> JavaClassSnapshotter.snapshot(it, granularity, includeDebugInfoInJavaSnapshot)
|
||||
it.classInfo.isKotlinClass -> metrics.measure(BuildTime.SNAPSHOT_KOTLIN_CLASSES) {
|
||||
snapshotKotlinClass(it, granularity)
|
||||
}
|
||||
else -> metrics.measure(BuildTime.SNAPSHOT_JAVA_CLASSES) {
|
||||
JavaClassSnapshotter.snapshot(it, granularity, includeDebugInfoInJavaSnapshot)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -96,7 +115,7 @@ object ClassSnapshotter {
|
||||
* NOTE: If we do not have enough info to determine whether a class is inaccessible, we will assume that the class is accessible to be
|
||||
* safe.
|
||||
*/
|
||||
private fun getInaccessibleClasses(classesInfo: List<BasicClassInfo>): List<BasicClassInfo> {
|
||||
private fun findInaccessibleClasses(classesInfo: List<BasicClassInfo>): Set<BasicClassInfo> {
|
||||
fun BasicClassInfo.isInaccessible(): Boolean {
|
||||
return if (this.isKotlinClass) {
|
||||
when (this.kotlinClassHeader!!.kind) {
|
||||
@@ -130,7 +149,7 @@ object ClassSnapshotter {
|
||||
}
|
||||
}
|
||||
|
||||
return classesInfo.filter { it.isTransitivelyInaccessible() }
|
||||
return classesInfo.filterTo(mutableSetOf()) { it.isTransitivelyInaccessible() }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-2
@@ -10,14 +10,15 @@ import java.io.Serializable
|
||||
class GradleBuildMetricsData : Serializable {
|
||||
val parentMetric: MutableMap<String, String?> = LinkedHashMap()
|
||||
val buildAttributeKind: MutableMap<String, String> = LinkedHashMap()
|
||||
val taskData: MutableMap<String, TaskData> = LinkedHashMap()
|
||||
val buildOperationData: MutableMap<String, BuildOperationData> = LinkedHashMap()
|
||||
|
||||
companion object {
|
||||
const val serialVersionUID = 0L
|
||||
}
|
||||
}
|
||||
|
||||
data class TaskData(
|
||||
/** Data for a build operation (e.g., task or transform). */
|
||||
data class BuildOperationData(
|
||||
val path: String,
|
||||
val typeFqName: String,
|
||||
val buildTimesMs: Map<String, Long>,
|
||||
|
||||
+1
-1
@@ -64,7 +64,7 @@ class BuildReportsIT : KGPBaseTest() {
|
||||
assertTrue { report.contains("Time metrics:") }
|
||||
assertTrue { report.contains("Run compilation:") }
|
||||
assertTrue { report.contains("Incremental compilation in daemon:") }
|
||||
assertTrue { report.contains("Build performance metrics:") }
|
||||
assertTrue { report.contains("Size metrics:") }
|
||||
assertTrue { report.contains("Total size of the cache directory:") }
|
||||
assertTrue { report.contains("Total compiler iteration:") }
|
||||
assertTrue { report.contains("ABI snapshot size:") }
|
||||
|
||||
+76
-5
@@ -6,26 +6,97 @@
|
||||
package org.jetbrains.kotlin.gradle.internal.transforms
|
||||
|
||||
import org.gradle.api.artifacts.transform.*
|
||||
import org.gradle.api.file.DirectoryProperty
|
||||
import org.gradle.api.file.FileSystemLocation
|
||||
import org.gradle.api.provider.Property
|
||||
import org.gradle.api.provider.Provider
|
||||
import org.gradle.api.tasks.Classpath
|
||||
import org.gradle.api.tasks.Internal
|
||||
import org.jetbrains.kotlin.build.report.metrics.*
|
||||
import org.jetbrains.kotlin.gradle.report.BuildMetricsReporterService
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_LEVEL
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_MEMBER_LEVEL
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotExternalizer
|
||||
import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter
|
||||
import org.jetbrains.kotlin.incremental.storage.saveToFile
|
||||
import java.io.File
|
||||
|
||||
/** Transform to create a snapshot of a classpath entry (directory or jar). */
|
||||
@CacheableTransform
|
||||
abstract class ClasspathEntrySnapshotTransform : TransformAction<TransformParameters.None> {
|
||||
abstract class ClasspathEntrySnapshotTransform : TransformAction<ClasspathEntrySnapshotTransform.Parameters> {
|
||||
|
||||
abstract class Parameters : TransformParameters {
|
||||
@get:Internal
|
||||
abstract val gradleUserHomeDir: DirectoryProperty
|
||||
|
||||
@get:Internal
|
||||
abstract val buildMetricsReporterService: Property<BuildMetricsReporterService>
|
||||
}
|
||||
|
||||
@get:Classpath
|
||||
@get:InputArtifact
|
||||
abstract val inputArtifact: Provider<FileSystemLocation>
|
||||
|
||||
override fun transform(outputs: TransformOutputs) {
|
||||
val classpathEntry = inputArtifact.get().asFile
|
||||
val snapshotFile = outputs.file(classpathEntry.name.replace('.', '_') + "-snapshot.bin")
|
||||
val classpathEntryInputDirOrJar = inputArtifact.get().asFile
|
||||
val snapshotOutputFile = outputs.file(classpathEntryInputDirOrJar.name.replace('.', '_') + "-snapshot.bin")
|
||||
|
||||
val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntry)
|
||||
ClasspathEntrySnapshotExternalizer.saveToFile(snapshotFile, snapshot)
|
||||
val granularity = getClassSnapshotGranularity(classpathEntryInputDirOrJar, parameters.gradleUserHomeDir.get().asFile)
|
||||
|
||||
val buildMetricsReporterService = parameters.buildMetricsReporterService.orNull
|
||||
val metricsReporter = buildMetricsReporterService?.let { BuildMetricsReporterImpl() } ?: DoNothingBuildMetricsReporter
|
||||
|
||||
val startTimeMs = System.currentTimeMillis()
|
||||
var failureMessage: String? = null
|
||||
try {
|
||||
doTransform(classpathEntryInputDirOrJar, snapshotOutputFile, granularity, metricsReporter)
|
||||
} catch (e: Throwable) {
|
||||
failureMessage = e.message
|
||||
throw e
|
||||
} finally {
|
||||
buildMetricsReporterService?.addTransformMetrics(
|
||||
transformPath = "${ClasspathEntrySnapshotTransform::class.simpleName} for ${classpathEntryInputDirOrJar.path}",
|
||||
transformClass = ClasspathEntrySnapshotTransform::class.java,
|
||||
isKotlinTransform = true,
|
||||
startTimeMs = startTimeMs,
|
||||
totalTimeMs = System.currentTimeMillis() - startTimeMs,
|
||||
buildMetrics = metricsReporter.getMetrics(),
|
||||
failureMessage = failureMessage
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the [ClassSnapshotGranularity] when taking a snapshot of the given [classpathEntryDirOrJar].
|
||||
*
|
||||
* As mentioned in [ClassSnapshotGranularity]'s kdoc, we will take [CLASS_LEVEL] snapshots for classes that are infrequently changed
|
||||
* (e.g., external libraries which are typically stored/transformed inside the Gradle user home, or a few hard-coded cases), and take
|
||||
* [CLASS_MEMBER_LEVEL] snapshots for the others.
|
||||
*/
|
||||
private fun getClassSnapshotGranularity(classpathEntryDirOrJar: File, gradleUserHomeDir: File): ClassSnapshotGranularity {
|
||||
return if (
|
||||
classpathEntryDirOrJar.startsWith(gradleUserHomeDir) ||
|
||||
classpathEntryDirOrJar.name == "android.jar"
|
||||
) CLASS_LEVEL
|
||||
else CLASS_MEMBER_LEVEL
|
||||
}
|
||||
|
||||
private fun doTransform(
|
||||
classpathEntryInputDirOrJar: File, snapshotOutputFile: File,
|
||||
granularity: ClassSnapshotGranularity, metrics: BuildMetricsReporter
|
||||
) {
|
||||
metrics.measure(BuildTime.CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM) {
|
||||
val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntryInputDirOrJar, granularity, metrics)
|
||||
metrics.measure(BuildTime.SAVE_CLASSPATH_ENTRY_SNAPSHOT) {
|
||||
ClasspathEntrySnapshotExternalizer.saveToFile(snapshotOutputFile, snapshot)
|
||||
}
|
||||
|
||||
metrics.addMetric(BuildPerformanceMetric.CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, 1)
|
||||
if (classpathEntryInputDirOrJar.extension.equals("jar", ignoreCase = true)) {
|
||||
metrics.addMetric(BuildPerformanceMetric.JAR_CLASSPATH_ENTRY_SIZE, classpathEntryInputDirOrJar.length())
|
||||
}
|
||||
metrics.addMetric(BuildPerformanceMetric.CLASSPATH_ENTRY_SNAPSHOT_SIZE, snapshotOutputFile.length())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-2
@@ -85,8 +85,7 @@ abstract class KotlinBasePluginWrapper : Plugin<Project> {
|
||||
|
||||
KotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, kotlinPluginVersion)
|
||||
|
||||
val buildMetricReporter =
|
||||
BuildMetricsReporterService.registerIfAbsent(project, BuildMetricsReporterService.getStartParameters(project))
|
||||
val buildMetricReporter = BuildMetricsReporterService.registerIfAbsent(project)
|
||||
|
||||
buildMetricReporter?.also { BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(it) }
|
||||
|
||||
|
||||
+3
-15
@@ -10,13 +10,12 @@ import org.gradle.api.logging.Logging
|
||||
import org.gradle.tooling.events.FinishEvent
|
||||
import org.gradle.tooling.events.OperationCompletionListener
|
||||
import org.gradle.tooling.events.task.TaskFinishEvent
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildMetricType
|
||||
import org.jetbrains.kotlin.build.report.metrics.SizeMetricType
|
||||
import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
|
||||
import org.jetbrains.kotlin.gradle.report.formatSize
|
||||
import org.jetbrains.kotlin.konan.file.File
|
||||
import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult
|
||||
import java.util.*
|
||||
import kotlin.collections.ArrayList
|
||||
import kotlin.collections.LinkedHashSet
|
||||
import kotlin.system.measureTimeMillis
|
||||
|
||||
class BuildScanStatisticsListener(
|
||||
@@ -26,9 +25,6 @@ class BuildScanStatisticsListener(
|
||||
val kotlinVersion: String,
|
||||
) : OperationCompletionListener, AutoCloseable {
|
||||
companion object {
|
||||
const val kbSize = 1024
|
||||
const val mbSize = kbSize * kbSize
|
||||
const val gbSize = kbSize * mbSize
|
||||
const val lengthLimit = 100_000
|
||||
}
|
||||
|
||||
@@ -83,7 +79,7 @@ class BuildScanStatisticsListener(
|
||||
data.buildTimesMetrics.map { (key, value) -> "${key.readableString}: ${value}ms" } //sometimes it is better to have separate variable to be able debug
|
||||
val perfData = data.performanceMetrics.map { (key, value) ->
|
||||
when (key.type) {
|
||||
BuildMetricType.FILE_SIZE -> "${key.readableString}: ${readableFileLength(value)}"
|
||||
SizeMetricType.BYTES -> "${key.readableString}: ${formatSize(value)}"
|
||||
else -> "${key.readableString}: $value}"
|
||||
}
|
||||
}
|
||||
@@ -111,12 +107,4 @@ class BuildScanStatisticsListener(
|
||||
splattedString.add(tempStr)
|
||||
return splattedString
|
||||
}
|
||||
|
||||
private fun readableFileLength(length: Long): String =
|
||||
when {
|
||||
length / gbSize > 0 -> "${length / gbSize} GB"
|
||||
length / mbSize > 0 -> "${length / mbSize} MB"
|
||||
length / kbSize > 0 -> "${length / kbSize} KB"
|
||||
else -> "$length B"
|
||||
}
|
||||
}
|
||||
|
||||
+97
-67
@@ -19,11 +19,10 @@ import org.gradle.tooling.events.task.TaskSkippedResult
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildTime
|
||||
import org.jetbrains.kotlin.gradle.plugin.KotlinGradleBuildServices
|
||||
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
|
||||
import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
|
||||
@@ -38,42 +37,67 @@ abstract class BuildMetricsReporterService : BuildService<BuildMetricsReporterSe
|
||||
|
||||
private val log = Logging.getLogger(this.javaClass)
|
||||
|
||||
//Task path to build metrics
|
||||
private val buildMetricsMap = ConcurrentHashMap<String, BuildMetricsReporter>()
|
||||
private val taskRecords = ConcurrentHashMap<String, TaskRecord>()
|
||||
private val failureMessages = ConcurrentLinkedQueue<String?>()
|
||||
private val taskClass = ConcurrentHashMap<String, String>()
|
||||
// Tasks and transforms' records
|
||||
private val buildOperationRecords = ConcurrentLinkedQueue<BuildOperationRecord>()
|
||||
private val failureMessages = ConcurrentLinkedQueue<String>()
|
||||
|
||||
open fun add(taskPath: String, type: String, metrics: BuildMetricsReporter) {
|
||||
if (buildMetricsMap.containsKey(taskPath)) {
|
||||
log.warn("Duplicate path $taskPath")
|
||||
// Info for tasks only
|
||||
private val taskPathToMetricsReporter = ConcurrentHashMap<String, BuildMetricsReporter>()
|
||||
private val taskPathToTaskClass = ConcurrentHashMap<String, String>()
|
||||
|
||||
open fun addTask(taskPath: String, taskClass: Class<*>, metricsReporter: BuildMetricsReporter) {
|
||||
taskPathToMetricsReporter.put(taskPath, metricsReporter).also {
|
||||
if (it != null) log.warn("Duplicate task path: $taskPath") // Should never happen but log it just in case
|
||||
}
|
||||
buildMetricsMap[taskPath] = metrics
|
||||
taskClass[taskPath] = type
|
||||
taskPathToTaskClass.put(taskPath, taskClass.name).also {
|
||||
if (it != null) log.warn("Duplicate task path: $taskPath") // Should never happen but log it just in case
|
||||
}
|
||||
}
|
||||
|
||||
open fun addTransformMetrics(
|
||||
transformPath: String,
|
||||
transformClass: Class<*>,
|
||||
isKotlinTransform: Boolean,
|
||||
startTimeMs: Long,
|
||||
totalTimeMs: Long,
|
||||
buildMetrics: BuildMetrics,
|
||||
failureMessage: String?
|
||||
) {
|
||||
buildOperationRecords.add(
|
||||
TransformRecord(transformPath, transformClass.name, isKotlinTransform, startTimeMs, totalTimeMs, buildMetrics)
|
||||
)
|
||||
failureMessage?.let { failureMessages.add(it) }
|
||||
}
|
||||
|
||||
override fun onFinish(event: FinishEvent?) {
|
||||
if (event is TaskFinishEvent) {
|
||||
val result = event.result
|
||||
val taskPath = event.descriptor.taskPath
|
||||
val startMs = event.result.startTime
|
||||
val finishMs = event.result.endTime
|
||||
val totalTimeMs = result.endTime - result.startTime
|
||||
|
||||
val buildMetrics = BuildMetrics()
|
||||
buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, totalTimeMs)
|
||||
taskPathToMetricsReporter[taskPath]?.let {
|
||||
buildMetrics.addAll(it.getMetrics())
|
||||
}
|
||||
val taskExecutionResult = TaskExecutionResults[taskPath]
|
||||
taskExecutionResult?.buildMetrics?.also { buildMetrics.addAll(it) }
|
||||
|
||||
buildOperationRecords.add(
|
||||
TaskRecord(
|
||||
path = taskPath,
|
||||
classFqName = taskPathToTaskClass[taskPath] ?: "unknown",
|
||||
startTimeMs = result.startTime,
|
||||
totalTimeMs = totalTimeMs,
|
||||
buildMetrics = buildMetrics,
|
||||
didWork = result is TaskExecutionResult,
|
||||
skipMessage = (result as? TaskSkippedResult)?.skipMessage,
|
||||
icLogLines = taskExecutionResult?.icLogLines ?: emptyList()
|
||||
)
|
||||
)
|
||||
if (result is TaskFailureResult) {
|
||||
failureMessages.addAll(result.failures.map { it.message })
|
||||
}
|
||||
val skipMessage = when (result) {
|
||||
is TaskSkippedResult -> result.skipMessage
|
||||
else -> null
|
||||
}
|
||||
val didWork = result is TaskExecutionResult
|
||||
val buildMetrics = BuildMetrics()
|
||||
buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, finishMs - startMs)
|
||||
|
||||
buildMetricsMap[taskPath]?.also { buildMetrics.addAll(it.getMetrics()) }
|
||||
val taskExecutionResult = TaskExecutionResults[taskPath]
|
||||
val icLogLines = taskExecutionResult?.icLogLines ?: emptyList()
|
||||
taskExecutionResult?.buildMetrics?.also { buildMetrics.addAll(it) }
|
||||
taskRecords[taskPath] = TaskRecord(taskPath, startMs, finishMs, skipMessage, didWork, icLogLines, buildMetrics, taskClass[taskPath] ?: "unknown")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,19 +105,16 @@ abstract class BuildMetricsReporterService : BuildService<BuildMetricsReporterSe
|
||||
val buildData = BuildExecutionData(
|
||||
startParameters = parameters.startParameters,
|
||||
failureMessages = failureMessages.toList(),
|
||||
taskExecutionData = taskRecords.values.sortedBy { it.startMs }
|
||||
buildOperationRecord = buildOperationRecords.sortedBy { it.startTimeMs }
|
||||
)
|
||||
parameters.buildDataProcessors.forEach { it.process(buildData, log) }
|
||||
buildMetricsMap.clear()
|
||||
taskRecords.clear()
|
||||
failureMessages.clear()
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
||||
fun getStartParameters(project: Project) = project.gradle.startParameter.let {
|
||||
val startParameters = arrayListOf<String>()
|
||||
startParameters.add("tasks = ${it.taskRequests.joinToString { it.args.toString() }}")
|
||||
startParameters.add("tasks = ${it.taskRequests.joinToString { task -> task.args.toString() }}")
|
||||
startParameters.add("excluded tasks = ${it.excludedTaskNames}")
|
||||
startParameters.add("current dir = ${it.currentDir}")
|
||||
startParameters.add("project properties args = ${it.projectProperties}")
|
||||
@@ -101,33 +122,34 @@ abstract class BuildMetricsReporterService : BuildService<BuildMetricsReporterSe
|
||||
startParameters
|
||||
}
|
||||
|
||||
fun registerIfAbsent(project: Project, startParameters: List<String>): Provider<BuildMetricsReporterService>? {
|
||||
val rootProject = project.gradle.rootProject
|
||||
val reportingSettings = reportingSettings(rootProject)
|
||||
fun registerIfAbsent(project: Project): Provider<BuildMetricsReporterService>? {
|
||||
val serviceClass = BuildMetricsReporterService::class.java
|
||||
val serviceName = "${serviceClass.name}_${serviceClass.classLoader.hashCode()}"
|
||||
|
||||
val buildDataProcessors = ArrayList<BuildExecutionDataProcessor>()
|
||||
reportingSettings.fileReportSettings?.let {
|
||||
buildDataProcessors.add(
|
||||
PlainTextBuildReportWriterDataProcessor(
|
||||
it,
|
||||
rootProject.name
|
||||
)
|
||||
)
|
||||
// Return early if the service was already registered to avoid the overhead of reading the reporting settings below
|
||||
project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
return it.service as Provider<BuildMetricsReporterService>
|
||||
}
|
||||
|
||||
if (reportingSettings.metricsOutputFile != null) {
|
||||
buildDataProcessors.add(MetricsWriter(reportingSettings.metricsOutputFile.absoluteFile))
|
||||
}
|
||||
|
||||
if (reportingSettings.buildReportOutputs.isEmpty() && buildDataProcessors.isEmpty()) {
|
||||
val reportingSettings = reportingSettings(project.rootProject)
|
||||
if (reportingSettings.buildReportOutputs.isEmpty()
|
||||
&& reportingSettings.fileReportSettings == null
|
||||
&& reportingSettings.metricsOutputFile == null
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
return project.gradle.sharedServices.registerIfAbsent(
|
||||
"build_metric_service_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}",
|
||||
BuildMetricsReporterService::class.java
|
||||
) {
|
||||
it.parameters.startParameters = startParameters
|
||||
return project.gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) {
|
||||
val buildDataProcessors = mutableListOf<BuildExecutionDataProcessor>()
|
||||
reportingSettings.fileReportSettings?.let { fileReportSettings ->
|
||||
buildDataProcessors.add(PlainTextBuildReportWriterDataProcessor(fileReportSettings, project.rootProject.name))
|
||||
}
|
||||
reportingSettings.metricsOutputFile?.let { metricsOutputFile ->
|
||||
buildDataProcessors.add(MetricsWriter(metricsOutputFile.absoluteFile))
|
||||
}
|
||||
|
||||
it.parameters.startParameters = getStartParameters(project)
|
||||
it.parameters.buildDataProcessors = buildDataProcessors
|
||||
it.parameters.reportingSettings = reportingSettings
|
||||
}!!
|
||||
@@ -138,19 +160,27 @@ abstract class BuildMetricsReporterService : BuildService<BuildMetricsReporterSe
|
||||
}
|
||||
|
||||
private class TaskRecord(
|
||||
override val taskPath: String,
|
||||
override val startMs: Long,
|
||||
override val endMs: Long,
|
||||
override val skipMessage: String?,
|
||||
override val didWork: Boolean,
|
||||
override val icLogLines: List<String>,
|
||||
override val path: String,
|
||||
override val classFqName: String,
|
||||
override val startTimeMs: Long,
|
||||
override val totalTimeMs: Long,
|
||||
override val buildMetrics: BuildMetrics,
|
||||
override val type: String
|
||||
) : TaskExecutionData {
|
||||
override val isKotlinTask by lazy {
|
||||
type.startsWith("org.jetbrains.kotlin")
|
||||
}
|
||||
override val didWork: Boolean,
|
||||
override val skipMessage: String?,
|
||||
override val icLogLines: List<String>
|
||||
) : BuildOperationRecord {
|
||||
override val isFromKotlinPlugin: Boolean = classFqName.startsWith("org.jetbrains.kotlin")
|
||||
}
|
||||
|
||||
override val totalTimeMs: Long
|
||||
get() = (endMs - startMs)
|
||||
}
|
||||
private class TransformRecord(
|
||||
override val path: String,
|
||||
override val classFqName: String,
|
||||
override val isFromKotlinPlugin: Boolean,
|
||||
override val startTimeMs: Long,
|
||||
override val totalTimeMs: Long,
|
||||
override val buildMetrics: BuildMetrics
|
||||
) : BuildOperationRecord {
|
||||
override val didWork: Boolean = true
|
||||
override val skipMessage: String? = null
|
||||
override val icLogLines: List<String> = emptyList()
|
||||
}
|
||||
|
||||
+12
-2
@@ -106,5 +106,15 @@ internal fun formatTime(ms: Long): String {
|
||||
return seconds.asString(2) + " s"
|
||||
}
|
||||
|
||||
internal fun Double.asString(decPoints: Int): String =
|
||||
String.format("%.${decPoints}f", this)
|
||||
private const val kbSize = 1024
|
||||
private const val mbSize = kbSize * 1024
|
||||
private const val gbSize = mbSize * 1024
|
||||
|
||||
internal fun formatSize(sizeInBytes: Long): String = when {
|
||||
sizeInBytes / gbSize >= 1 -> "${(sizeInBytes.toDouble() / gbSize).asString(1)} GB"
|
||||
sizeInBytes / mbSize >= 1 -> "${(sizeInBytes.toDouble() / mbSize).asString(1)} MB"
|
||||
sizeInBytes / kbSize >= 1 -> "${(sizeInBytes.toDouble() / kbSize).asString(1)} KB"
|
||||
else -> "$sizeInBytes B"
|
||||
}
|
||||
|
||||
internal fun Double.asString(decPoints: Int): String = "%,.${decPoints}f".format(this)
|
||||
|
||||
+6
-6
@@ -9,7 +9,7 @@ import org.gradle.api.logging.Logger
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildTime
|
||||
import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData
|
||||
import org.jetbrains.kotlin.gradle.internal.build.metrics.TaskData
|
||||
import org.jetbrains.kotlin.gradle.internal.build.metrics.BuildOperationData
|
||||
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
|
||||
@@ -34,11 +34,11 @@ internal class MetricsWriter(
|
||||
buildMetricsData.buildAttributeKind[attr.name] = attr.kind.name
|
||||
}
|
||||
|
||||
for (data in build.taskExecutionData) {
|
||||
buildMetricsData.taskData[data.taskPath] =
|
||||
TaskData(
|
||||
path = data.taskPath,
|
||||
typeFqName = data.type,
|
||||
for (data in build.buildOperationRecord) {
|
||||
buildMetricsData.buildOperationData[data.path] =
|
||||
BuildOperationData(
|
||||
path = data.path,
|
||||
typeFqName = data.classFqName,
|
||||
buildTimesMs = data.buildMetrics.buildTimes.asMapMs().mapKeys { it.key.name },
|
||||
performanceMetrics = data.buildMetrics.buildPerformanceMetrics.asMap().mapKeys { it.key.name },
|
||||
buildAttributes = data.buildMetrics.buildAttributes.asMap().mapKeys { it.key.name },
|
||||
|
||||
+47
-22
@@ -9,7 +9,7 @@ import org.gradle.api.logging.Logger
|
||||
import org.jetbrains.kotlin.build.report.metrics.*
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
|
||||
import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
|
||||
import org.jetbrains.kotlin.gradle.utils.Printer
|
||||
import java.io.File
|
||||
import java.io.Serializable
|
||||
@@ -59,8 +59,15 @@ internal class PlainTextBuildReportWriter(
|
||||
}
|
||||
|
||||
private fun printBuildReport(build: BuildExecutionData) {
|
||||
// NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms.
|
||||
// Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound
|
||||
// a bit unfamiliar.
|
||||
// TODO: If it is confusing, consider renaming "tasks" to "build operations" in this class.
|
||||
printBuildInfo(build)
|
||||
printMetrics(build.aggregatedMetrics)
|
||||
if (printMetrics) {
|
||||
printMetrics(build.aggregatedMetrics)
|
||||
p.println()
|
||||
}
|
||||
printTaskOverview(build)
|
||||
printTasksLog(build)
|
||||
}
|
||||
@@ -69,17 +76,15 @@ internal class PlainTextBuildReportWriter(
|
||||
p.withIndent("Gradle start parameters:") {
|
||||
build.startParameters.forEach { p.println(it) }
|
||||
}
|
||||
|
||||
p.println()
|
||||
|
||||
if (build.failureMessages.isNotEmpty()) {
|
||||
p.println("Build failed: ${build.failureMessages}")
|
||||
p.println()
|
||||
}
|
||||
p.println()
|
||||
}
|
||||
|
||||
private fun printMetrics(buildMetrics: BuildMetrics) {
|
||||
if (!printMetrics) return
|
||||
|
||||
printBuildTimes(buildMetrics.buildTimes)
|
||||
printBuildPerformanceMetrics(buildMetrics.buildPerformanceMetrics)
|
||||
printBuildAttributes(buildMetrics.buildAttributes)
|
||||
@@ -114,19 +119,38 @@ internal class PlainTextBuildReportWriter(
|
||||
printBuildTime(buildTime)
|
||||
}
|
||||
}
|
||||
p.println()
|
||||
}
|
||||
|
||||
private fun printBuildPerformanceMetrics(buildMetrics: BuildPerformanceMetrics) {
|
||||
val allBuildMetrics = buildMetrics.asMap()
|
||||
if (allBuildMetrics.isEmpty()) return
|
||||
|
||||
p.withIndent("Build performance metrics:") {
|
||||
p.withIndent("Size metrics:") {
|
||||
for (metric in BuildPerformanceMetric.values()) {
|
||||
allBuildMetrics[metric]?.let { p.println("${metric.readableString}: $it") }
|
||||
allBuildMetrics[metric]?.let { printSizeMetric(metric, it) }
|
||||
}
|
||||
}
|
||||
p.println()
|
||||
}
|
||||
|
||||
private fun printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) {
|
||||
fun BuildPerformanceMetric.numberOfAncestors(): Int {
|
||||
var count = 0
|
||||
var parent: BuildPerformanceMetric? = parent
|
||||
while (parent != null) {
|
||||
count++
|
||||
parent = parent.parent
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
val indentLevel = sizeMetric.numberOfAncestors()
|
||||
|
||||
repeat(indentLevel) { p.pushIndent() }
|
||||
when (sizeMetric.type) {
|
||||
SizeMetricType.BYTES -> p.println("${sizeMetric.readableString}: ${formatSize(value)}")
|
||||
SizeMetricType.NUMBER -> p.println("${sizeMetric.readableString}: $value")
|
||||
}
|
||||
repeat(indentLevel) { p.popIndent() }
|
||||
}
|
||||
|
||||
private fun printBuildAttributes(buildAttributes: BuildAttributes) {
|
||||
@@ -139,19 +163,18 @@ internal class PlainTextBuildReportWriter(
|
||||
printMap(p, kind.name, attributesCounts.map { (k, v) -> k.readableString to v }.toMap())
|
||||
}
|
||||
}
|
||||
p.println()
|
||||
}
|
||||
|
||||
private fun printTaskOverview(build: BuildExecutionData) {
|
||||
var allTasksTimeMs = 0L
|
||||
var kotlinTotalTimeMs = 0L
|
||||
val kotlinTasks = ArrayList<TaskExecutionData>()
|
||||
val kotlinTasks = ArrayList<BuildOperationRecord>()
|
||||
|
||||
for (task in build.taskExecutionData) {
|
||||
for (task in build.buildOperationRecord) {
|
||||
val taskTimeMs = task.totalTimeMs
|
||||
allTasksTimeMs += taskTimeMs
|
||||
|
||||
if (task.isKotlinTask) {
|
||||
if (task.isFromKotlinPlugin) {
|
||||
kotlinTotalTimeMs += taskTimeMs
|
||||
kotlinTasks.add(task)
|
||||
}
|
||||
@@ -166,37 +189,39 @@ internal class PlainTextBuildReportWriter(
|
||||
p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)")
|
||||
|
||||
val table = TextTable("Time", "% of Kotlin time", "Task")
|
||||
for (task in kotlinTasks.sortedByDescending { it.totalTimeMs }) {
|
||||
for (task in kotlinTasks.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) {
|
||||
val timeMs = task.totalTimeMs
|
||||
val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1)
|
||||
table.addRow(formatTime(timeMs), "$percent %", task.taskPath)
|
||||
table.addRow(formatTime(timeMs), "$percent %", task.path)
|
||||
}
|
||||
table.printTo(p)
|
||||
p.println()
|
||||
}
|
||||
|
||||
private fun printTasksLog(build: BuildExecutionData) {
|
||||
for (task in build.taskExecutionData) {
|
||||
for (task in build.buildOperationRecord.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) {
|
||||
printTaskLog(task)
|
||||
p.println()
|
||||
}
|
||||
}
|
||||
|
||||
private fun printTaskLog(task: TaskExecutionData) {
|
||||
private fun printTaskLog(task: BuildOperationRecord) {
|
||||
val skipMessage = task.skipMessage
|
||||
if (skipMessage != null) {
|
||||
p.println("Task '${task.taskPath}' was skipped: $skipMessage")
|
||||
p.println("Task '${task.path}' was skipped: $skipMessage")
|
||||
} else {
|
||||
p.println("Task '${task.taskPath}' finished in ${formatTime(task.totalTimeMs)}")
|
||||
p.println("Task '${task.path}' finished in ${formatTime(task.totalTimeMs)}")
|
||||
}
|
||||
|
||||
if (task.icLogLines.isNotEmpty()) {
|
||||
p.withIndent("Compilation log for task '${task.taskPath}':") {
|
||||
p.withIndent("Compilation log for task '${task.path}':") {
|
||||
task.icLogLines.forEach { p.println(it) }
|
||||
}
|
||||
}
|
||||
|
||||
printMetrics(task.buildMetrics)
|
||||
if (printMetrics) {
|
||||
printMetrics(task.buildMetrics)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
-1
@@ -22,7 +22,6 @@ data class ReportingSettings(
|
||||
}
|
||||
|
||||
data class FileReportSettings(
|
||||
val metricsOutputFile: File? = null,
|
||||
val buildReportDir: File,
|
||||
val includeMetricsInReport: Boolean = false,
|
||||
) : Serializable {
|
||||
|
||||
+2
-2
@@ -10,11 +10,11 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
|
||||
class BuildExecutionData(
|
||||
val startParameters: Collection<String>,
|
||||
val failureMessages: List<String?>,
|
||||
val taskExecutionData: Collection<TaskExecutionData>
|
||||
val buildOperationRecord: Collection<BuildOperationRecord>
|
||||
) {
|
||||
val aggregatedMetrics by lazy {
|
||||
BuildMetrics().also { acc ->
|
||||
taskExecutionData.forEach { acc.addAll(it.buildMetrics) }
|
||||
buildOperationRecord.forEach { acc.addAll(it.buildMetrics) }
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-12
@@ -5,20 +5,17 @@
|
||||
|
||||
package org.jetbrains.kotlin.gradle.report.data
|
||||
|
||||
import org.gradle.api.Task
|
||||
import org.gradle.api.tasks.TaskState
|
||||
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
|
||||
|
||||
interface TaskExecutionData {
|
||||
val taskPath: String
|
||||
val startMs: Long
|
||||
val endMs: Long
|
||||
/** Data for a build operation (e.g., task or transform). */
|
||||
interface BuildOperationRecord {
|
||||
val path: String
|
||||
val classFqName: String
|
||||
val isFromKotlinPlugin: Boolean
|
||||
val startTimeMs: Long // Measured by System.currentTimeMillis()
|
||||
val totalTimeMs: Long
|
||||
val skipMessage: String?
|
||||
val didWork: Boolean
|
||||
val icLogLines: List<String>
|
||||
val buildMetrics: BuildMetrics
|
||||
val isKotlinTask: Boolean
|
||||
val type: String
|
||||
val didWork: Boolean
|
||||
val skipMessage: String?
|
||||
val icLogLines: List<String>
|
||||
}
|
||||
|
||||
+6
-1
@@ -392,7 +392,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
|
||||
executeImpl(inputChanges, outputsBackup)
|
||||
}
|
||||
|
||||
buildMetricsReporterService.orNull?.also { it.add(path, this::class.java.name, buildMetrics) }
|
||||
buildMetricsReporterService.orNull?.also { it.addTask(path, this.javaClass, buildMetrics) }
|
||||
}
|
||||
|
||||
protected open fun skipCondition(): Boolean =
|
||||
@@ -560,13 +560,18 @@ abstract class KotlinCompile @Inject constructor(
|
||||
}
|
||||
project.extensions.extraProperties[TRANSFORMS_REGISTERED] = true
|
||||
|
||||
val buildMetricsReporterService = BuildMetricsReporterService.registerIfAbsent(project)
|
||||
project.dependencies.registerTransform(ClasspathEntrySnapshotTransform::class.java) {
|
||||
it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, JAR_ARTIFACT_TYPE)
|
||||
it.to.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE)
|
||||
it.parameters.gradleUserHomeDir.set(project.gradle.gradleUserHomeDir)
|
||||
buildMetricsReporterService?.apply { it.parameters.buildMetricsReporterService.set(this) }
|
||||
}
|
||||
project.dependencies.registerTransform(ClasspathEntrySnapshotTransform::class.java) {
|
||||
it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_ARTIFACT_TYPE)
|
||||
it.to.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE)
|
||||
it.parameters.gradleUserHomeDir.set(project.gradle.gradleUserHomeDir)
|
||||
buildMetricsReporterService?.apply { it.parameters.buildMetricsReporterService.set(this) }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user