Add minimal statistic report for JPS build

Fix build stat for gradle 8

#KT-56438 Fixed
This commit is contained in:
nataliya.valtman
2023-02-20 12:17:46 +01:00
committed by Space Team
parent f3ca465647
commit e34dd043da
59 changed files with 1176 additions and 789 deletions
@@ -74,6 +74,7 @@ dependencies {
testImplementation(project(":kotlin-compiler-embeddable"))
testImplementation(commonDependency("org.jetbrains.intellij.deps:jdom"))
testImplementation(project(":compiler:cli-common"))
testImplementation(project(":kotlin-build-statistic"))
// testCompileOnly dependency on non-shaded artifacts is needed for IDE support
// testRuntimeOnly on shaded artifact is needed for running tests with shaded compiler
testCompileOnly(project(":kotlin-gradle-plugin-test-utils-embeddable"))
@@ -17,9 +17,8 @@ import io.ktor.server.engine.*
import io.ktor.server.netty.*
import io.ktor.util.*
import io.ktor.util.collections.*
import org.gradle.api.logging.LogLevel
import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.gradle.plugin.stat.*
import org.jetbrains.kotlin.build.report.statistic.*
import org.jetbrains.kotlin.gradle.report.BuildReportType
import org.jetbrains.kotlin.gradle.testbase.*
import org.junit.jupiter.api.DisplayName
@@ -52,7 +52,6 @@ dependencies {
commonCompileOnly(intellijCore())
commonCompileOnly(commonDependency("org.jetbrains.teamcity:serviceMessages"))
commonCompileOnly("com.gradle:gradle-enterprise-gradle-plugin:3.12.4")
commonCompileOnly(commonDependency("com.google.code.gson:gson"))
commonCompileOnly(commonDependency("com.google.guava:guava"))
commonCompileOnly("de.undercouch:gradle-download-task:4.1.1")
commonCompileOnly("com.github.gundy:semver4j:0.16.4:nodeps") {
@@ -67,6 +66,7 @@ dependencies {
commonImplementation(project(":native:kotlin-klib-commonizer-api"))
commonImplementation(project(":kotlin-project-model"))
commonImplementation(project(":compiler:build-tools:kotlin-build-tools-api"))
commonImplementation(project(":kotlin-build-statistic"))
commonRuntimeOnly(project(":kotlin-compiler-embeddable"))
commonRuntimeOnly(project(":kotlin-annotation-processing-embeddable"))
@@ -111,6 +111,7 @@ dependencies {
testImplementation(commonDependency("junit:junit"))
testImplementation(project(":kotlin-gradle-statistics"))
testImplementation(project(":kotlin-tooling-metadata"))
testImplementation(projectTests(":kotlin-build-statistic"))
}
configurations.commonCompileClasspath.get().exclude("org.jetbrains.kotlinx", "kotlinx-coroutines-core")
@@ -15,6 +15,7 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.gradle.logging.*
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.build.report.statistic.StatTag
import org.jetbrains.kotlin.gradle.report.*
import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState
@@ -26,6 +27,7 @@ import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
import org.jetbrains.kotlin.incremental.util.ExceptionLocation
import org.jetbrains.kotlin.incremental.util.reportException
import org.jetbrains.kotlin.util.removeSuffixIfPresent
import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
import org.slf4j.LoggerFactory
import java.io.*
import java.net.URLClassLoader
@@ -34,6 +36,7 @@ import java.util.*
import java.util.concurrent.Callable
import java.util.concurrent.Executors
import javax.inject.Inject
import kotlin.collections.HashSet
internal class ProjectFilesForCompilation(
val projectRootFile: File,
@@ -41,10 +44,9 @@ internal class ProjectFilesForCompilation(
val sessionFlagFile: File,
val buildDir: File
) : Serializable {
//TODO
constructor(logger: Logger, projectDir:File, buildDir: File, prjectName: String, projectCacheDirProvider: File, sessionDir: File) : this(
constructor(logger: Logger, projectDir:File, buildDir: File, projectName: String, projectCacheDirProvider: File, sessionDir: File) : this(
projectRootFile = projectDir,
clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(logger, prjectName),
clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(logger, projectName),
sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(logger, sessionDir, projectCacheDirProvider),
buildDir = buildDir
)
@@ -114,7 +116,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
TaskLoggers.get(taskPath)?.let { GradleKotlinLogger(it).apply { debug("Using '$taskPath' logger") } }
?: run {
val logger = LoggerFactory.getLogger("GradleKotlinCompilerWork")
val kotlinLogger = if (logger is org.gradle.api.logging.Logger) {
val kotlinLogger = if (logger is Logger) {
GradleKotlinLogger(logger)
} else SL4JKotlinLogger(logger)
@@ -144,8 +146,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
kotlinLanguageVersion = kotlinLanguageVersion,
changedFiles = incrementalCompilationEnvironment?.changedFiles,
compilerArguments = if (reportingSettings.includeCompilerArguments) compilerArgs else emptyArray(),
withAbiSnapshot = incrementalCompilationEnvironment?.withAbiSnapshot,
withArtifactTransform = incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled
tags = collectStatTags(),
)
metrics.endMeasure(BuildTime.RUN_COMPILATION_IN_WORKER)
val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines, taskInfo = taskInfo)
@@ -153,6 +154,15 @@ internal class GradleKotlinCompilerWork @Inject constructor(
}
}
private fun collectStatTags(): Set<StatTag> {
val statTags = HashSet<StatTag>()
incrementalCompilationEnvironment?.withAbiSnapshot?.ifTrue { statTags.add(StatTag.ABI_SNAPSHOT) }
if (incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
statTags.add(StatTag.ARTIFACT_TRANSFORM)
}
return statTags
}
private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): Pair<ExitCode, KotlinCompilerExecutionStrategy> {
with(log) {
kotlinDebug { "Kotlin compiler class: $compilerClassName" }
@@ -12,8 +12,6 @@ import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.gradle.utils.projectCacheDir
import java.io.File
@@ -65,9 +63,6 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
override fun close() {
buildHandler.buildFinished(parameters.projectCacheDir)
log.kotlinDebug(DISPOSE_MESSAGE)
TaskLoggers.clear()
TaskExecutionResults.clear()
}
companion object {
@@ -98,13 +98,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
kotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, kotlinPluginVersion)
}
BuildMetricsService.registerIfAbsent(project)?.also { buildMetricsService ->
val buildEventsListenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project)
buildEventsListenerRegistryHolder.listenerRegistry.onTaskCompletion(buildMetricsService)
BuildReportsService.registerIfAbsent(project, buildMetricsService).also {
buildEventsListenerRegistryHolder.listenerRegistry.onTaskCompletion(it)
}
}
BuildMetricsService.registerIfAbsent(project)
}
private fun addKotlinCompilerConfiguration(project: Project) {
@@ -1,88 +0,0 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.plugin.stat
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import java.text.SimpleDateFormat
import java.util.*
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
//Sensitive data. This object is used directly for statistic via http
private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC")}
data class CompileStatisticsData(
val version: Int = 2,
val projectName: String?,
val label: String?,
val taskName: String?,
val taskResult: String,
val durationMs: Long,
val tags: List<StatTag>,
val changes: List<String>,
val buildUuid: String = "Unset",
val kotlinVersion: String,
val kotlinLanguageVersion: KotlinVersion?,
val hostName: String? = "Unset",
val finishTime: Long,
val timestamp: String = formatter.format(finishTime),
val compilerArguments: List<String>,
val nonIncrementalAttributes: Set<BuildAttribute>,
//TODO think about it,time in milliseconds
val buildTimesMetrics: Map<BuildTime, Long>,
val performanceMetrics: Map<BuildPerformanceMetric, Long>,
val gcTimeMetrics: Map<String, Long>?,
val gcCountMetrics: Map<String, Long>?,
val type: String = BuildDataType.TASK_DATA.name
)
enum class StatTag(val readableString: String) {
ABI_SNAPSHOT("ABI Snapshot"),
ARTIFACT_TRANSFORM("Classpath Snapshot"),
INCREMENTAL("Incremental compilation"),
NON_INCREMENTAL("Non incremental compilation"),
INCREMENTAL_AND_NON_INCREMENTAL("Incremental and Non incremental compilation"),
GRADLE_DEBUG("Gradle debug enabled"),
KOTLIN_DEBUG("Kotlin debug enabled"),
CONFIGURATION_CACHE("Configuration cache enabled"),
BUILD_CACHE("Build cache enabled"),
KOTLIN_1("Kotlin language version 1"),
KOTLIN_2("Kotlin language version 2"),
KOTLIN_1_AND_2("Kotlin language version 1 and 2"),
}
enum class BuildDataType {
TASK_DATA,
BUILD_DATA
}
//Sensitive data. This object is used directly for statistic via http
data class GradleBuildStartParameters(
val tasks: List<String>,
val excludedTasks: Set<String>,
val currentDir: String?,
val projectProperties: List<String>,
val systemProperties: List<String>,
) : java.io.Serializable
//Sensitive data. This object is used directly for statistic via http
data class BuildFinishStatisticsData(
val projectName: String,
val startParameters: GradleBuildStartParameters,
val buildUuid: String = "Unset",
val label: String?,
val totalTime: Long,
val type: String = BuildDataType.BUILD_DATA.name,
val finishTime: Long,
val timestamp: String = formatter.format(finishTime),
val hostName: String? = "Unset",
val tags: List<StatTag>,
val gitBranch: String = "Unset"
)
@@ -7,14 +7,16 @@ package org.jetbrains.kotlin.gradle.report
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal
import org.gradle.tooling.events.FailureResult
import org.gradle.tooling.events.FinishEvent
import org.gradle.tooling.events.OperationCompletionListener
import org.gradle.tooling.events.task.TaskExecutionResult
import org.gradle.tooling.events.task.TaskFailureResult
@@ -24,13 +26,22 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.statistic.HttpReportService
import org.jetbrains.kotlin.build.report.statistic.HttpReportServiceImpl
import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters
import org.jetbrains.kotlin.build.report.statistic.StatTag
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.report.BuildReportsService.Companion.getStartParameters
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
@@ -40,13 +51,27 @@ internal interface UsesBuildMetricsService : Task {
val buildMetricsService: Property<BuildMetricsService?>
}
abstract class BuildMetricsService : BuildService<BuildServiceParameters.None>, OperationCompletionListener {
abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters>, AutoCloseable {
//Part of BuildReportService
interface Parameters : BuildServiceParameters {
val startParameters: Property<GradleBuildStartParameters>
val reportingSettings: Property<ReportingSettings>
val httpService: Property<HttpReportService>
val projectDir: DirectoryProperty
val label: Property<String?>
val projectName: Property<String>
val kotlinVersion: Property<String>
val buildConfigurationTags: ListProperty<StatTag>
}
private val log = Logging.getLogger(this.javaClass)
private val buildReportService = BuildReportsService()
// Tasks and transforms' records
internal val buildOperationRecords = ConcurrentLinkedQueue<BuildOperationRecord>()
internal val failureMessages = ConcurrentLinkedQueue<String>()
private val buildOperationRecords = ConcurrentLinkedQueue<BuildOperationRecord>()
private val failureMessages = ConcurrentLinkedQueue<String>()
// Info for tasks only
private val taskPathToMetricsReporter = ConcurrentHashMap<String, BuildMetricsReporter>()
@@ -76,69 +101,87 @@ abstract class BuildMetricsService : BuildService<BuildServiceParameters.None>,
failureMessage?.let { failureMessages.add(it) }
}
override fun onFinish(event: FinishEvent?) {
if (event is TaskFinishEvent) {
val result = event.result
val taskPath = event.descriptor.taskPath
val totalTimeMs = result.endTime - result.startTime
private fun updateBuildOperationRecord(event: TaskFinishEvent): TaskRecord {
val result = event.result
val taskPath = event.descriptor.taskPath
val totalTimeMs = result.endTime - result.startTime
val buildMetrics = BuildMetrics()
buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, totalTimeMs)
taskPathToMetricsReporter[taskPath]?.let {
buildMetrics.addAll(it.getMetrics())
}
val taskExecutionResult = TaskExecutionResults[taskPath]
taskExecutionResult?.buildMetrics?.also {
buildMetrics.addAll(it)
val buildMetrics = BuildMetrics()
buildMetrics.buildTimes.addTimeMs(BuildTime.GRADLE_TASK, totalTimeMs)
taskPathToMetricsReporter[taskPath]?.let {
buildMetrics.addAll(it.getMetrics())
}
val taskExecutionResult = TaskExecutionResults[taskPath]
taskExecutionResult?.buildMetrics?.also {
buildMetrics.addAll(it)
KotlinBuildStatsService.applyIfInitialised { collector ->
collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
KotlinBuildStatsService.applyIfInitialised { collector ->
collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
val linesOfCode = metricsMap[BuildPerformanceMetric.ANALYZED_LINES_NUMBER]
if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) {
collector.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode)
collector.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode)
metricsMap[BuildPerformanceMetric.ANALYSIS_LPS]?.also {
collector.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, it, null, linesOfCode)
}
metricsMap[BuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value ->
collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
}
val linesOfCode = metricsMap[BuildPerformanceMetric.ANALYZED_LINES_NUMBER]
if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) {
collector.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode)
collector.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode)
metricsMap[BuildPerformanceMetric.ANALYSIS_LPS]?.also { value ->
collector.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, value, null, linesOfCode)
}
metricsMap[BuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value ->
collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
}
collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
collector.report(
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
)
}
}
buildOperationRecords.add(
TaskRecord(
path = taskPath,
classFqName = taskPathToTaskClass[taskPath] ?: "unknown",
startTimeMs = result.startTime,
totalTimeMs = totalTimeMs,
buildMetrics = buildMetrics,
didWork = result is TaskExecutionResult,
skipMessage = (result as? TaskSkippedResult)?.skipMessage,
icLogLines = taskExecutionResult?.icLogLines ?: emptyList(),
kotlinLanguageVersion = taskExecutionResult?.taskInfo?.kotlinLanguageVersion
collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
collector.report(
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
)
)
if (result is TaskFailureResult) {
failureMessages.addAll(result.failures.map { it.message })
}
}
val buildOperation = TaskRecord(
path = taskPath,
classFqName = taskPathToTaskClass[taskPath] ?: "unknown",
startTimeMs = result.startTime,
totalTimeMs = totalTimeMs,
buildMetrics = buildMetrics,
didWork = result is TaskExecutionResult,
skipMessage = (result as? TaskSkippedResult)?.skipMessage,
icLogLines = taskExecutionResult?.icLogLines ?: emptyList(),
changedFiles = taskExecutionResult?.taskInfo?.changedFiles,
compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments ?: emptyArray(),
kotlinLanguageVersion = taskExecutionResult?.taskInfo?.kotlinLanguageVersion,
statTags = taskExecutionResult?.taskInfo?.tags ?: emptySet()
)
buildOperationRecords.add(buildOperation)
if (result is TaskFailureResult) {
failureMessages.addAll(result.failures.map { it.message })
}
return buildOperation
}
override fun close() {
buildReportService.close(buildOperationRecords, failureMessages.toList(), parameters.toBuildReportParameters())
}
companion object {
private val serviceClass = BuildMetricsService::class.java
private val serviceName = "${serviceClass.name}_${serviceClass.classLoader.hashCode()}"
private fun registerIfAbsentImpl(project: Project): Provider<BuildMetricsService>? {
private fun Parameters.toBuildReportParameters() = BuildReportParameters(
startParameters = startParameters.get(),
reportingSettings = reportingSettings.get(),
httpService = httpService.orNull,
projectDir = projectDir.asFile.get(),
label = label.orNull,
projectName = projectName.get(),
kotlinVersion = kotlinVersion.get(),
additionalTags = HashSet(buildConfigurationTags.get())
)
private fun registerIfAbsentImpl(
project: Project,
): Provider<BuildMetricsService>? {
// Return early if the service was already registered to avoid the overhead of reading the reporting settings below
project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
@Suppress("UNCHECKED_CAST")
@@ -151,7 +194,59 @@ abstract class BuildMetricsService : BuildService<BuildServiceParameters.None>,
return null
}
return project.gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) {}!!
val kotlinVersion = project.getKotlinPluginVersion()
return project.gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) {
it.parameters.label.set(reportingSettings.buildReportLabel)
it.parameters.projectName.set(project.rootProject.name)
it.parameters.kotlinVersion.set(kotlinVersion)
it.parameters.startParameters.set(getStartParameters(project))
it.parameters.reportingSettings.set(reportingSettings)
reportingSettings.httpReportSettings?.let { httpSettings ->
it.parameters.httpService.set(
HttpReportServiceImpl(
httpSettings.url,
httpSettings.user,
httpSettings.password
)
)
}
it.parameters.projectDir.set(project.rootProject.layout.projectDirectory)
//init gradle tags for build scan and http reports
it.parameters.buildConfigurationTags.value(setupTags(project.gradle))
}.also {
subscribeForTaskEvents(project, it)
}
}
private fun subscribeForTaskEvents(project: Project, buildMetricService: Provider<BuildMetricsService>) {
// BuildScanExtension cant be parameter nor BuildService's field
val buildScanExtension = project.rootProject.extensions.findByName("buildScan")
val buildScan = buildScanExtension?.let { BuildScanExtensionHolder(it) }
val buildReportService = buildMetricService.map { it.buildReportService }
BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(project.provider {
OperationCompletionListener { event ->
if (event is TaskFinishEvent) {
val buildOperation = buildMetricService.get().updateBuildOperationRecord(event)
val buildParameters = buildMetricService.get().parameters.toBuildReportParameters()
buildReportService.get().onFinish(event, buildOperation, buildParameters, buildScan)
}
}
})
val buildScanReportSettings = buildMetricService.get().parameters.reportingSettings.orNull?.buildScanReportSettings
if (buildScanReportSettings != null) {
buildScan?.also {
buildReportService.get().initBuildScanTags(
it, buildMetricService.get().parameters.label.orNull
)
it.buildScan.buildFinished {
buildReportService.get().addCollectedTags(buildScan)
}
}
}
}
fun registerIfAbsent(project: Project) = registerIfAbsentImpl(project)?.also { serviceProvider ->
@@ -161,6 +256,17 @@ abstract class BuildMetricsService : BuildService<BuildServiceParameters.None>,
}
}
}
private fun setupTags(gradle: Gradle): ArrayList<StatTag> {
val additionalTags = ArrayList<StatTag>()
if (isConfigurationCacheAvailable(gradle)) {
additionalTags.add(StatTag.CONFIGURATION_CACHE)
}
if (gradle.startParameter.isBuildCacheEnabled) {
additionalTags.add(StatTag.BUILD_CACHE)
}
return additionalTags
}
}
}
@@ -174,7 +280,10 @@ internal class TaskRecord(
override val didWork: Boolean,
override val skipMessage: String?,
override val icLogLines: List<String>,
val kotlinLanguageVersion: KotlinVersion?
val kotlinLanguageVersion: KotlinVersion?,
val changedFiles: ChangedFiles? = null,
val compilerArguments: Array<String> = emptyArray(),
val statTags: Set<StatTag> = emptySet(),
) : BuildOperationRecord {
override val isFromKotlinPlugin: Boolean = classFqName.startsWith("org.jetbrains.kotlin")
}
@@ -6,33 +6,21 @@
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal
import org.gradle.tooling.events.FinishEvent
import org.gradle.tooling.events.OperationCompletionListener
import org.gradle.tooling.events.task.TaskFinishEvent
import org.jetbrains.kotlin.build.report.metrics.ValueType
import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
import org.jetbrains.kotlin.gradle.plugin.stat.BuildFinishStatisticsData
import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters
import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
import org.jetbrains.kotlin.build.report.statistic.HttpReportService
import org.jetbrains.kotlin.build.report.statistic.file.FileReportService
import org.jetbrains.kotlin.build.report.statistic.formatSize
import org.jetbrains.kotlin.build.report.statistic.BuildFinishStatisticsData
import org.jetbrains.kotlin.build.report.statistic.CompileStatisticsData
import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters
import org.jetbrains.kotlin.build.report.statistic.StatTag
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.gradle.utils.formatSize
import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.utils.addToStdlib.measureTimeMillisWithResult
import java.io.File
import java.lang.management.ManagementFactory
import java.net.InetAddress
import java.text.SimpleDateFormat
import java.util.*
@@ -41,18 +29,16 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import kotlin.system.measureTimeMillis
internal interface UsesBuildReportsService : Task {
@get:Internal
val buildReportsService: Property<BuildReportsService?>
}
abstract class BuildReportsService : BuildService<BuildReportsService.Parameters>, AutoCloseable, OperationCompletionListener {
//Because of https://github.com/gradle/gradle/issues/23359 gradle issue, two build services interaction is not reliable at the end of the build
//Switch back to proper BuildService as soon as this issue is fixed
class BuildReportsService {
private val log = Logging.getLogger(this.javaClass)
private val loggerAdapter = GradleLoggerAdapter(log)
private val startTime = System.nanoTime()
private val buildUuid = UUID.randomUUID().toString()
private var executorService: ExecutorService = Executors.newSingleThreadExecutor()
private val executorService: ExecutorService = Executors.newSingleThreadExecutor()
private val tags = LinkedHashSet<StatTag>()
private var customValues = 0 // doesn't need to be thread-safe
@@ -61,33 +47,46 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
log.info("Build report service is registered. Unique build id: $buildUuid")
}
interface Parameters : BuildServiceParameters {
val startParameters: Property<GradleBuildStartParameters>
val reportingSettings: Property<ReportingSettings>
var buildMetricsService: Provider<BuildMetricsService>
val httpService: Property<HttpReportService>
val projectDir: DirectoryProperty
val label: Property<String?>
val projectName: Property<String>
val kotlinVersion: Property<String>
val additionalTags: ListProperty<StatTag>
}
override fun close() {
fun close(
buildOperationRecords: Collection<BuildOperationRecord>,
failureMessages: List<String>,
parameters: BuildReportParameters
) {
val buildData = BuildExecutionData(
startParameters = parameters.startParameters.get(),
failureMessages = parameters.buildMetricsService.orNull?.failureMessages?.toList() ?: emptyList(),
buildOperationRecord = parameters.buildMetricsService.orNull?.buildOperationRecords?.sortedBy { it.startTimeMs } ?: emptyList()
startParameters = parameters.startParameters,
failureMessages = failureMessages,
buildOperationRecord = buildOperationRecords.sortedBy { it.startTimeMs }
)
val reportingSettings = parameters.reportingSettings.get()
val reportingSettings = parameters.reportingSettings
reportingSettings.httpReportSettings?.also {
executorService.submit { reportBuildFinish() } //
executorService.submit { reportBuildFinish(parameters) }
}
reportingSettings.fileReportSettings?.also {
reportBuildStatInFile(it, buildData)
FileReportService.reportBuildStatInFile(
it.buildReportDir,
parameters.projectName,
it.includeMetricsInReport,
buildOperationRecords.mapNotNull {
prepareData(
taskResult = null,
it.path,
it.startTimeMs,
it.totalTimeMs + it.startTimeMs,
parameters.projectName,
buildUuid,
parameters.label,
parameters.kotlinVersion,
it,
onlyKotlinTask = false,
parameters.additionalTags
)
},
parameters.startParameters,
failureMessages.filter { it.isNotEmpty() },
loggerAdapter
)
}
reportingSettings.singleOutputFile?.also { singleOutputFile ->
@@ -98,26 +97,20 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
executorService.shutdown()
}
override fun onFinish(event: FinishEvent?) {
addHttpReport(event)
fun onFinish(
event: TaskFinishEvent, buildOperation: BuildOperationRecord,
parameters: BuildReportParameters, buildScan: BuildScanExtensionHolder?
) {
buildScan?.also { addBuildScanReport(event, buildOperation, parameters, it) }
addHttpReport(event, buildOperation, parameters)
}
private fun reportBuildStatInFile(fileReportSettings: FileReportSettings, buildData: BuildExecutionData) {
val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time)
val reportFile = fileReportSettings.buildReportDir.resolve("${parameters.projectName.get()}-build-$ts.txt")
PlainTextBuildReportWriter(
outputFile = reportFile,
printMetrics = fileReportSettings.includeMetricsInReport
).process(buildData, log)
}
private fun reportBuildFinish() {
val httpReportSettings = parameters.reportingSettings.get().httpReportSettings ?: return
private fun reportBuildFinish(parameters: BuildReportParameters) {
val httpReportSettings = parameters.reportingSettings.httpReportSettings ?: return
val branchName = if (httpReportSettings.includeGitBranchName) {
val process = ProcessBuilder("git", "rev-parse", "--abbrev-ref", "HEAD")
.directory(parameters.projectDir.asFile.get())
.directory(parameters.projectDir)
.start().also {
it.waitFor(5, TimeUnit.SECONDS)
}
@@ -125,19 +118,19 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
} else "is not set"
val buildFinishData = BuildFinishStatisticsData(
projectName = parameters.projectName.get(),
startParameters = parameters.startParameters.get()
.includeVerboseEnvironment(parameters.reportingSettings.get().httpReportSettings?.verboseEnvironment ?: false),
projectName = parameters.projectName,
startParameters = parameters.startParameters
.includeVerboseEnvironment(parameters.reportingSettings.httpReportSettings.verboseEnvironment),
buildUuid = buildUuid,
label = parameters.label.orNull,
label = parameters.label,
totalTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)),
finishTime = System.currentTimeMillis(),
hostName = hostName,
tags = tags.toList(),
tags = tags,
gitBranch = branchName
)
parameters.httpService.orNull?.sendData(buildFinishData, log)
parameters.httpService?.sendData(buildFinishData, loggerAdapter)
}
private fun GradleBuildStartParameters.includeVerboseEnvironment(verboseEnvironment: Boolean): GradleBuildStartParameters {
@@ -154,69 +147,72 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
}
}
private fun addHttpReport(event: FinishEvent?) {
parameters.httpService.orNull?.also { httpService ->
if (event is TaskFinishEvent) {
val data =
prepareData(
event,
parameters.projectName.get(),
buildUuid,
parameters.label.orNull,
parameters.kotlinVersion.get(),
parameters.buildMetricsService.get().buildOperationRecords,
parameters.additionalTags.get()
)
data?.also {
executorService.submit {
httpService.sendData(data, log)
}
private fun addHttpReport(
event: TaskFinishEvent,
buildOperationRecord: BuildOperationRecord,
parameters: BuildReportParameters
) {
parameters.httpService?.also { httpService ->
val data =
prepareData(
event,
parameters.projectName,
buildUuid,
parameters.label,
parameters.kotlinVersion,
buildOperationRecord,
onlyKotlinTask = true,
parameters.additionalTags
)
data?.also {
executorService.submit {
httpService.sendData(data, loggerAdapter)
}
}
}
}
private fun addBuildScanReport(
event: TaskFinishEvent,
buildOperationRecord: BuildOperationRecord,
parameters: BuildReportParameters,
buildScanExtension: BuildScanExtensionHolder
) {
val buildScanSettings = parameters.reportingSettings.buildScanReportSettings ?: return
private fun addBuildScanReport(event: FinishEvent?, buildScan: BuildScanExtensionHolder) {
val buildScanSettings = parameters.reportingSettings.orNull?.buildScanReportSettings
if (buildScanSettings != null && buildScan.buildScan != null) {
if (event is TaskFinishEvent) {
val (collectDataDuration, compileStatData) = measureTimeMillisWithResult {
prepareData(
event, parameters.projectName.get(), buildUuid, parameters.label.orNull,
parameters.kotlinVersion.get(),
parameters.buildMetricsService.get().buildOperationRecords,
metricsToShow = buildScanSettings.metrics
)
}
log.debug("Collect data takes $collectDataDuration: $compileStatData")
val (collectDataDuration, compileStatData) = measureTimeMillisWithResult {
prepareData(
event,
parameters.projectName, buildUuid, parameters.label,
parameters.kotlinVersion,
buildOperationRecord,
metricsToShow = buildScanSettings.metrics
)
}
log.debug("Collect data takes $collectDataDuration: $compileStatData")
compileStatData?.also {
addBuildScanReport(it, buildScanSettings.customValueLimit, buildScan)
}
}
compileStatData?.also {
addBuildScanReport(it, buildScanSettings.customValueLimit, buildScanExtension)
}
}
private fun addBuildScanReport(data: CompileStatisticsData, customValuesLimit: Int, buildScan: BuildScanExtensionHolder) {
val elapsedTime = measureTimeMillis {
data.tags.forEach { tags.add(it) }
buildScan.buildScan?.also {
if (customValues < customValuesLimit) {
readableString(data).forEach {
if (customValues < customValuesLimit) {
addBuildScanValue(buildScan, data, it)
} else {
log.debug(
"Can't add any more custom values into build scan." +
" Statistic data for ${data.taskName} was cut due to custom values limit."
)
}
tags.addAll(data.tags)
if (customValues < customValuesLimit) {
readableString(data).forEach {
if (customValues < customValuesLimit) {
addBuildScanValue(buildScan, data, it)
} else {
log.debug(
"Can't add any more custom values into build scan." +
" Statistic data for ${data.taskName} was cut due to custom values limit."
)
}
} else {
log.debug("Can't add any more custom values into build scan.")
}
} else {
log.debug("Can't add any more custom values into build scan.")
}
}
@@ -228,7 +224,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
data: CompileStatisticsData,
customValue: String
) {
buildScan.buildScan?.value(data.taskName, customValue)
buildScan.buildScan.value(data.taskName, customValue)
customValues++
}
@@ -245,7 +241,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
) { it.readableString }
}
data.kotlinLanguageVersion?.version?.also {
data.kotlinLanguageVersion?.also {
readableString.append("Kotlin language version: $it; ")
}
@@ -283,14 +279,18 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
return splattedString
}
private fun initBuildScanTags(buildScan: BuildScanExtensionHolder) {
buildScan.buildScan?.tag(buildUuid)
parameters.label.orNull?.also {
buildScan.buildScan?.tag(it)
internal fun initBuildScanTags(buildScan: BuildScanExtensionHolder, label: String?) {
buildScan.buildScan.tag(buildUuid)
label?.also {
buildScan.buildScan.tag(it)
}
val debugConfiguration = "-agentlib:"
if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) {
buildScan.buildScan.tag(StatTag.GRADLE_DEBUG.readableString)
}
}
private fun addCollectedTags(buildScan: BuildScanExtensionHolder) {
internal fun addCollectedTags(buildScan: BuildScanExtensionHolder) {
replaceWithCombinedTag(
StatTag.KOTLIN_1,
StatTag.KOTLIN_2,
@@ -303,7 +303,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
StatTag.INCREMENTAL_AND_NON_INCREMENTAL
)
tags.forEach { buildScan.buildScan?.tag(it.readableString) }
tags.forEach { buildScan.buildScan.tag(it.readableString) }
}
private fun replaceWithCombinedTag(firstTag: StatTag, secondTag: StatTag, combinedTag: StatTag) {
@@ -331,78 +331,6 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
)
}
private fun registerIfAbsentImpl(
project: Project,
buildMetricsService: Provider<BuildMetricsService>
): Provider<BuildReportsService>? {
val serviceClass = BuildReportsService::class.java
val serviceName = "${serviceClass.name}_${serviceClass.classLoader.hashCode()}"
val reportingSettings = reportingSettings(project)
if (reportingSettings.buildReportOutputs.isEmpty()) {
return null //no need to collect data
}
val kotlinVersion = project.getKotlinPluginVersion()
val gradle = project.gradle
project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
@Suppress("UNCHECKED_CAST")
return it.service as Provider<BuildReportsService>
}
return gradle.sharedServices.registerIfAbsent(serviceName, serviceClass) {
it.parameters.label.set(reportingSettings.buildReportLabel)
it.parameters.projectName.set(project.rootProject.name)
it.parameters.kotlinVersion.set(kotlinVersion)
it.parameters.startParameters.set(getStartParameters(project))
it.parameters.reportingSettings.set(reportingSettings)
reportingSettings.httpReportSettings?.let { httpSettings -> it.parameters.httpService.set(HttpReportServiceImpl(httpSettings)) }
it.parameters.buildMetricsService = buildMetricsService
it.parameters.projectDir.set(project.rootProject.layout.projectDirectory)
//init gradle tags for build scan and http reports
it.parameters.additionalTags.value(setupTags(gradle))
}.also { buildServiceProvider ->
if (reportingSettings.httpReportSettings != null) {
BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildServiceProvider)
}
val buildScanExtension = project.rootProject.extensions.findByName("buildScan")
if (reportingSettings.buildScanReportSettings != null && buildScanExtension != null) {
val buildScan = BuildScanExtensionHolder(buildScanExtension)
buildServiceProvider.get().initBuildScanTags(buildScan)
BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(project.provider {
OperationCompletionListener { event ->
buildServiceProvider.get().addBuildScanReport(event, buildScan)
}
})
buildScan.buildScan?.buildFinished {
buildServiceProvider.get().addCollectedTags(buildScan)
}
}
}
}
fun registerIfAbsent(project: Project, buildMetricsService: Provider<BuildMetricsService>) =
registerIfAbsentImpl(project, buildMetricsService)?.also { serviceProvider ->
SingleActionPerProject.run(project, UsesBuildReportsService::class.java.name) {
project.tasks.withType<UsesBuildReportsService>().configureEach { task ->
task.usesService(serviceProvider)
}
}
}
private fun setupTags(gradle: Gradle): ArrayList<StatTag> {
val additionalTags = ArrayList<StatTag>()
if (isConfigurationCacheAvailable(gradle)) {
additionalTags.add(StatTag.CONFIGURATION_CACHE)
}
if (gradle.startParameter.isBuildCacheEnabled) {
additionalTags.add(StatTag.BUILD_CACHE)
}
return additionalTags
}
val hostName: String? = try {
InetAddress.getLocalHost().hostName
} catch (_: Exception) {
@@ -422,3 +350,15 @@ enum class TaskExecutionState {
UP_TO_DATE
;
}
data class BuildReportParameters(
val startParameters: GradleBuildStartParameters,
val reportingSettings: ReportingSettings,
val httpService: HttpReportService?,
val projectDir: File,
val label: String?,
val projectName: String,
val kotlinVersion: String,
val additionalTags: Set<StatTag>
)
@@ -7,6 +7,6 @@ package org.jetbrains.kotlin.gradle.report
import com.gradle.scan.plugin.BuildScanExtension
class BuildScanExtensionHolder(val buildScan: BuildScanExtension?) : java.io.Serializable {
constructor(extension: Any?) : this(extension as BuildScanExtension)
class BuildScanExtensionHolder(val buildScan: BuildScanExtension) : java.io.Serializable {
constructor(extension: Any) : this(extension as BuildScanExtension)
}
@@ -0,0 +1,31 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.logging.Logger
class GradleLoggerAdapter(private val log: Logger) : org.jetbrains.kotlin.util.Logger {
override fun log(message: String) {
log.info(message)
}
override fun warning(message: String) {
log.warn(message)
}
override fun fatal(message: String): Nothing {
log.error(message)
kotlin.error(message)
}
override fun error(message: String, throwable: Throwable?) {
throwable?.let { log.error(message, throwable) } ?: log.error(message)
}
override fun lifecycle(message: String) {
log.lifecycle(message)
}
}
@@ -1,81 +0,0 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import com.google.gson.Gson
import org.gradle.api.logging.Logger
import java.io.IOException
import java.io.Serializable
import java.net.HttpURLConnection
import java.net.URL
import java.util.*
import kotlin.system.measureTimeMillis
interface HttpReportService {
fun sendData(data: Any, log: Logger)
}
class HttpReportServiceImpl(
private val url: String,
private val password: String?,
private val user: String?,
) : HttpReportService, Serializable {
constructor(httpSettings: HttpReportSettings) : this(httpSettings.url, httpSettings.password, httpSettings.user)
private var invalidUrl = false
private var requestPreviousFailed = false
private fun checkResponseAndLog(connection: HttpURLConnection, log: Logger) {
val isResponseBad = connection.responseCode !in 200..299
if (isResponseBad) {
val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}"
if (!requestPreviousFailed) {
log.warn(message)
} else {
log.debug(message)
}
requestPreviousFailed = true
}
}
override fun sendData(data: Any, log: Logger) {
val elapsedTime = measureTimeMillis {
if (invalidUrl) {
return
}
val connection = try {
URL(url).openConnection() as HttpURLConnection
} catch (e: IOException) {
log.warn("Unable to open connection to ${url}: ${e.message}")
invalidUrl = true
return
}
try {
if (user != null && password != null) {
val auth = Base64.getEncoder()
.encode("${user}:${password}".toByteArray())
.toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
connection.addRequestProperty("Content-Type", "application/json")
connection.requestMethod = "POST"
connection.doOutput = true
connection.outputStream.use {
it.write(Gson().toJson(data).toByteArray())
}
connection.connect()
checkResponseAndLog(connection, log)
} catch (e: Exception) {
log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}")
checkResponseAndLog(connection, log)
} finally {
connection.disconnect()
}
}
log.debug("Report statistic by http takes $elapsedTime ms")
}
}
@@ -1,284 +0,0 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.gradle.utils.Printer
import org.jetbrains.kotlin.gradle.utils.asString
import org.jetbrains.kotlin.gradle.utils.formatSize
import org.jetbrains.kotlin.gradle.utils.formatTime
import java.io.File
import java.io.Serializable
import java.util.*
import kotlin.math.max
internal class PlainTextBuildReportWriter(
private val outputFile: File,
private val printMetrics: Boolean
) : Serializable {
private lateinit var p: Printer
fun process(build: BuildExecutionData, log: Logger) {
val buildReportPath = outputFile.toPath().toUri().toString()
try {
outputFile.parentFile.mkdirs()
if (!(outputFile.parentFile.exists() && outputFile.parentFile.isDirectory)) {
log.error("Kotlin build report cannot be created: '$outputFile.parentFile' is a file or do not have permissions to create")
return
}
outputFile.bufferedWriter().use { writer ->
p = Printer(writer)
printBuildReport(build)
}
log.lifecycle("Kotlin build report is written to $buildReportPath")
} catch (e: Exception) {
log.error("Could not write Kotlin build report to $buildReportPath", e)
}
}
private fun printBuildReport(build: BuildExecutionData) {
// NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms.
// Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound
// a bit unfamiliar.
// TODO: If it is confusing, consider renaming "tasks" to "build operations" in this class.
printBuildInfo(build)
if (printMetrics) {
printMetrics(build.aggregatedMetrics, aggregatedMetric = true)
p.println()
}
printTaskOverview(build)
printTasksLog(build)
}
private fun printBuildInfo(build: BuildExecutionData) {
p.withIndent("Gradle start parameters:") {
build.startParameters.let {
p.println("tasks = ${it.tasks}")
p.println("excluded tasks = ${it.excludedTasks}")
p.println("current dir = ${it.currentDir}")
p.println("project properties args = ${it.projectProperties}")
p.println(" system properties args = ${it.systemProperties}")
}
}
p.println()
if (build.failureMessages.isNotEmpty()) {
p.println("Build failed: ${build.failureMessages}")
p.println()
}
}
private fun printMetrics(buildMetrics: BuildMetrics, aggregatedMetric: Boolean = false) {
printBuildTimes(buildMetrics.buildTimes)
if (aggregatedMetric) p.println()
printBuildPerformanceMetrics(buildMetrics.buildPerformanceMetrics)
if (aggregatedMetric) p.println()
printBuildAttributes(buildMetrics.buildAttributes)
//TODO: KT-57310 Implement build GC metric in
if (!aggregatedMetric) {
printGcMetrics(buildMetrics.gcMetrics)
}
}
private fun printBuildTimes(buildTimes: BuildTimes) {
val buildTimesMs = buildTimes.asMapMs()
if (buildTimesMs.isEmpty()) return
p.println("Time metrics:")
p.withIndent {
val visitedBuildTimes = HashSet<BuildTime>()
fun printBuildTime(buildTime: BuildTime) {
if (!visitedBuildTimes.add(buildTime)) return
val timeMs = buildTimesMs[buildTime]
if (timeMs != null) {
p.println("${buildTime.readableString}: ${formatTime(timeMs)}")
p.withIndent {
BuildTime.children[buildTime]?.forEach { printBuildTime(it) }
}
} else {
//Skip formatting if parent metric does not set
BuildTime.children[buildTime]?.forEach { printBuildTime(it) }
}
}
for (buildTime in BuildTime.values()) {
if (buildTime.parent != null) continue
printBuildTime(buildTime)
}
}
}
private fun printBuildPerformanceMetrics(buildMetrics: BuildPerformanceMetrics) {
val allBuildMetrics = buildMetrics.asMap()
if (allBuildMetrics.isEmpty()) return
p.withIndent("Size metrics:") {
for (metric in BuildPerformanceMetric.values()) {
allBuildMetrics[metric]?.let { printSizeMetric(metric, it) }
}
}
}
private fun printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) {
fun BuildPerformanceMetric.numberOfAncestors(): Int {
var count = 0
var parent: BuildPerformanceMetric? = parent
while (parent != null) {
count++
parent = parent.parent
}
return count
}
val indentLevel = sizeMetric.numberOfAncestors()
repeat(indentLevel) { p.pushIndent() }
when (sizeMetric.type) {
ValueType.BYTES -> p.println("${sizeMetric.readableString}: ${formatSize(value)}")
ValueType.NUMBER -> p.println("${sizeMetric.readableString}: $value")
}
repeat(indentLevel) { p.popIndent() }
}
private fun printBuildAttributes(buildAttributes: BuildAttributes) {
val allAttributes = buildAttributes.asMap()
if (allAttributes.isEmpty()) return
p.withIndent("Build attributes:") {
val attributesByKind = allAttributes.entries.groupBy { it.key.kind }.toSortedMap()
for ((kind, attributesCounts) in attributesByKind) {
printMap(p, kind.name, attributesCounts.map { (k, v) -> k.readableString to v }.toMap())
}
}
}
private fun printGcMetrics(gcMetrics: GcMetrics) {
val allGcMetrics = gcMetrics.asMap()
if (allGcMetrics.isEmpty()) return
p.withIndent("GC metrics:") {
for (gcMetric in allGcMetrics) {
p.println("${gcMetric.key}:")
p.withIndent {
p.println("GC count: ${gcMetric.value.count}")
p.println("GC time: ${formatTime(gcMetric.value.time)}")
}
}
}
}
private fun printTaskOverview(build: BuildExecutionData) {
var allTasksTimeMs = 0L
var kotlinTotalTimeMs = 0L
val kotlinTasks = ArrayList<BuildOperationRecord>()
for (task in build.buildOperationRecord) {
val taskTimeMs = task.totalTimeMs
allTasksTimeMs += taskTimeMs
if (task.isFromKotlinPlugin) {
kotlinTotalTimeMs += taskTimeMs
kotlinTasks.add(task)
}
}
if (kotlinTasks.isEmpty()) {
p.println("No Kotlin task was run")
return
}
val ktTaskPercent = (kotlinTotalTimeMs.toDouble() / allTasksTimeMs * 100).asString(1)
p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)")
val table = TextTable("Time", "% of Kotlin time", "Task")
for (task in kotlinTasks.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) {
val timeMs = task.totalTimeMs
val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1)
table.addRow(formatTime(timeMs), "$percent %", task.path)
}
table.printTo(p)
p.println()
}
private fun printTasksLog(build: BuildExecutionData) {
for (task in build.buildOperationRecord.sortedWith(compareBy({ -it.totalTimeMs }, { it.startTimeMs }))) {
printTaskLog(task)
p.println()
}
}
private fun printTaskLog(task: BuildOperationRecord) {
val skipMessage = task.skipMessage
if (skipMessage != null) {
p.println("Task '${task.path}' was skipped: $skipMessage")
} else {
p.println("Task '${task.path}' finished in ${formatTime(task.totalTimeMs)}")
}
if (task.icLogLines.isNotEmpty()) {
p.withIndent("Compilation log for task '${task.path}':") {
task.icLogLines.forEach { p.println(it) }
}
}
if (printMetrics) {
printMetrics(task.buildMetrics)
}
}
}
private fun printMap(p: Printer, name: String, mapping: Map<String, Int>) {
if (mapping.isEmpty()) return
if (mapping.size == 1) {
p.println("$name: ${mapping.keys.single()}")
return
}
p.withIndent("$name:") {
val sortedEnumMap = mapping.toSortedMap()
for ((k, v) in sortedEnumMap) {
p.println("$k($v)")
}
}
}
private class TextTable(vararg columnNames: String) {
private val rows = ArrayList<List<String>>()
private val columnsCount = columnNames.size
private val maxLengths = IntArray(columnsCount) { columnNames[it].length }
init {
rows.add(columnNames.toList())
}
fun addRow(vararg row: String) {
check(row.size == columnsCount) { "Row size ${row.size} differs from columns count $columnsCount" }
rows.add(row.toList())
for ((i, col) in row.withIndex()) {
maxLengths[i] = max(maxLengths[i], col.length)
}
}
fun printTo(p: Printer) {
for (row in rows) {
val rowStr = row.withIndex().joinToString("|") { (i, col) -> col.padEnd(maxLengths[i], ' ') }
p.println(rowStr)
}
}
}
@@ -5,6 +5,8 @@
package org.jetbrains.kotlin.gradle.report
import org.jetbrains.kotlin.build.report.FileReportSettings
import org.jetbrains.kotlin.build.report.HttpReportSettings
import java.io.File
import java.io.Serializable
@@ -23,27 +25,6 @@ data class ReportingSettings(
}
}
data class FileReportSettings(
val buildReportDir: File,
val includeMetricsInReport: Boolean = false,
) : Serializable {
companion object {
const val serialVersionUID: Long = 0
}
}
data class HttpReportSettings(
val url: String,
val password: String?,
val user: String?,
val verboseEnvironment: Boolean,
val includeGitBranchName: Boolean
) : Serializable {
companion object {
const val serialVersionUID: Long = 0
}
}
data class BuildScanSettings(
val customValueLimit: Int,
val metrics: Set<String>?
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.report
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.build.report.statistic.StatTag
import org.jetbrains.kotlin.incremental.ChangedFiles
internal class TaskExecutionResult(
@@ -19,6 +20,5 @@ internal class TaskExecutionInfo(
val kotlinLanguageVersion: KotlinVersion? = null,
val changedFiles: ChangedFiles? = null,
val compilerArguments: Array<String> = emptyArray(),
val withArtifactTransform: Boolean? = false,
val withAbiSnapshot: Boolean? = false
val tags: Set<StatTag> = emptySet(),
)
@@ -6,6 +6,8 @@
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.Project
import org.jetbrains.kotlin.build.report.FileReportSettings
import org.jetbrains.kotlin.build.report.HttpReportSettings
import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
@@ -14,7 +16,6 @@ import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLI
import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled
import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
private val availableMetrics = BuildTime.values().map { it.name } + BuildPerformanceMetric.values().map { it.name }
internal fun reportingSettings(project: Project): ReportingSettings {
@@ -6,7 +6,7 @@
package org.jetbrains.kotlin.gradle.report.data
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters
import org.jetbrains.kotlin.build.report.statistic.GradleBuildStartParameters
class BuildExecutionData(
val startParameters: GradleBuildStartParameters,
@@ -10,20 +10,24 @@ import org.gradle.tooling.events.task.TaskFinishEvent
import org.gradle.tooling.events.task.TaskSkippedResult
import org.gradle.tooling.events.task.TaskSuccessResult
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
import org.jetbrains.kotlin.build.report.statistic.CompileStatisticsData
import org.jetbrains.kotlin.build.report.statistic.StatTag
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
import java.lang.management.ManagementFactory
import java.util.ArrayList
import java.util.concurrent.TimeUnit
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
private fun availableForStat(taskPath: String): Boolean {
return taskPath.contains("Kotlin") && (TaskExecutionResults[taskPath] != null)
internal fun getTaskResult(event: TaskFinishEvent) = when (val result = event.result) {
is TaskSuccessResult -> when {
result.isFromCache -> TaskExecutionState.FROM_CACHE
result.isUpToDate -> TaskExecutionState.UP_TO_DATE
else -> TaskExecutionState.SUCCESS
}
is TaskSkippedResult -> TaskExecutionState.SKIPPED
is TaskFailureResult -> TaskExecutionState.FAILED
else -> TaskExecutionState.UNKNOWN
}
internal fun prepareData(
@@ -32,95 +36,112 @@ internal fun prepareData(
uuid: String,
label: String?,
kotlinVersion: String,
buildOperationRecords: Collection<BuildOperationRecord>,
additionalTags: List<StatTag> = emptyList(),
buildOperationRecord: BuildOperationRecord,
onlyKotlinTask: Boolean = true,
additionalTags: Set<StatTag> = emptySet(),
metricsToShow: Set<String>? = null
): CompileStatisticsData? {
val result = event.result
val taskPath = event.descriptor.taskPath
val durationMs = result.endTime - result.startTime
val taskResult = when (result) {
is TaskSuccessResult -> when {
result.isFromCache -> TaskExecutionState.FROM_CACHE
result.isUpToDate -> TaskExecutionState.UP_TO_DATE
else -> TaskExecutionState.SUCCESS
}
return prepareData(getTaskResult(event), taskPath, result.startTime, result.endTime - result.startTime, projectName, uuid,
label, kotlinVersion, buildOperationRecord, onlyKotlinTask, additionalTags, metricsToShow)
}
is TaskSkippedResult -> TaskExecutionState.SKIPPED
is TaskFailureResult -> TaskExecutionState.FAILED
else -> TaskExecutionState.UNKNOWN
}
if (!availableForStat(taskPath)) {
internal fun prepareData(
taskResult: TaskExecutionState?,
taskPath: String,
startTime: Long,
finishTime: Long,
projectName: String,
uuid: String,
label: String?,
kotlinVersion: String,
buildOperationRecord: BuildOperationRecord,
onlyKotlinTask: Boolean = true,
additionalTags: Set<StatTag> = emptySet(),
metricsToShow: Set<String>? = null
): CompileStatisticsData? {
if (onlyKotlinTask && !(buildOperationRecord is TaskRecord && buildOperationRecord.isFromKotlinPlugin)) {
return null
}
val taskExecutionResult = TaskExecutionResults[taskPath]
val buildMetrics = buildOperationRecords.firstOrNull { it.path == taskPath }?.buildMetrics
val buildMetrics = buildOperationRecord.buildMetrics
val performanceMetrics = collectBuildPerformanceMetrics(taskExecutionResult, buildMetrics)
val performanceMetrics = collectBuildPerformanceMetrics(buildMetrics)
val buildTimesMetrics = collectBuildMetrics(
taskExecutionResult, buildMetrics, performanceMetrics, result.startTime,
System.currentTimeMillis()
buildMetrics, startTime, System.currentTimeMillis()
)
val buildAttributes = collectBuildAttributes(taskExecutionResult, buildMetrics)
val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
else -> emptyList<String>()
val buildAttributes = collectBuildAttributes(buildMetrics)
val changes = if (buildOperationRecord is TaskRecord && buildOperationRecord.changedFiles is ChangedFiles.Known) {
buildOperationRecord.changedFiles.modified.map { it.absolutePath } + buildOperationRecord.changedFiles.removed.map { it.absolutePath }
} else {
emptyList<String>()
}
val kotlinLanguageVersion = if (buildOperationRecord is TaskRecord) buildOperationRecord.kotlinLanguageVersion else null
return CompileStatisticsData(
durationMs = durationMs,
taskResult = taskResult.name,
durationMs = buildOperationRecord.totalTimeMs,
taskResult = taskResult?.name,
label = label,
buildTimesMetrics = filterMetrics(metricsToShow, buildTimesMetrics),
performanceMetrics = filterMetrics(metricsToShow, performanceMetrics),
projectName = projectName,
taskName = taskPath,
changes = changes,
tags = collectTags(taskExecutionResult, buildMetrics, additionalTags),
tags = collectTags(buildOperationRecord, additionalTags),
nonIncrementalAttributes = buildAttributes,
hostName = BuildReportsService.hostName,
kotlinVersion = kotlinVersion,
kotlinLanguageVersion = taskExecutionResult?.taskInfo?.kotlinLanguageVersion,
kotlinLanguageVersion = kotlinLanguageVersion?.version,
buildUuid = uuid,
finishTime = System.currentTimeMillis(),
compilerArguments = taskExecutionResult?.taskInfo?.compilerArguments?.asList() ?: emptyList(),
gcCountMetrics = buildMetrics?.gcMetrics?.asGcCountMap(),
gcTimeMetrics = buildMetrics?.gcMetrics?.asGcTimeMap()
compilerArguments = collectCompilerArguments(buildOperationRecord),
gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(),
gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(),
finishTime = finishTime,
startTimeMs = startTime,
fromKotlinPlugin = buildOperationRecord.isFromKotlinPlugin,
skipMessage = buildOperationRecord.skipMessage,
icLogLines = buildOperationRecord.icLogLines
)
}
fun collectCompilerArguments(buildOperationRecord: BuildOperationRecord?): List<String> {
return if (buildOperationRecord is TaskRecord) {
buildOperationRecord.compilerArguments.asList()
} else emptyList()
}
private fun <E : Enum<E>> filterMetrics(
expectedMetrics: Set<String>?,
buildTimesMetrics: Map<E, Long>
): Map<E, Long> = expectedMetrics?.let { buildTimesMetrics.filterKeys { metric -> it.contains(metric.name) } } ?: buildTimesMetrics
private fun collectBuildAttributes(taskExecutionResult: TaskExecutionResult?, buildMetrics: BuildMetrics?): Set<BuildAttribute> {
val attributes = HashSet<BuildAttribute>()
buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys?.also { attributes.addAll(it) }
taskExecutionResult?.buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys?.also { attributes.addAll(it) }
return attributes
private fun collectBuildAttributes(buildMetrics: BuildMetrics?): Set<BuildAttribute> {
return buildMetrics?.buildAttributes?.asMap()?.filter { it.value > 0 }?.keys ?: emptySet()
}
private fun collectBuildPerformanceMetrics(
taskExecutionResult: TaskExecutionResult?,
buildMetrics: BuildMetrics?
): Map<BuildPerformanceMetric, Long> {
val taskBuildPerformanceMetrics = HashMap<BuildPerformanceMetric, Long>()
taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.let { taskBuildPerformanceMetrics.putAll(it) }
buildMetrics?.buildPerformanceMetrics?.asMap()?.let { taskBuildPerformanceMetrics.putAll(it) }
return taskBuildPerformanceMetrics.filterValues { value -> value != 0L }
return buildMetrics?.buildPerformanceMetrics?.asMap()
?.filterValues { value -> value != 0L }
?.filterKeys { key ->
key !in listOf(
BuildPerformanceMetric.START_WORKER_EXECUTION,
BuildPerformanceMetric.CALL_WORKER,
BuildPerformanceMetric.CALL_KOTLIN_DAEMON,
BuildPerformanceMetric.START_KOTLIN_DAEMON_EXECUTION
)
}
?: emptyMap()
}
private fun collectBuildMetrics(
taskExecutionResult: TaskExecutionResult?,
buildMetrics: BuildMetrics?,
performanceMetrics: Map<BuildPerformanceMetric, Long>,
gradleTaskStartTime: Long? = null,
taskFinishEventTime: Long? = null,
): Map<BuildTime, Long> {
val taskBuildMetrics = HashMap<BuildTime, Long>()
taskExecutionResult?.buildMetrics?.buildTimes?.asMapMs()?.let { taskBuildMetrics.putAll(it) }
buildMetrics?.buildTimes?.asMapMs()?.let { taskBuildMetrics.putAll(it) }
val taskBuildMetrics = HashMap<BuildTime, Long>(buildMetrics?.buildTimes?.asMapMs())
val performanceMetrics = buildMetrics?.buildPerformanceMetrics?.asMap() ?: emptyMap()
gradleTaskStartTime?.let { startTime ->
performanceMetrics[BuildPerformanceMetric.START_TASK_ACTION_EXECUTION]?.let { actionStartTime ->
taskBuildMetrics.put(BuildTime.GRADLE_TASK_PREPARATION, actionStartTime - startTime)
@@ -141,41 +162,34 @@ private fun collectBuildMetrics(
}
private fun collectTags(
taskExecutionResult: TaskExecutionResult?,
buildMetrics: BuildMetrics?,
additionalTags: List<StatTag>
): List<StatTag> {
val tags = collectTags(taskExecutionResult, additionalTags)
val nonIncrementalAttributes = collectBuildAttributes(taskExecutionResult, buildMetrics)
buildOperation: BuildOperationRecord?,
additionalTags: Set<StatTag>
): Set<StatTag> {
val tags = HashSet(additionalTags)
if (buildOperation is TaskRecord) {
tags.addAll(collectTaskRecordTags(buildOperation))
}
val nonIncrementalAttributes = collectBuildAttributes(buildOperation?.buildMetrics)
if (nonIncrementalAttributes.isEmpty()) {
tags.add(StatTag.INCREMENTAL)
} else {
tags.add(StatTag.NON_INCREMENTAL)
}
return tags
}
private fun collectTags(
taskExecutionResult: TaskExecutionResult?,
additionalTags: List<StatTag>,
): MutableList<StatTag> {
val tags = ArrayList(additionalTags)
val taskInfo = taskExecutionResult?.taskInfo
private fun collectTaskRecordTags(
taskRecord: TaskRecord?,
): Set<StatTag> {
val tags = HashSet<StatTag>()
taskInfo?.withAbiSnapshot?.ifTrue {
tags.add(StatTag.ABI_SNAPSHOT)
}
taskInfo?.withArtifactTransform?.ifTrue {
tags.add(StatTag.ARTIFACT_TRANSFORM)
}
taskInfo?.kotlinLanguageVersion?.also {
taskRecord?.kotlinLanguageVersion?.also {
tags.add(getLanguageVersionTag(it))
}
val debugConfiguration = "-agentlib:"
if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) {
tags.add(StatTag.GRADLE_DEBUG)
}
taskRecord?.statTags?.let { tags.addAll(it) }
return tags
}
@@ -36,7 +36,6 @@ import org.jetbrains.kotlin.gradle.internal.UsesClassLoadersCachingBuildService
import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerArgumentsProducer.CreateCompilerArgumentsContext.Companion.default
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_SUPPRESS_EXPERIMENTAL_IC_OPTIMIZATIONS_WARNING
import org.jetbrains.kotlin.gradle.plugin.UsesBuildFinishedListenerService
import org.jetbrains.kotlin.gradle.plugin.UsesVariantImplementationFactories
@@ -57,7 +56,6 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
) : AbstractKotlinCompileTool<T>(objectFactory),
CompileUsingKotlinDaemonWithNormalization,
UsesBuildMetricsService,
UsesBuildReportsService,
UsesIncrementalModuleInfoBuildService,
UsesCompilerSystemPropertiesService,
UsesVariantImplementationFactories,
@@ -114,7 +112,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
@get:Internal
internal abstract val suppressKotlinOptionsFreeArgsModificationWarning: Property<Boolean>
internal fun reportingSettings() = buildReportsService.orNull?.parameters?.reportingSettings?.orNull ?: ReportingSettings()
internal fun reportingSettings() = buildMetricsService.orNull?.parameters?.reportingSettings?.orNull ?: ReportingSettings()
@get:Internal
protected val multiModuleICSettings: MultiModuleICSettings
@@ -25,7 +25,6 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.associateWithClosure
import org.jetbrains.kotlin.gradle.plugin.mpp.internal
import org.jetbrains.kotlin.gradle.plugin.sources.applyLanguageSettingsToCompilerOptions
import org.jetbrains.kotlin.gradle.report.BuildMetricsService
import org.jetbrains.kotlin.gradle.report.BuildReportsService
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
import org.jetbrains.kotlin.gradle.tasks.KOTLIN_BUILD_DIR_NAME
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
@@ -64,7 +63,6 @@ internal abstract class AbstractKotlinCompileConfig<TASK : AbstractKotlinCompile
val compilerSystemPropertiesService = CompilerSystemPropertiesService.registerIfAbsent(project)
val buildMetricsService = BuildMetricsService.registerIfAbsent(project)
val buildReportsService = buildMetricsService?.let { BuildReportsService.registerIfAbsent(project, buildMetricsService) }
val incrementalModuleInfoProvider =
IncrementalModuleInfoBuildService.registerIfAbsent(project, objectFactory.providerWithLazyConvention {
GradleCompilerRunner.buildModulesInfo(project.gradle)
@@ -84,9 +82,6 @@ internal abstract class AbstractKotlinCompileConfig<TASK : AbstractKotlinCompile
task.localStateDirectories.from(task.taskBuildLocalStateDirectory).disallowChanges()
buildMetricsService?.also { metricsService ->
task.buildMetricsService.value(metricsService).disallowChanges()
buildReportsService?.also { reportsService ->
task.buildReportsService.value(reportsService).disallowChanges()
}
}
task.systemPropertiesService.value(compilerSystemPropertiesService).disallowChanges()
task.incrementalModuleInfoProvider.value(incrementalModuleInfoProvider).disallowChanges()
@@ -1,81 +0,0 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.gradle.utils
import java.io.IOException
private val LINE_SEPARATOR = System.getProperty("line.separator")
internal class Printer(
private val out: Appendable,
private val indentUnit: String = " ",
private var indent: String = ""
) {
private fun append(s: String) {
try {
out.append(s)
} catch (e: IOException) { // Do nothing
}
}
fun println(vararg strings: String) {
this.print(*strings)
printLineSeparator()
}
private fun printLineSeparator() {
append(LINE_SEPARATOR)
}
fun print(vararg strings: String) {
if (strings.isNotEmpty()) {
this.printIndent()
}
this.printWithNoIndent(*strings)
}
private fun printIndent() {
append(indent)
}
private fun printWithNoIndent(vararg strings: String) {
for (s in strings) {
append(s)
}
}
fun pushIndent() {
indent += indentUnit
}
fun popIndent() {
check(indent.length >= indentUnit.length) { "No indentation to pop" }
indent = indent.substring(indentUnit.length)
}
inline fun <T> withIndent(headLine: String? = null, fn: () -> T): T {
if (headLine != null) {
this.println(headLine)
}
pushIndent()
return try {
fn()
} finally {
popIndent()
}
}
}
@@ -115,8 +115,9 @@ internal fun getAllDependencies(dependency: ResolvedDependencyResult): Set<Resol
internal class GradleLoggerAdapter(private val gradleLogger: Logger) : org.jetbrains.kotlin.util.Logger {
override fun log(message: String) = gradleLogger.info(message)
override fun warning(message: String) = gradleLogger.warn(message)
override fun error(message: String) = kotlin.error(message)
override fun error(message: String, throwable: Throwable?) = kotlin.error(message)
override fun fatal(message: String): Nothing = kotlin.error(message)
override fun lifecycle(message: String) = gradleLogger.lifecycle(message)
}
private fun libraryFilter(artifact: ResolvedArtifactResult): Boolean = artifact.file.absolutePath.endsWith(".klib")
@@ -64,21 +64,3 @@ internal fun formatContentLength(bytes: Long): String = when {
}
}
}
internal fun formatTime(ms: Long): String {
val seconds = ms.toDouble() / 1_000
return seconds.asString(2) + " s"
}
private const val kbSize = 1024
private const val mbSize = kbSize * 1024
private const val gbSize = mbSize * 1024
internal fun formatSize(sizeInBytes: Long): String = when {
sizeInBytes / gbSize >= 1 -> "${(sizeInBytes.toDouble() / gbSize).asString(1)} GB"
sizeInBytes / mbSize >= 1 -> "${(sizeInBytes.toDouble() / mbSize).asString(1)} MB"
sizeInBytes / kbSize >= 1 -> "${(sizeInBytes.toDouble() / kbSize).asString(1)} KB"
else -> "$sizeInBytes B"
}
internal fun Double.asString(decPoints: Int): String = "%,.${decPoints}f".format(this)
@@ -42,7 +42,7 @@ fun Project.probeRemoteFileLength(url: String, probingTimeoutMs: Int = 0): Long?
else {
logger.kotlinDebug(::probeRemoteFileLength.name + "($url, $probingTimeoutMs): Failed to obtain content-length during the probing timeout.")
@Suppress("UNCHECKED_CAST")
null as Long?
null
}
} finally {
connection.disconnect()
@@ -11,10 +11,7 @@ import org.gradle.tooling.events.task.TaskFinishEvent
import org.gradle.tooling.events.task.TaskOperationDescriptor
import org.gradle.tooling.events.task.TaskOperationResult
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
import org.jetbrains.kotlin.gradle.report.TaskExecutionInfo
import org.jetbrains.kotlin.gradle.report.TaskExecutionResult
import org.jetbrains.kotlin.build.report.statistic.StatTag
import org.jetbrains.kotlin.gradle.report.TaskRecord
import org.jetbrains.kotlin.gradle.report.prepareData
import org.junit.Ignore
@@ -32,11 +29,6 @@ class ReportDataTest {
@Test
@Suppress("DEPRECATION")
fun testTags() {
TaskExecutionResults[kotlinTaskPath] = TaskExecutionResult(
buildMetrics = BuildMetrics(buildAttributes = BuildAttributes()),
taskInfo = TaskExecutionInfo(kotlinLanguageVersion = KotlinVersion.KOTLIN_1_4),
icLogLines = emptyList(),
)
val buildOperationRecord =
taskRecord(BuildMetrics(buildAttributes = BuildAttributes().also { it.add(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) }))
val statisticData = prepareData(
@@ -45,8 +37,9 @@ class ReportDataTest {
uuid = "uuid",
label = "label",
kotlinVersion = "version",
buildOperationRecords = listOf(buildOperationRecord),
additionalTags = listOf(StatTag.KOTLIN_DEBUG)
onlyKotlinTask = true,
buildOperationRecord = buildOperationRecord,
additionalTags = setOf(StatTag.KOTLIN_DEBUG)
)
assertNotNull(statisticData)
@@ -57,41 +50,34 @@ class ReportDataTest {
private fun taskRecord(buildMetrics: BuildMetrics) = TaskRecord(
path = kotlinTaskPath,
classFqName = "class name",
classFqName = "org.jetbrains.kotlin.TestTask",
startTimeMs = 10,
totalTimeMs = 20,
buildMetrics = buildMetrics,
didWork = true,
skipMessage = null,
icLogLines = emptyList(),
kotlinLanguageVersion = KotlinVersion.KOTLIN_1_8
kotlinLanguageVersion = KotlinVersion.KOTLIN_1_8,
changedFiles = null,
compilerArguments = emptyArray(),
statTags = emptySet()
)
@Test
fun testMetricFilter() {
TaskExecutionResults["testKotlin"] = TaskExecutionResult(
buildMetrics = BuildMetrics(
buildPerformanceMetrics = BuildPerformanceMetrics().also {
it.add(BuildPerformanceMetric.BUNDLE_SIZE)
it.add(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE)
},
buildTimes = BuildTimes().also {
it.addTimeMs(BuildTime.RESTORE_OUTPUT_FROM_BACKUP, 10)
it.addTimeMs(BuildTime.IC_ANALYZE_JAR_FILES, 10)
}
),
taskInfo = TaskExecutionInfo(),
icLogLines = emptyList()
)
val buildOperationRecord = taskRecord(
BuildMetrics(
buildPerformanceMetrics = BuildPerformanceMetrics().also {
it.add(BuildPerformanceMetric.COMPILE_ITERATION)
it.add(BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT)
it.add(BuildPerformanceMetric.BUNDLE_SIZE)
it.add(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE)
},
buildTimes = BuildTimes().also {
it.addTimeMs(BuildTime.STORE_BUILD_INFO, 20)
it.addTimeMs(BuildTime.GRADLE_TASK_ACTION, 100)
it.addTimeMs(BuildTime.RESTORE_OUTPUT_FROM_BACKUP, 10)
it.addTimeMs(BuildTime.IC_ANALYZE_JAR_FILES, 10)
}
)
)
@@ -102,8 +88,9 @@ class ReportDataTest {
uuid = "uuid",
label = "label",
kotlinVersion = "version",
buildOperationRecords = listOf(buildOperationRecord),
additionalTags = listOf(StatTag.KOTLIN_DEBUG),
buildOperationRecord = buildOperationRecord,
onlyKotlinTask = true,
additionalTags = setOf(StatTag.KOTLIN_DEBUG),
metricsToShow = setOf(
BuildPerformanceMetric.BUNDLE_SIZE.name,// from TaskExecutionResult
BuildTime.GRADLE_TASK_ACTION.name,// from buildOperationRecord
@@ -132,19 +119,11 @@ class ReportDataTest {
val startWorker = 110L
val finishGradleTask = System.nanoTime()
TaskExecutionResults["testKotlin"] = TaskExecutionResult(
buildMetrics = BuildMetrics(
buildPerformanceMetrics = BuildPerformanceMetrics().also {
it.add(BuildPerformanceMetric.FINISH_KOTLIN_DAEMON_EXECUTION, System.currentTimeMillis())
it.add(BuildPerformanceMetric.START_WORKER_EXECUTION, TimeUnit.MILLISECONDS.toNanos(startWorker))
}
),
taskInfo = TaskExecutionInfo(),
icLogLines = emptyList()
)
val buildOperationRecord = taskRecord(
BuildMetrics(
buildPerformanceMetrics = BuildPerformanceMetrics().also {
it.add(BuildPerformanceMetric.FINISH_KOTLIN_DAEMON_EXECUTION, System.currentTimeMillis())
it.add(BuildPerformanceMetric.START_WORKER_EXECUTION, TimeUnit.MILLISECONDS.toNanos(startWorker))
it.add(BuildPerformanceMetric.START_TASK_ACTION_EXECUTION, startTaskAction)
it.add(BuildPerformanceMetric.CALL_WORKER, TimeUnit.MILLISECONDS.toNanos(callWorker))
}
@@ -157,8 +136,9 @@ class ReportDataTest {
uuid = "uuid",
label = "label",
kotlinVersion = "version",
buildOperationRecords = listOf(buildOperationRecord),
additionalTags = listOf(StatTag.KOTLIN_DEBUG),
buildOperationRecord = buildOperationRecord,
onlyKotlinTask = true,
additionalTags = setOf(StatTag.KOTLIN_DEBUG),
)
assertNotNull(statisticData)
assertEquals(startTaskAction - startGradleTask, statisticData.buildTimesMetrics[BuildTime.GRADLE_TASK_PREPARATION])