Add IC metrics reporting

This commit is contained in:
Alexey Tsvetkov
2020-03-02 14:26:49 +03:00
committed by Alexander Likhachev
parent 5bde6457b1
commit 36387d97ad
69 changed files with 1396 additions and 432 deletions
@@ -0,0 +1,12 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build
import java.io.Serializable
enum class ExecutionStrategy : Serializable {
DAEMON, IN_PROCESS, OUT_OF_PROCESS
}
@@ -0,0 +1,24 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter
open class BuildReporter(
protected open val icReporter: ICReporter,
protected open val buildMetricsReporter: BuildMetricsReporter
) : ICReporter by icReporter, BuildMetricsReporter by buildMetricsReporter
class RemoteBuildReporter(
override val icReporter: RemoteICReporter,
override val buildMetricsReporter: RemoteBuildMetricsReporter
) : BuildReporter(icReporter, buildMetricsReporter), RemoteReporter {
override fun flush() {
icReporter.flush()
buildMetricsReporter.flush()
}
}
@@ -3,7 +3,7 @@
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental
package org.jetbrains.kotlin.build.report
import org.jetbrains.kotlin.cli.common.ExitCode
import java.io.File
@@ -16,21 +16,6 @@ interface ICReporter {
fun reportMarkDirtyClass(affectedFiles: Iterable<File>, classFqName: String)
fun reportMarkDirtyMember(affectedFiles: Iterable<File>, scope: String, name: String)
fun reportMarkDirty(affectedFiles: Iterable<File>, reason: String)
fun startMeasure(metric: String, startNs: Long)
fun endMeasure(metric: String, endNs: Long)
}
fun <T> ICReporter?.measure(metric: String, fn: () -> T): T {
if (this == null) return fn()
val start = System.nanoTime()
startMeasure(metric, start)
try {
return fn()
} finally {
val end = System.nanoTime()
endMeasure(metric, end)
}
}
@@ -1,9 +1,9 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental
package org.jetbrains.kotlin.build.report
import java.io.File
@@ -33,10 +33,4 @@ abstract class ICReporterBase(private val pathsBase: File? = null) : ICReporter
protected fun File.relativeOrCanonical(): File =
pathsBase?.let { relativeToOrNull(it) } ?: canonicalFile
override fun startMeasure(metric: String, startNs: Long) {
}
override fun endMeasure(metric: String, endNs: Long) {
}
}
@@ -0,0 +1,9 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report
interface RemoteICReporter : ICReporter,
RemoteReporter
@@ -0,0 +1,11 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report
interface RemoteReporter {
fun flush()
}
@@ -0,0 +1,37 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import java.io.Serializable
enum class BuildAttributeKind : Serializable {
REBUILD_REASON;
companion object {
const val serialVersionUID = 0L
}
}
enum class BuildAttribute(val kind: BuildAttributeKind) : Serializable {
NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON),
CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON),
UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON),
JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON),
JAVA_CHANGE_UNEXPECTED_PSI(BuildAttributeKind.REBUILD_REASON),
JAVA_CHANGE_UNKNOWN_QUALIFIER(BuildAttributeKind.REBUILD_REASON),
DEP_CHANGE_REMOVED_ENTRY(BuildAttributeKind.REBUILD_REASON),
DEP_CHANGE_HISTORY_IS_NOT_FOUND(BuildAttributeKind.REBUILD_REASON),
DEP_CHANGE_HISTORY_CANNOT_BE_READ(BuildAttributeKind.REBUILD_REASON),
DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS(BuildAttributeKind.REBUILD_REASON),
DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP(BuildAttributeKind.REBUILD_REASON),
IN_PROCESS_EXECUTION(BuildAttributeKind.REBUILD_REASON),
OUT_OF_PROCESS_EXECUTION(BuildAttributeKind.REBUILD_REASON),
IC_IS_NOT_ENABLED(BuildAttributeKind.REBUILD_REASON);
companion object {
const val serialVersionUID = 0L
}
}
@@ -0,0 +1,30 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import java.io.Serializable
import java.util.*
class BuildAttributes : Serializable {
private val myAttributes =
EnumMap<BuildAttribute, Int>(
BuildAttribute::class.java
)
fun add(attr: BuildAttribute, count: Int = 1) {
myAttributes[attr] = myAttributes.getOrDefault(attr, 0) + count
}
fun addAll(other: BuildAttributes) {
other.myAttributes.forEach { (attr, n) -> add(attr, n) }
}
fun asMap(): Map<BuildAttribute, Int> = myAttributes
companion object {
const val serialVersionUID = 0L
}
}
@@ -0,0 +1,22 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import java.io.Serializable
data class BuildMetrics(
val buildTimes: BuildTimes = BuildTimes(),
val buildAttributes: BuildAttributes = BuildAttributes()
) : Serializable {
fun addAll(other: BuildMetrics) {
buildTimes.addAll(other.buildTimes)
buildAttributes.addAll(other.buildAttributes)
}
companion object {
const val serialVersionUID = 0L
}
}
@@ -0,0 +1,28 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
interface BuildMetricsReporter {
fun startMeasure(metric: BuildTime, startNs: Long)
fun endMeasure(metric: BuildTime, endNs: Long)
fun addAttribute(attribute: BuildAttribute)
fun getMetrics(): BuildMetrics
fun addMetrics(metrics: BuildMetrics?)
}
inline fun <T> BuildMetricsReporter.measure(metric: BuildTime, fn: () -> T): T {
val start = System.nanoTime()
startMeasure(metric, start)
try {
return fn()
} finally {
val end = System.nanoTime()
endMeasure(metric, end)
}
}
@@ -0,0 +1,47 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import java.util.*
class BuildMetricsReporterImpl : BuildMetricsReporter {
private val myBuildTimeStartNs: EnumMap<BuildTime, Long> =
EnumMap(
BuildTime::class.java
)
private val myBuildTimes = BuildTimes()
private val myBuildAttributes = BuildAttributes()
override fun startMeasure(metric: BuildTime, startNs: Long) {
if (metric in myBuildTimeStartNs) {
error("$metric was restarted before it finished")
}
myBuildTimeStartNs[metric] = startNs
}
override fun endMeasure(metric: BuildTime, endNs: Long) {
val startNs = myBuildTimeStartNs.remove(metric) ?: error("$metric finished before it started")
val durationNs = endNs - startNs
myBuildTimes.add(metric, durationNs)
}
override fun addAttribute(attribute: BuildAttribute) {
myBuildAttributes.add(attribute)
}
override fun getMetrics(): BuildMetrics =
BuildMetrics(
buildTimes = myBuildTimes,
buildAttributes = myBuildAttributes
)
override fun addMetrics(metrics: BuildMetrics?) {
if (metrics == null) return
myBuildAttributes.addAll(metrics.buildAttributes)
myBuildTimes.addAll(metrics.buildTimes)
}
}
@@ -0,0 +1,43 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import java.io.Serializable
@Suppress("Reformat")
enum class BuildTime(val parent: BuildTime? = null) : Serializable {
GRADLE_TASK,
CLEAR_OUTPUT(GRADLE_TASK),
BACKUP_OUTPUT(GRADLE_TASK),
RESTORE_OUTPUT_FROM_BACKUP(GRADLE_TASK),
CONNECT_TO_DAEMON(GRADLE_TASK),
CLEAR_JAR_CACHE(GRADLE_TASK),
RUN_COMPILER(GRADLE_TASK),
NON_INCREMENTAL_COMPILATION_IN_PROCESS(RUN_COMPILER),
NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILER),
NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILER),
INCREMENTAL_COMPILATION(RUN_COMPILER),
IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION),
IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET),
IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES),
IC_ANALYZE_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES),
IC_ANALYZE_CHANGES_IN_JAVA_SOURCES(IC_CALCULATE_INITIAL_DIRTY_SET),
IC_ANALYZE_CHANGES_IN_ANDROID_LAYOUTS(IC_CALCULATE_INITIAL_DIRTY_SET),
IC_DETECT_REMOVED_CLASSES(IC_CALCULATE_INITIAL_DIRTY_SET),
CLEAR_OUTPUT_ON_REBUILD(INCREMENTAL_COMPILATION),
IC_UPDATE_CACHES(INCREMENTAL_COMPILATION),
INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION),
NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION),
IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION);
companion object {
const val serialVersionUID = 0L
val children by lazy {
values().filter { it.parent != null }.groupBy { it.parent }
}
}
}
@@ -0,0 +1,29 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import java.io.Serializable
import java.util.*
class BuildTimes : Serializable {
private val myBuildTimes = EnumMap<BuildTime, Long>(BuildTime::class.java)
fun addAll(other: BuildTimes) {
for ((bt, timeNs) in other.myBuildTimes) {
add(bt, timeNs)
}
}
fun add(buildTime: BuildTime, timeNs: Long) {
myBuildTimes[buildTime] = myBuildTimes.getOrDefault(buildTime, 0) + timeNs
}
fun asMap(): Map<BuildTime, Long> = myBuildTimes
companion object {
const val serialVersionUID = 0L
}
}
@@ -0,0 +1,25 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
object DoNothingBuildMetricsReporter : BuildMetricsReporter {
override fun startMeasure(metric: BuildTime, startNs: Long) {
}
override fun endMeasure(metric: BuildTime, endNs: Long) {
}
override fun addAttribute(attribute: BuildAttribute) {
}
override fun getMetrics(): BuildMetrics =
BuildMetrics(
BuildTimes(),
BuildAttributes()
)
override fun addMetrics(metrics: BuildMetrics?) {}
}
@@ -0,0 +1,11 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.build.report.metrics
import org.jetbrains.kotlin.build.report.RemoteReporter
interface RemoteBuildMetricsReporter : BuildMetricsReporter,
RemoteReporter
@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.JvmSourceRoot
import org.jetbrains.kotlin.build.isModuleMappingFile
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
@@ -29,6 +29,7 @@ enum class CompilationResultCategory(val code: Int) {
IC_COMPILE_ITERATION(0),
BUILD_REPORT_LINES(1),
VERBOSE_BUILD_REPORT_LINES(2),
BUILD_METRICS(3)
}
interface CompilationResultsAsync {
@@ -21,6 +21,9 @@ import com.intellij.openapi.util.Disposer
import com.intellij.openapi.vfs.impl.ZipHandler
import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.RemoteBuildReporter
import org.jetbrains.kotlin.build.report.RemoteReporter
import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY
@@ -286,7 +289,7 @@ abstract class CompileServiceImplBase(
createMessageCollector: (ServicesFacadeT, CompilationOptions) -> MessageCollector,
createReporter: (ServicesFacadeT, CompilationOptions) -> DaemonMessageReporter,
createServices: (JpsServicesFacadeT, EventManager, Profiler) -> Services,
getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteICReporter
getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteBuildReporter
) = kotlin.run {
val messageCollector = createMessageCollector(servicesFacade, compilationOptions)
val daemonReporter = createReporter(servicesFacade, compilationOptions)
@@ -514,7 +517,7 @@ abstract class CompileServiceImplBase(
args: K2JSCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: RemoteICReporter
reporter: RemoteBuildReporter
): ExitCode {
val allKotlinFiles = arrayListOf<File>()
val freeArgsWithoutKotlinFiles = arrayListOf<String>()
@@ -554,7 +557,7 @@ abstract class CompileServiceImplBase(
k2jvmArgs: K2JVMCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: RemoteICReporter
reporter: RemoteBuildReporter
): ExitCode {
val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS +
(incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct()
@@ -784,7 +787,7 @@ class CompileServiceImpl(
createMessageCollector = ::CompileServicesFacadeMessageCollector,
createReporter = ::DaemonMessageReporter,
createServices = this::createCompileServices,
getICReporter = { a, b, c -> getICReporter(a, b!!, c)}
getICReporter = { a, b, c -> getBuildReporter(a, b!!, c)}
)
override fun leaseReplSession(
@@ -8,23 +8,19 @@ package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteICReporter
import java.io.File
import java.util.*
import kotlin.collections.ArrayList
// todo: sync BuildReportICReporterAsync
internal class BuildReportICReporter(
private val compilationResults: CompilationResults,
rootDir: File,
private val isVerbose: Boolean = false,
// todo: default value
// todo: sync BuildReportICReporterAsync
private val reportMetrics: Boolean = true
private val isVerbose: Boolean = false
) : ICReporterBase(rootDir), RemoteICReporter {
private val icLogLines = arrayListOf<String>()
private val recompilationReason = HashMap<File, String>()
private val rootMetric = Metric("<root>", 0)
private val metrics = ArrayDeque<Metric>().apply { add(rootMetric) }
override fun report(message: () -> String) {
icLogLines.add(message())
@@ -36,30 +32,6 @@ internal class BuildReportICReporter(
}
}
override fun startMeasure(metric: String, startNs: Long) {
if (!reportMetrics) return
val newMetric = Metric(metric, startNs)
if (metrics.isNotEmpty()) {
metrics.peekLast().children.add(newMetric)
}
metrics.addLast(newMetric)
}
override fun endMeasure(metric: String, endNs: Long) {
if (!reportMetrics) return
while (metrics.isNotEmpty()) {
val lastMetric = metrics.peekLast()
if (lastMetric.name == metric) {
lastMetric.endNs = endNs
break
} else {
metrics.removeLast()
}
}
}
override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection<File>, exitCode: ExitCode) {
if (!incremental) return
@@ -76,25 +48,6 @@ internal class BuildReportICReporter(
}
override fun flush() {
if (reportMetrics) {
icLogLines.add("Performance metrics:")
reportMetric(rootMetric)
}
compilationResults.add(CompilationResultCategory.BUILD_REPORT_LINES.code, icLogLines)
}
private fun reportMetric(metric: Metric, level: Int = 0) {
if (level > 0) {
val timeMs = metric.endNs?.let { (it - metric.startNs) / 1_000_000L }
icLogLines.add(" ".repeat(level) + "{perf_metric:${metric.name}} ${timeMs ?: "<unknown>"} ms")
}
metric.children.forEach { reportMetric(it, level + 1) }
}
}
private class Metric(val name: String, val startNs: Long) {
var endNs: Long? = null
val children = ArrayList<Metric>()
}
@@ -9,7 +9,8 @@ import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
import org.jetbrains.kotlin.daemon.common.CompileIterationResult
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteICReporter
import java.io.File
internal class CompileIterationICReporter(
@@ -5,11 +5,11 @@
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.build.report.RemoteICReporter
import org.jetbrains.kotlin.cli.common.ExitCode
import java.io.File
internal class CompositeICReporter(private val reporters: Iterable<RemoteICReporter>) :
RemoteICReporter {
internal class CompositeICReporter(private val reporters: Iterable<RemoteICReporter>) : RemoteICReporter {
override fun report(message: () -> String) {
reporters.forEach { it.report(message) }
}
@@ -34,14 +34,6 @@ internal class CompositeICReporter(private val reporters: Iterable<RemoteICRepor
reporters.forEach { it.reportMarkDirty(affectedFiles, reason) }
}
override fun startMeasure(metric: String, startNs: Long) {
reporters.forEach { it.startMeasure(metric, startNs) }
}
override fun endMeasure(metric: String, endNs: Long) {
reporters.forEach { it.endMeasure(metric, endNs) }
}
override fun flush() {
reporters.forEach { it.flush() }
}
@@ -7,7 +7,8 @@ package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteICReporter
import java.io.File
internal class DebugMessagesICReporter(
@@ -0,0 +1,27 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
class RemoteBuildMetricsReporterAdapter(
private val delegate: BuildMetricsReporter,
private val shouldReport: Boolean,
private val compilationResults: CompilationResults
) :
BuildMetricsReporter by delegate,
RemoteBuildMetricsReporter {
override fun flush() {
if (shouldReport) {
val metrics = delegate.getMetrics()
compilationResults.add(CompilationResultCategory.BUILD_METRICS.code, metrics)
}
}
}
@@ -1,12 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.incremental.ICReporter
interface RemoteICReporter : ICReporter {
fun flush()
}
@@ -9,8 +9,14 @@ import kotlinx.coroutines.*
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.report.CompositeICReporter
import org.jetbrains.kotlin.daemon.report.RemoteICReporter
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteBuildReporter
import org.jetbrains.kotlin.build.report.RemoteICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter
import org.jetbrains.kotlin.daemon.report.BuildReportICReporter
import org.jetbrains.kotlin.daemon.report.CompileIterationICReporter
import java.io.File
internal class DebugMessagesICReporterAsync(
@@ -106,7 +112,7 @@ fun getICReporterAsync(
servicesFacade: CompilerServicesFacadeBaseAsync,
compilationResults: CompilationResultsAsync?,
compilationOptions: IncrementalCompilationOptions
): RemoteICReporter {
): RemoteBuildReporter {
val root = compilationOptions.modulesInfo.projectRoot
val reporters = ArrayList<RemoteICReporter>()
@@ -120,19 +126,26 @@ fun getICReporterAsync(
.mapNotNullTo(HashSet()) { resultCode ->
CompilationResultCategory.values().getOrNull(resultCode)
}
requestedResults.mapTo(reporters) { requestedResult ->
for (requestedResult in requestedResults) {
when (requestedResult) {
CompilationResultCategory.IC_COMPILE_ITERATION -> {
CompileIterationICReporterAsync(compilationResults)
reporters.add(CompileIterationICReporterAsync(compilationResults))
}
CompilationResultCategory.BUILD_REPORT_LINES -> {
BuildReportICReporterAsync(compilationResults, root)
reporters.add(BuildReportICReporterAsync(compilationResults, root))
}
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> {
BuildReportICReporterAsync(compilationResults, root, isVerbose = true)
reporters.add(BuildReportICReporterAsync(compilationResults, root, isVerbose = true))
}
}
}
return CompositeICReporter(reporters)
val icReporter = CompositeICReporter(reporters)
val metricsReporter = DoNothingRemoteBuildMetricsReporter
return RemoteBuildReporter(icReporter, metricsReporter)
}
object DoNothingRemoteBuildMetricsReporter : BuildMetricsReporter by DoNothingBuildMetricsReporter, RemoteBuildMetricsReporter {
override fun flush() {
}
}
@@ -16,15 +16,18 @@
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.build.report.RemoteBuildReporter
import org.jetbrains.kotlin.build.report.RemoteICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.daemon.common.*
import java.io.File
import java.util.*
fun getICReporter(
fun getBuildReporter(
servicesFacade: CompilerServicesFacadeBase,
compilationResults: CompilationResults,
compilationOptions: IncrementalCompilationOptions
): RemoteICReporter {
): RemoteBuildReporter {
val root = compilationOptions.modulesInfo.projectRoot
val reporters = ArrayList<RemoteICReporter>()
@@ -38,21 +41,25 @@ fun getICReporter(
.mapNotNullTo(HashSet()) { resultCode ->
CompilationResultCategory.values().getOrNull(resultCode)
}
requestedResults.mapTo(reporters) { requestedResult ->
for (requestedResult in requestedResults) {
when (requestedResult) {
CompilationResultCategory.IC_COMPILE_ITERATION -> {
CompileIterationICReporter(compilationResults)
reporters.add(CompileIterationICReporter(compilationResults))
}
CompilationResultCategory.BUILD_REPORT_LINES -> {
BuildReportICReporter(compilationResults, root)
reporters.add(BuildReportICReporter(compilationResults, root))
}
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> {
BuildReportICReporter(compilationResults, root, isVerbose = true)
reporters.add(BuildReportICReporter(compilationResults, root, isVerbose = true))
}
}
}
val areBuildMetricsNeeded = CompilationResultCategory.BUILD_METRICS in requestedResults
val metricsReporter =
(if (areBuildMetricsNeeded) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter)
.let { RemoteBuildMetricsReporterAdapter(it, areBuildMetricsNeeded, compilationResults) }
return CompositeICReporter(reporters)
return RemoteBuildReporter(CompositeICReporter(reporters), metricsReporter)
}
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.name.FqName
import java.io.File
import java.io.IOException
@@ -19,6 +19,8 @@ package org.jetbrains.kotlin.incremental
import com.intellij.psi.PsiClass
import com.intellij.psi.PsiFile
import com.intellij.psi.PsiJavaFile
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import java.io.File
import java.util.*
@@ -37,7 +39,7 @@ internal class ChangedJavaFilesProcessor(
if (removedJava.any()) {
reporter.report { "Some java files are removed: [${removedJava.joinToString()}]" }
return ChangesEither.Unknown()
return ChangesEither.Unknown(BuildAttribute.JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED)
}
val symbols = HashSet<LookupSymbol>()
@@ -47,7 +49,7 @@ internal class ChangedJavaFilesProcessor(
val psiFile = psiFileFactory(javaFile)
if (psiFile !is PsiJavaFile) {
reporter.report { "Expected PsiJavaFile, got ${psiFile?.javaClass}" }
return ChangesEither.Unknown()
return ChangesEither.Unknown(BuildAttribute.JAVA_CHANGE_UNEXPECTED_PSI)
}
psiFile.classes.forEach { it.addLookupSymbols(symbols) }
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.name.FqName
internal sealed class ChangesEither {
@@ -24,5 +25,5 @@ internal sealed class ChangesEither {
val fqNames: Collection<FqName> = emptyList()
) : ChangesEither()
internal class Unknown(val reason: String? = null) : ChangesEither()
internal class Unknown(val reason: BuildAttribute) : ChangesEither()
}
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.name.FqName
import java.io.File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
@@ -18,6 +18,10 @@ package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -41,7 +45,7 @@ abstract class IncrementalCompilerRunner<
>(
private val workingDir: File,
cacheDirName: String,
protected val reporter: ICReporter,
protected val reporter: BuildReporter,
private val buildHistoryFile: File,
// there might be some additional output directories (e.g. for generated java in kapt)
// to remove them correctly on rebuild, we pass them as additional argument
@@ -65,20 +69,33 @@ abstract class IncrementalCompilerRunner<
// otherwise we track source files changes ourselves.
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION) {
compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir)
}
private fun compileImpl(
allSourceFiles: List<File>,
args: Args,
messageCollector: MessageCollector,
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode {
assert(isICEnabled()) { "Incremental compilation is not enabled" }
var caches = createCacheManager(args, projectDir)
fun rebuild(reason: () -> String): ExitCode {
reporter.report(reason)
fun rebuild(reason: BuildAttribute): ExitCode {
reporter.report { "Non-incremental compilation will be performed: $reason" }
caches.close(false)
clearLocalStateOnRebuild(args)
// todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
clearLocalStateOnRebuild(args)
}
caches = createCacheManager(args, projectDir)
if (providedChangedFiles == null) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
}
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(), messageCollector)
return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector)
}
return try {
@@ -90,16 +107,17 @@ abstract class IncrementalCompilerRunner<
compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector)
}
is CompilationMode.Rebuild -> {
rebuild { "Non-incremental compilation will be performed: ${compilationMode.reason}" }
rebuild(compilationMode.reason)
}
}
if (!caches.close(flush = true)) throw RuntimeException("Could not flush caches")
return exitCode
} catch (e: Exception) {
} catch (e: Exception) { // todo: catch only cache corruption
// todo: warn?
rebuild { "Possible cache corruption. Rebuilding. $e" }
reporter.report { "Rebuilding because of possible caches corruption: $e" }
rebuild(BuildAttribute.CACHE_CORRUPTION)
}
}
@@ -139,10 +157,17 @@ abstract class IncrementalCompilerRunner<
): CompilationMode =
when (changedFiles) {
is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector)
is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" }
is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
}
protected abstract fun calculateSourcesToCompile(
private fun calculateSourcesToCompile(
caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector
): CompilationMode =
reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector)
}
protected abstract fun calculateSourcesToCompileImpl(
caches: CacheManager,
changedFiles: ChangedFiles.Known,
args: Args,
@@ -161,9 +186,7 @@ abstract class IncrementalCompilerRunner<
protected sealed class CompilationMode {
class Incremental(val dirtyFiles: DirtyFilesContainer) : CompilationMode()
class Rebuild(getReason: () -> String = { "" }) : CompilationMode() {
val reason: String by lazy(getReason)
}
class Rebuild(val reason: BuildAttribute) : CompilationMode()
}
protected abstract fun updateCaches(
@@ -211,9 +234,17 @@ abstract class IncrementalCompilerRunner<
): ExitCode {
preBuildHook(args, compilationMode)
val buildTimeMode: BuildTime
val dirtySources = when (compilationMode) {
is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableList()
is CompilationMode.Rebuild -> allKotlinSources.toMutableList()
is CompilationMode.Incremental -> {
buildTimeMode = BuildTime.INCREMENTAL_ITERATION
compilationMode.dirtyFiles.toMutableList()
}
is CompilationMode.Rebuild -> {
buildTimeMode = BuildTime.NON_INCREMENTAL_ITERATION
reporter.addAttribute(compilationMode.reason)
allKotlinSources.toMutableList()
}
}
val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
@@ -247,7 +278,9 @@ abstract class IncrementalCompilerRunner<
val bufferingMessageCollector = BufferingMessageCollector()
val messageCollectorAdapter = MessageCollectorToOutputItemsCollectorAdapter(bufferingMessageCollector, outputItemsCollector)
exitCode = runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter)
exitCode = reporter.measure(buildTimeMode) {
runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter)
}
val generatedFiles = outputItemsCollector.outputs.map(SimpleOutputItem::toGeneratedFile)
if (compilationMode is CompilationMode.Incremental) {
@@ -268,12 +301,13 @@ abstract class IncrementalCompilerRunner<
dirtySourcesSinceLastTimeFile.delete()
caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
val changesCollector = ChangesCollector()
updateCaches(services, caches, generatedFiles, changesCollector)
reporter.measure(BuildTime.IC_UPDATE_CACHES) {
caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
updateCaches(services, caches, generatedFiles, changesCollector)
}
if (compilationMode is CompilationMode.Rebuild) break
val (dirtyLookupSymbols, dirtyClassFqNames) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
@@ -339,7 +373,7 @@ abstract class IncrementalCompilerRunner<
compilationMode: CompilationMode,
currentBuildInfo: BuildInfo,
dirtyData: DirtyData
) {
) = reporter.measure(BuildTime.IC_WRITE_HISTORY_FILE) {
val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList()
val newDiff = if (compilationMode is CompilationMode.Incremental) {
BuildDifference(currentBuildInfo.startTS, true, dirtyData)
@@ -17,6 +17,10 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.isIrBackendEnabled
@@ -45,9 +49,10 @@ fun makeJsIncrementally(
.filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList()
val buildHistoryFile = File(cachesDir, "build-history.bin")
val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
withJsIC {
val compiler = IncrementalJsCompilerRunner(
cachesDir, reporter,
cachesDir, buildReporter,
buildHistoryFile = buildHistoryFile,
modulesApiHistory = EmptyModulesApiHistory,
scopeExpansion = scopeExpansion
@@ -69,7 +74,7 @@ inline fun <R> withJsIC(fn: () -> R): R {
class IncrementalJsCompilerRunner(
private val workingDir: File,
reporter: ICReporter,
reporter: BuildReporter,
buildHistoryFile: File,
private val modulesApiHistory: ModulesApiHistory,
private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER
@@ -90,14 +95,14 @@ class IncrementalJsCompilerRunner(
override fun destinationDir(args: K2JSCompilerArguments): File =
File(args.outputFile).parentFile
override fun calculateSourcesToCompile(
override fun calculateSourcesToCompileImpl(
caches: IncrementalJsCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JSCompilerArguments,
messageCollector: MessageCollector
): CompilationMode {
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
?: return CompilationMode.Rebuild { "No information on previous build" }
?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
initDirtyFiles(dirtyFiles, changedFiles)
@@ -107,9 +112,9 @@ class IncrementalJsCompilerRunner(
@Suppress("UNUSED_VARIABLE") // for sealed when
val unused = when (classpathChanges) {
is ChangesEither.Unknown -> return CompilationMode.Rebuild {
// todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
"Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}"
is ChangesEither.Unknown -> {
reporter.report { "Could not get classpath's changes: ${classpathChanges.reason}" }
return CompilationMode.Rebuild(classpathChanges.reason)
}
is ChangesEither.Known -> {
dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols)
@@ -117,7 +122,6 @@ class IncrementalJsCompilerRunner(
}
}
val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)
dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNames)
@@ -25,6 +25,10 @@ import com.intellij.psi.PsiJavaFile
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
@@ -64,11 +68,12 @@ fun makeIncrementally(
val sourceFiles = files.filter { it.extension.toLowerCase() in allExtensions }.toList()
val buildHistoryFile = File(cachesDir, "build-history.bin")
args.javaSourceRoots = sourceRoots.map { it.absolutePath }.toTypedArray()
val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
withIC {
val compiler = IncrementalJvmCompilerRunner(
cachesDir,
reporter,
buildReporter,
// Use precise setting in case of non-Gradle build
usePreciseJavaTracking = true,
outputFiles = emptyList(),
@@ -104,7 +109,7 @@ inline fun <R> withIC(enabled: Boolean = true, fn: () -> R): R {
class IncrementalJvmCompilerRunner(
workingDir: File,
reporter: ICReporter,
reporter: BuildReporter,
private val usePreciseJavaTracking: Boolean,
buildHistoryFile: File,
outputFiles: Collection<File>,
@@ -154,7 +159,7 @@ class IncrementalJvmCompilerRunner(
else
null
override fun calculateSourcesToCompile(
override fun calculateSourcesToCompileImpl(
caches: IncrementalJvmCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JVMCompilerArguments,
@@ -176,16 +181,20 @@ class IncrementalJvmCompilerRunner(
val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
initDirtyFiles(dirtyFiles, changedFiles)
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild { "No information on previous build" }
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
val classpathChanges = getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
val classpathChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
}
@Suppress("UNUSED_VARIABLE") // for sealed when
val unused = when (classpathChanges) {
is ChangesEither.Unknown -> return CompilationMode.Rebuild {
// todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
"Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}"
is ChangesEither.Unknown -> {
reporter.report {
"Could not get classpath's changes: ${classpathChanges.reason}"
}
return CompilationMode.Rebuild(classpathChanges.reason)
}
is ChangesEither.Known -> {
dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols)
@@ -194,21 +203,26 @@ class IncrementalJvmCompilerRunner(
}
}
if (!usePreciseJavaTracking) {
val javaFilesChanges = javaFilesProcessor!!.process(changedFiles)
val affectedJavaSymbols = when (javaFilesChanges) {
is ChangesEither.Known -> javaFilesChanges.lookupSymbols
is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" }
}
dirtyFiles.addByDirtySymbols(affectedJavaSymbols)
} else {
if (!processChangedJava(changedFiles, caches)) {
return CompilationMode.Rebuild { "Could not get changes for java files" }
reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_JAVA_SOURCES) {
if (!usePreciseJavaTracking) {
val javaFilesChanges = javaFilesProcessor!!.process(changedFiles)
val affectedJavaSymbols = when (javaFilesChanges) {
is ChangesEither.Known -> javaFilesChanges.lookupSymbols
is ChangesEither.Unknown -> return CompilationMode.Rebuild(javaFilesChanges.reason)
}
dirtyFiles.addByDirtySymbols(affectedJavaSymbols)
} else {
val rebuildReason = processChangedJava(changedFiles, caches)
if (rebuildReason != null) return CompilationMode.Rebuild(rebuildReason)
}
}
val androidLayoutChanges = processLookupSymbolsForAndroidLayouts(changedFiles)
val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)
val androidLayoutChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_ANDROID_LAYOUTS) {
processLookupSymbolsForAndroidLayouts(changedFiles)
}
val removedClassesChanges = reporter.measure(BuildTime.IC_DETECT_REMOVED_CLASSES) {
getRemovedClassesChanges(caches, changedFiles)
}
dirtyFiles.addByDirtySymbols(androidLayoutChanges)
dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
@@ -217,7 +231,7 @@ class IncrementalJvmCompilerRunner(
return CompilationMode.Incremental(dirtyFiles)
}
private fun processChangedJava(changedFiles: ChangedFiles.Known, caches: IncrementalJvmCachesManager): Boolean {
private fun processChangedJava(changedFiles: ChangedFiles.Known, caches: IncrementalJvmCachesManager): BuildAttribute? {
val javaFiles = (changedFiles.modified + changedFiles.removed).filter(File::isJavaFile)
for (javaFile in javaFiles) {
@@ -225,20 +239,20 @@ class IncrementalJvmCompilerRunner(
if (!javaFile.exists()) {
// todo: can we do this more optimal?
reporter.report { "Could not get changed for untracked removed java file $javaFile" }
return false
return BuildAttribute.JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED
}
val psiFile = psiFileProvider.javaFile(javaFile)
if (psiFile !is PsiJavaFile) {
reporter.report { "[Precise Java tracking] Expected PsiJavaFile, got ${psiFile?.javaClass}" }
return false
return BuildAttribute.JAVA_CHANGE_UNEXPECTED_PSI
}
for (psiClass in psiFile.classes) {
val qualifiedName = psiClass.qualifiedName
if (qualifiedName == null) {
reporter.report { "[Precise Java tracking] Class with unknown qualified name in $javaFile" }
return false
return BuildAttribute.JAVA_CHANGE_UNKNOWN_QUALIFIER
}
processChangedUntrackedJavaClass(psiClass, ClassId.topLevel(FqName(qualifiedName)))
@@ -247,7 +261,7 @@ class IncrementalJvmCompilerRunner(
}
caches.platformCache.markDirty(javaFiles)
return true
return null
}
private fun processChangedUntrackedJavaClass(psiClass: PsiClass, classId: ClassId) {
@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.incremental
import com.intellij.util.containers.MultiMap
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.SourceToOutputFilesMap
@@ -5,6 +5,10 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
import org.jetbrains.kotlin.incremental.util.Either
import org.jetbrains.kotlin.name.FqName
@@ -15,7 +19,7 @@ internal fun getClasspathChanges(
changedFiles: ChangedFiles.Known,
lastBuildInfo: BuildInfo,
modulesApiHistory: ModulesApiHistory,
reporter: ICReporter?
reporter: BuildReporter
): ChangesEither {
val classpathSet = HashSet<File>()
for (file in classpath) {
@@ -29,7 +33,10 @@ internal fun getClasspathChanges(
val removedClasspath = changedFiles.removed.filterTo(HashSet()) { it in classpathSet }
// todo: removed classes could be processed normally
if (removedClasspath.isNotEmpty()) return ChangesEither.Unknown("Some files are removed from classpath $removedClasspath")
if (removedClasspath.isNotEmpty()) {
reporter.report { "Some files are removed from classpath: $removedClasspath" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_REMOVED_ENTRY)
}
if (modifiedClasspath.isEmpty()) return ChangesEither.Known()
@@ -38,28 +45,47 @@ internal fun getClasspathChanges(
val symbols = HashSet<LookupSymbol>()
val fqNames = HashSet<FqName>()
val historyFilesEither = modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
val historyFilesEither =
reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) {
modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
}
val historyFiles = when (historyFilesEither) {
is Either.Success<Set<File>> -> historyFilesEither.value
is Either.Error -> return ChangesEither.Unknown(historyFilesEither.reason)
}
for (historyFile in historyFiles) {
val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
?: return ChangesEither.Unknown("Could not read diffs from $historyFile")
val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
if (knownBuilds.isEmpty()) {
return ChangesEither.Unknown("No previously known builds for $historyFile")
}
for (buildDiff in newBuilds) {
if (!buildDiff.isIncremental) return ChangesEither.Unknown("Non-incremental build from dependency $historyFile")
val dirtyData = buildDiff.dirtyData
symbols.addAll(dirtyData.dirtyLookupSymbols)
fqNames.addAll(dirtyData.dirtyClassesFqNames)
is Either.Error -> {
reporter.report { "Could not find history files: ${historyFilesEither.reason}" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND)
}
}
return ChangesEither.Known(symbols, fqNames)
fun analyzeHistoryFiles(): ChangesEither {
for (historyFile in historyFiles) {
val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
?: return run {
reporter.report { "Could not read diffs from $historyFile" }
ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ)
}
val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
if (knownBuilds.isEmpty()) {
reporter.report { "No previously known builds for $historyFile" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS)
}
for (buildDiff in newBuilds) {
if (!buildDiff.isIncremental) {
reporter.report { "Non-incremental build from dependency $historyFile" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP)
}
val dirtyData = buildDiff.dirtyData
symbols.addAll(dirtyData.dirtyLookupSymbols)
fqNames.addAll(dirtyData.dirtyClassesFqNames)
}
}
return ChangesEither.Known(symbols, fqNames)
}
return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) {
analyzeHistoryFiles()
}
}
@@ -17,7 +17,7 @@
package org.jetbrains.kotlin.incremental.utils
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import java.io.File
class TestICReporter : ICReporterBase() {
@@ -42,7 +42,7 @@ import org.jetbrains.kotlin.daemon.common.isDaemonEnabled
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
import org.jetbrains.kotlin.jps.model.kotlinKind
@@ -18,6 +18,8 @@ import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel
import org.jetbrains.kotlin.backend.common.serialization.KlibIrVersion
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
@@ -316,7 +318,7 @@ class GenerateIrRuntime {
withJsIC {
val buildHistoryFile = File(cachesDir, "build-history.bin")
val compiler = IncrementalJsCompilerRunner(
cachesDir, EmptyICReporter,
cachesDir, BuildReporter(EmptyICReporter, DoNothingBuildMetricsReporter),
buildHistoryFile = buildHistoryFile,
modulesApiHistory = EmptyModulesApiHistory
)
@@ -360,7 +362,7 @@ class GenerateIrRuntime {
withJsIC {
val buildHistoryFile = File(cachesDir, "build-history.bin")
val compiler = IncrementalJsCompilerRunner(
cachesDir, EmptyICReporter,
cachesDir, BuildReporter(EmptyICReporter, DoNothingBuildMetricsReporter),
buildHistoryFile = buildHistoryFile,
modulesApiHistory = EmptyModulesApiHistory
)
@@ -0,0 +1,24 @@
description = "kotlin-gradle-statistics"
plugins {
kotlin("jvm")
id("jps-compatible")
}
dependencies {
compileOnly(kotlinStdlib())
testImplementation(project(":kotlin-test:kotlin-test-junit"))
testImplementation("junit:junit:4.12")
}
sourceSets {
"main" { projectDefault() }
"test" { projectDefault() }
}
projectTest {
workingDir = rootDir
}
publish()
@@ -0,0 +1,31 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.internal.build.metrics
import java.io.Serializable
class GradleBuildMetricsData : Serializable {
val parentMetric: MutableMap<String, String?> = LinkedHashMap()
val buildAttributeKind: MutableMap<String, String> = LinkedHashMap()
val taskData: MutableMap<String, TaskData> = LinkedHashMap()
companion object {
const val serialVersionUID = 0L
}
}
data class TaskData(
val path: String,
val typeFqName: String,
val timeMetrics: Map<String, Long>,
val buildAttributes: Map<String, Int>,
val didWork: Boolean
) : Serializable {
companion object {
const val serialVersionUID = 0L
}
}
@@ -52,6 +52,8 @@ dependencies {
compileOnly(project(":kotlin-scripting-compiler"))
compileOnly(project(":kotlin-gradle-statistics"))
embedded(project(":kotlin-gradle-statistics"))
compileOnly(project(":kotlin-gradle-build-metrics"))
embedded(project(":kotlin-gradle-build-metrics"))
compile("com.google.code.gson:gson:${rootProject.extra["versions.jar.gson"]}")
compile("de.undercouch:gradle-download-task:4.0.2")
@@ -1,5 +1,7 @@
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface
@@ -22,7 +24,10 @@ internal class GradleCompilationResults(
LoopbackNetworkInterface.serverLoopbackSocketFactory
) {
var icLogLines: List<String>? = null
var icLogLines: List<String> = emptyList()
private val buildMetricsReporter = BuildMetricsReporterImpl()
val buildMetrics: BuildMetrics
get() = buildMetricsReporter.getMetrics()
@Throws(RemoteException::class)
override fun add(compilationResultCategory: Int, value: Serializable) {
@@ -42,7 +47,10 @@ internal class GradleCompilationResults(
CompilationResultCategory.BUILD_REPORT_LINES.code,
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES.code -> {
@Suppress("UNCHECKED_CAST")
icLogLines = value as? List<String>
(value as? List<String>)?.let { icLogLines = it }
}
CompilationResultCategory.BUILD_METRICS.code -> {
buildMetricsReporter.addMetrics(value as? BuildMetrics)
}
}
}
@@ -8,7 +8,7 @@ package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.file.FileCollection
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector
import org.jetbrains.kotlin.gradle.report.BuildReportMode
import org.jetbrains.kotlin.gradle.report.ReportingSettings
import org.jetbrains.kotlin.gradle.tasks.findToolsJar
import java.io.File
@@ -17,7 +17,7 @@ internal class GradleCompilerEnvironment(
messageCollector: GradlePrintingMessageCollector,
outputItemsCollector: OutputItemsCollector,
val outputFiles: FileCollection,
val buildReportMode: BuildReportMode?,
val reportingSettings: ReportingSettings,
val incrementalCompilationEnvironment: IncrementalCompilationEnvironment? = null,
val kotlinScriptExtensions: Array<String> = emptyArray()
) : CompilerEnvironment(Services.EMPTY, messageCollector, outputItemsCollector) {
@@ -160,7 +160,7 @@ internal open class GradleCompilerRunner(protected val taskProvider: GradleCompi
incrementalModuleInfo = modulesInfo,
outputFiles = environment.outputFiles.toList(),
taskPath = pathProvider,
buildReportMode = environment.buildReportMode,
reportingSettings = environment.reportingSettings,
kotlinScriptExtensions = environment.kotlinScriptExtensions,
allWarningsAsErrors = compilerArgs.allWarningsAsErrors
)
@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.build.ExecutionStrategy
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.config.Services
@@ -15,6 +17,7 @@ import org.jetbrains.kotlin.gradle.logging.*
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.gradle.report.BuildReportMode
import org.jetbrains.kotlin.gradle.report.ReportingSettings
import org.jetbrains.kotlin.gradle.report.TaskExecutionResult
import org.jetbrains.kotlin.gradle.tasks.clearLocalState
import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError
@@ -59,7 +62,7 @@ internal class GradleKotlinCompilerWorkArguments(
val incrementalModuleInfo: IncrementalModuleInfo?,
val outputFiles: List<File>,
val taskPath: String,
val buildReportMode: BuildReportMode?,
val reportingSettings: ReportingSettings,
val kotlinScriptExtensions: Array<String>,
val allWarningsAsErrors: Boolean
) : Serializable {
@@ -98,10 +101,12 @@ internal class GradleKotlinCompilerWork @Inject constructor(
private val incrementalModuleInfo = config.incrementalModuleInfo
private val outputFiles = config.outputFiles
private val taskPath = config.taskPath
private val buildReportMode = config.buildReportMode
private val reportingSettings = config.reportingSettings
private val kotlinScriptExtensions = config.kotlinScriptExtensions
private val allWarningsAsErrors = config.allWarningsAsErrors
private val buildDir = config.projectFiles.buildDir
private val metrics = if (reportingSettings.reportMetrics) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter
private var icLogLines: List<String> = emptyList()
private val log: KotlinLogger =
TaskLoggers.get(taskPath)?.let { GradleKotlinLogger(it).apply { debug("Using '$taskPath' logger") } }
@@ -120,13 +125,18 @@ internal class GradleKotlinCompilerWork @Inject constructor(
get() = incrementalCompilationEnvironment != null
override fun run() {
val messageCollector = GradlePrintingMessageCollector(log, allWarningsAsErrors)
val exitCode = compileWithDaemonOrFallbackImpl(messageCollector)
if (incrementalCompilationEnvironment?.disableMultiModuleIC == true) {
incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete()
}
try {
val messageCollector = GradlePrintingMessageCollector(log, allWarningsAsErrors)
val exitCode = compileWithDaemonOrFallbackImpl(messageCollector)
if (incrementalCompilationEnvironment?.disableMultiModuleIC == true) {
incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete()
}
throwGradleExceptionIfError(exitCode)
throwGradleExceptionIfError(exitCode)
} finally {
val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines)
TaskExecutionResults[taskPath] = result
}
}
private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): ExitCode {
@@ -159,20 +169,21 @@ internal class GradleKotlinCompilerWork @Inject constructor(
val isDebugEnabled = log.isDebugEnabled || System.getProperty("kotlin.daemon.debug.log")?.toBoolean() ?: true
val daemonMessageCollector =
if (isDebugEnabled) messageCollector else MessageCollector.NONE
val connection =
try {
GradleCompilerRunner.getDaemonConnectionImpl(
clientIsAliveFlagFile,
sessionFlagFile,
compilerFullClasspath,
daemonMessageCollector,
isDebugEnabled = isDebugEnabled
)
} catch (e: Throwable) {
log.error("Caught an exception trying to connect to Kotlin Daemon:")
log.error(e.stackTraceAsString())
null
metrics.measure(BuildTime.CONNECT_TO_DAEMON) {
try {
GradleCompilerRunner.getDaemonConnectionImpl(
clientIsAliveFlagFile,
sessionFlagFile,
compilerFullClasspath,
daemonMessageCollector,
isDebugEnabled = isDebugEnabled
)
} catch (e: Throwable) {
log.error("Caught an exception trying to connect to Kotlin Daemon:")
log.error(e.stackTraceAsString())
null
}
}
if (connection == null) {
if (isIncremental) {
@@ -215,7 +226,9 @@ internal class GradleKotlinCompilerWork @Inject constructor(
// often source of the NoSuchObjectException and UnmarshalException, probably caused by the failed/crashed/exited daemon
// TODO: implement a proper logic to avoid remote calls in such cases
try {
daemon.clearJarCache()
metrics.measure(BuildTime.CLEAR_JAR_CACHE) {
daemon.clearJarCache()
}
} catch (e: RemoteException) {
log.warn("Unable to clear jar cache after compilation, maybe daemon is already down: $e")
}
@@ -229,6 +242,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
targetPlatform: CompileService.TargetPlatform,
bufferingMessageCollector: GradleBufferingMessageCollector
): CompileService.CallResult<Int> {
metrics.addAttribute(BuildAttribute.IC_IS_NOT_ENABLED)
val compilationOptions = CompilationOptions(
compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER,
targetPlatform = targetPlatform,
@@ -238,15 +252,8 @@ internal class GradleKotlinCompilerWork @Inject constructor(
kotlinScriptExtensions = kotlinScriptExtensions
)
val servicesFacade = GradleCompilerServicesFacadeImpl(log, bufferingMessageCollector)
return try {
return metrics.measure(BuildTime.NON_INCREMENTAL_COMPILATION_DAEMON) {
daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults = null)
} finally {
reportExecutionResultIfNeeded {
TaskExecutionResult(
executionStrategy = DAEMON_EXECUTION_STRATEGY,
icLogLines = nonIcBuildLog("incremental compilation is not enabled for '$taskPath'")
)
}
}
}
@@ -258,14 +265,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
): CompileService.CallResult<Int> {
val icEnv = incrementalCompilationEnvironment ?: error("incrementalCompilationEnvironment is null!")
val knownChangedFiles = icEnv.changedFiles as? ChangedFiles.Known
val requestedCompilationResults = EnumSet.of(CompilationResultCategory.IC_COMPILE_ITERATION)
when (buildReportMode) {
BuildReportMode.SIMPLE -> CompilationResultCategory.BUILD_REPORT_LINES
BuildReportMode.VERBOSE -> CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES
null -> null
}?.let { requestedCompilationResults.add(it) }
val requestedCompilationResults = requestedCompilationResults()
val compilationOptions = IncrementalCompilationOptions(
areFileChangesKnown = knownChangedFiles != null,
modifiedFiles = knownChangedFiles?.modified,
@@ -286,36 +286,28 @@ internal class GradleKotlinCompilerWork @Inject constructor(
log.info("Options for KOTLIN DAEMON: $compilationOptions")
val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(log, bufferingMessageCollector)
val compilationResults = GradleCompilationResults(log, projectRootFile)
val result = daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults)
reportExecutionResultIfNeeded {
TaskExecutionResult(
executionStrategy = DAEMON_EXECUTION_STRATEGY,
icLogLines = compilationResults.icLogLines
)
return metrics.measure(BuildTime.RUN_COMPILER) {
daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults)
}.also {
metrics.addMetrics(compilationResults.buildMetrics)
icLogLines = compilationResults.icLogLines
}
return result
}
private fun compileOutOfProcess(): ExitCode {
clearLocalState(outputFiles, log, reason = "out-of-process execution strategy is non-incremental")
metrics.addAttribute(BuildAttribute.OUT_OF_PROCESS_EXECUTION)
clearLocalState(outputFiles, log, metrics, reason = "out-of-process execution strategy is non-incremental")
return try {
return metrics.measure(BuildTime.NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS) {
runToolInSeparateProcess(compilerArgs, compilerClassName, compilerFullClasspath, log, buildDir)
} finally {
reportExecutionResultIfNeeded {
TaskExecutionResult(
executionStrategy = OUT_OF_PROCESS_EXECUTION_STRATEGY,
icLogLines = nonIcBuildLog("$OUT_OF_PROCESS_EXECUTION_STRATEGY execution strategy does not support incremental compilation")
)
}
}
}
private fun compileInProcess(messageCollector: MessageCollector): ExitCode {
clearLocalState(outputFiles, log, reason = "in-process execution strategy is non-incremental")
metrics.addAttribute(BuildAttribute.IN_PROCESS_EXECUTION)
clearLocalState(outputFiles, log, metrics, reason = "in-process execution strategy is non-incremental")
metrics.startMeasure(BuildTime.NON_INCREMENTAL_COMPILATION_IN_PROCESS, System.nanoTime())
// in-process compiler should always be run in a different thread
// to avoid leaking thread locals from compiler (see KT-28037)
val threadPool = Executors.newSingleThreadExecutor()
@@ -329,12 +321,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
bufferingMessageCollector.flush(messageCollector)
threadPool.shutdown()
reportExecutionResultIfNeeded {
TaskExecutionResult(
executionStrategy = IN_PROCESS_EXECUTION_STRATEGY,
icLogLines = nonIcBuildLog("$IN_PROCESS_EXECUTION_STRATEGY execution strategy does not support incremental compilation")
)
}
metrics.endMeasure(BuildTime.NON_INCREMENTAL_COMPILATION_IN_PROCESS, System.nanoTime())
}
}
@@ -366,6 +353,19 @@ internal class GradleKotlinCompilerWork @Inject constructor(
return exitCode
}
private fun requestedCompilationResults(): EnumSet<CompilationResultCategory> {
val requestedCompilationResults = EnumSet.of(CompilationResultCategory.IC_COMPILE_ITERATION)
when (reportingSettings.buildReportMode) {
BuildReportMode.NONE -> null
BuildReportMode.SIMPLE -> CompilationResultCategory.BUILD_REPORT_LINES
BuildReportMode.VERBOSE -> CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES
}?.let { requestedCompilationResults.add(it) }
if (reportingSettings.reportMetrics) {
requestedCompilationResults.add(CompilationResultCategory.BUILD_METRICS)
}
return requestedCompilationResults
}
private fun reportCategories(verbose: Boolean): Array<Int> =
if (!verbose) {
arrayOf(ReportCategory.COMPILER_MESSAGE.code)
@@ -379,14 +379,4 @@ internal class GradleKotlinCompilerWork @Inject constructor(
} else {
ReportSeverity.DEBUG.code
}
private inline fun reportExecutionResultIfNeeded(fn: () -> TaskExecutionResult) {
if (buildReportMode != null) {
val result = fn()
TaskExecutionResults[taskPath] = result
}
}
private fun nonIcBuildLog(reason: String): List<String> =
listOf("Performing non-incremental build: $reason")
}
@@ -5,6 +5,8 @@ import org.gradle.api.file.FileCollection
import org.gradle.api.internal.ConventionTask
import org.gradle.api.tasks.*
import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.ClasspathSnapshot
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptClasspathChanges
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges
@@ -137,6 +139,10 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState {
protected val javaSourceRoots: Set<File>
get() = unfilteredJavaSourceRoots.filterTo(HashSet(), ::isRootAllowed)
@get:Internal
override val metrics: BuildMetricsReporter =
BuildMetricsReporterImpl()
private fun isRootAllowed(file: File): Boolean =
file.exists() &&
!isAncestor(destinationDir, file) &&
@@ -84,7 +84,7 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
private var processIncrementally = false
private val javaPackagePrefix by project.optionalProvider { kotlinCompileTask.javaPackagePrefix }
private val buildReportMode by project.optionalProvider { kotlinCompileTask.buildReportMode }
private val reportingSettings by project.provider { kotlinCompileTask.reportingSettings }
@TaskAction
fun compile(inputs: IncrementalTaskInputs) {
@@ -104,7 +104,7 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
val outputItemCollector = OutputItemsCollectorImpl()
val environment = GradleCompilerEnvironment(
compilerClasspath, messageCollector, outputItemCollector,
buildReportMode = buildReportMode,
reportingSettings = reportingSettings,
outputFiles = allOutputFiles()
)
if (environment.toolsJar == null && !isAtLeastJava9) {
@@ -7,9 +7,15 @@ package org.jetbrains.kotlin.gradle.internal.tasks
import org.gradle.api.Task
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.Internal
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.gradle.utils.outputsCompatible
internal interface TaskWithLocalState : Task {
fun localStateDirectories(): FileCollection
@get:Internal
val metrics: BuildMetricsReporter
}
internal fun TaskWithLocalState.allOutputFiles(): FileCollection =
@@ -13,10 +13,9 @@ import org.gradle.api.logging.Logging
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.gradle.report.configureBuildReporter
import org.jetbrains.kotlin.gradle.report.configureReporting
import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
//Support Gradle 6 and less. Move to
internal class KotlinGradleBuildServices private constructor(
private val gradle: Gradle
@@ -99,7 +98,7 @@ internal class KotlinGradleBuildServices private constructor(
TaskLoggers.clear()
TaskExecutionResults.clear()
configureBuildReporter(gradle, log)
configureReporting(gradle)
}
override fun buildFinished(result: BuildResult) {
@@ -57,9 +57,15 @@ internal class PropertiesProvider private constructor(private val project: Proje
val coroutines: Coroutines?
get() = property("kotlin.coroutines")?.let { Coroutines.byCompilerArgument(it) }
val singleBuildMetricsFile: File?
get() = property("kotlin.internal.single.build.metrics.file")?.let { File(it) }
val buildReportEnabled: Boolean
get() = booleanProperty("kotlin.build.report.enable") ?: false
val buildReportMetrics: Boolean
get() = booleanProperty("kotlin.build.report.metrics") ?: false
val buildReportVerbose: Boolean
get() = booleanProperty("kotlin.build.report.verbose") ?: false
@@ -0,0 +1,101 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.BuildAdapter
import org.gradle.BuildResult
import org.gradle.api.Task
import org.gradle.api.execution.TaskExecutionListener
import org.gradle.api.invocation.Gradle
import org.gradle.api.tasks.TaskState
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.gradle.internal.tasks.TaskWithLocalState
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData
import java.util.*
internal class BuildDataRecorder(
private val gradle: Gradle,
private val buildDataProcessors: Iterable<BuildExecutionDataProcessor>
) : BuildAdapter(), TaskExecutionListener {
private val taskRecords = HashMap<Task, TaskRecord>()
@Synchronized
override fun beforeExecute(task: Task) {
val startNs = System.nanoTime()
taskRecords[task] = TaskRecord(task, startNs)
}
@Synchronized
override fun afterExecute(task: Task, state: TaskState) {
taskRecords[task]?.apply { processResult(state, TaskExecutionResults[task.path]) }
}
override fun buildFinished(result: BuildResult) {
val startParams = arrayListOf<String>()
gradle.startParameter.apply {
startParams.add("tasks = ${taskRequests.joinToString { it.args.toString() }}")
startParams.add("excluded tasks = $excludedTaskNames")
startParams.add("current dir = $currentDir")
startParams.add("project properties args = $projectProperties")
startParams.add("system properties args = $systemPropertiesArgs")
}
val buildData = BuildExecutionData(
startParameters = startParams,
failure = result.failure,
taskExecutionData = taskRecords.values.sortedBy { it.startNs }
)
buildDataProcessors.forEach { it.process(buildData) }
}
}
private class TaskRecord(
override val task: Task,
override val startNs: Long
) : TaskExecutionData {
private var myEndNs: Long = 0
private lateinit var myTaskState: TaskState
private var myIcLogLines: List<String> = emptyList()
private val myBuildMetrics = BuildMetrics()
override val isKotlinTask by lazy {
task.javaClass.name.startsWith("org.jetbrains.kotlin")
}
override val endNs: Long
get() = myEndNs
override val totalTimeNs: Long
get() = endNs - startNs
override val resultState: TaskState
get() = myTaskState
override val icLogLines: List<String>
get() = myIcLogLines
override val buildMetrics: BuildMetrics
get() = myBuildMetrics
fun processResult(state: TaskState, executionResult: TaskExecutionResult?) {
myEndNs = System.nanoTime()
myTaskState = state
myBuildMetrics.buildTimes.add(BuildTime.GRADLE_TASK, totalTimeNs)
if (executionResult != null) {
myBuildMetrics.addAll(executionResult.buildMetrics)
myIcLogLines = executionResult.icLogLines
}
if (task is TaskWithLocalState) {
myBuildMetrics.addAll(task.metrics.getMetrics())
}
}
}
@@ -5,7 +5,14 @@
package org.jetbrains.kotlin.gradle.report
enum class BuildReportMode {
import java.io.Serializable
enum class BuildReportMode : Serializable {
NONE,
SIMPLE,
VERBOSE
}
VERBOSE;
companion object {
const val serialVersionUID: Long = 0
}
}
@@ -1,120 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.BuildAdapter
import org.gradle.BuildResult
import org.gradle.api.Task
import org.gradle.api.execution.TaskExecutionListener
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logger
import org.gradle.api.tasks.TaskState
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
import java.io.File
import java.text.SimpleDateFormat
import java.util.*
internal fun configureBuildReporter(gradle: Gradle, log: Logger) {
val rootProject = gradle.rootProject
val properties = PropertiesProvider(rootProject)
if (!properties.buildReportEnabled) return
val perfLogDir = properties.buildReportDir
?: rootProject.buildDir.resolve("reports/kotlin-build").apply { mkdirs() }
if (perfLogDir.isFile) {
log.error("Kotlin build report cannot be created: '$perfLogDir' is a file")
return
}
perfLogDir.mkdirs()
val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time)
val perfReportFile = perfLogDir.resolve("${gradle.rootProject.name}-build-$ts.txt")
if (!isConfigurationCacheAvailable(gradle)) {
val reporter = KotlinBuildReporter(gradle, perfReportFile)
gradle.addBuildListener(reporter)
}
val buildReportMode = if (properties.buildReportVerbose) BuildReportMode.VERBOSE else BuildReportMode.SIMPLE
gradle.taskGraph.whenReady { graph ->
graph.allTasks.asSequence()
.filterIsInstance<AbstractKotlinCompile<*>>()
.forEach { it.buildReportMode = buildReportMode }
}
log.kotlinDebug { "Configured Kotlin build reporter" }
}
internal class KotlinBuildReporter(
private val gradle: Gradle,
private val perfReportFile: File
) : BuildAdapter(), TaskExecutionListener {
init {
val dir = perfReportFile.parentFile
check(dir.isDirectory) { "$dir does not exist or is a file" }
check(!perfReportFile.isFile) { "Build report log file $perfReportFile exists already" }
}
private val taskStartNs = HashMap<Task, Long>()
private val kotlinTaskTimeNs = HashMap<Task, Long>()
private val tasksSb = StringBuilder()
@Volatile
private var allTasksTimeNs: Long = 0L
@Synchronized
override fun beforeExecute(task: Task) {
taskStartNs[task] = System.nanoTime()
}
@Synchronized
override fun afterExecute(task: Task, state: TaskState) {
val startNs = taskStartNs[task] ?: return
val endNs = System.nanoTime()
val timeNs = endNs - startNs
allTasksTimeNs += timeNs
if (!task.javaClass.name.startsWith("org.jetbrains.kotlin")) {
return
}
kotlinTaskTimeNs[task] = timeNs
tasksSb.appendln()
val skipMessage = state.skipMessage
if (skipMessage != null) {
tasksSb.appendln("$task was skipped: $skipMessage")
} else {
tasksSb.appendln("$task finished in ${formatTime(timeNs)}")
}
val path = task.path
val executionResult = TaskExecutionResults[path]
if (executionResult != null) {
tasksSb.appendln("Execution strategy: ${executionResult.executionStrategy}")
executionResult.icLogLines?.let { lines ->
tasksSb.appendln("Compilation log for $task:")
lines.forEach { tasksSb.appendln(" $it") }
}
}
}
@Synchronized
override fun buildFinished(result: BuildResult) {
KotlinBuildReporterHandler().buildFinished(gradle, perfReportFile, kotlinTaskTimeNs.mapKeys{it.key.path}, allTasksTimeNs, result.failure)
}
}
@@ -5,7 +5,6 @@
package org.jetbrains.kotlin.gradle.report
import com.intellij.util.text.DateFormatUtil.formatTime
import org.gradle.api.invocation.Gradle
import org.gradle.tooling.events.FinishEvent
import org.gradle.tooling.events.OperationCompletionListener
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData
import org.jetbrains.kotlin.gradle.internal.build.metrics.TaskData
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
import java.io.File
import java.io.ObjectOutputStream
internal class MetricsWriter(
private val outputFile: File,
private val log: Logger
) : BuildExecutionDataProcessor {
override fun process(build: BuildExecutionData) {
if (build.failure != null) return
try {
outputFile.parentFile?.apply { mkdirs() }
val buildMetricsData = GradleBuildMetricsData()
for (metric in BuildTime.values()) {
buildMetricsData.parentMetric[metric.name] = metric.parent?.name
}
for (attr in BuildAttribute.values()) {
buildMetricsData.buildAttributeKind[attr.name] = attr.kind.name
}
for (data in build.taskExecutionData) {
val path = data.task.path
val type = data.task::class.java.canonicalName
val buildTimes = data.buildMetrics.buildTimes.asMap().mapKeys { (k, _) -> k.name }
val buildAttributes = data.buildMetrics.buildAttributes.asMap().mapKeys { (k, _) -> k.name }
buildMetricsData.taskData[path] =
TaskData(
path = path,
typeFqName = type,
timeMetrics = buildTimes,
buildAttributes = buildAttributes,
didWork = data.task.didWork
)
}
ObjectOutputStream(outputFile.outputStream().buffered()).use { out ->
out.writeObject(buildMetricsData)
}
} catch (e: Exception) {
log.kotlinDebug { "Could not write metrics to $outputFile: $e" }
}
}
}
@@ -0,0 +1,206 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.build.report.metrics.BuildAttributes
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.BuildTimes
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData
import org.jetbrains.kotlin.gradle.utils.Printer
import java.io.File
import java.util.*
import kotlin.math.max
internal class PlainTextBuildReportWriter(
private val outputFile: File,
private val printMetrics: Boolean,
private val log: Logger
) : BuildExecutionDataProcessor {
private lateinit var p: Printer
override fun process(build: BuildExecutionData) {
try {
outputFile.bufferedWriter().use { writer ->
p = Printer(writer)
printBuildReport(build)
}
log.lifecycle("Kotlin build report is written to ${outputFile.canonicalPath}")
} catch (e: Exception) {
log.error("Could not write Kotlin build report to ${outputFile.canonicalPath}", e)
}
}
private fun printBuildReport(build: BuildExecutionData) {
printBuildInfo(build)
printMetrics(build.aggregatedMetrics)
printTaskOverview(build)
printTasksLog(build)
}
private fun printBuildInfo(build: BuildExecutionData) {
p.withIndent("Gradle start parameters:") {
build.startParameters.forEach { p.println(it) }
}
p.println()
if (build.failure != null) {
p.println("Build failed: ${build.failure}")
}
p.println()
}
private fun printMetrics(buildMetrics: BuildMetrics) {
if (!printMetrics) return
printBuildTimes(buildMetrics.buildTimes)
printBuildAttributes(buildMetrics.buildAttributes)
}
private fun printBuildTimes(buildTimes: BuildTimes) {
val collectedBuildTimes = buildTimes.asMap()
if (collectedBuildTimes.isEmpty()) return
p.println("Time metrics:")
p.withIndent {
val visitedBuildTimes = HashSet<BuildTime>()
fun printBuildTime(buildTime: BuildTime) {
if (!visitedBuildTimes.add(buildTime)) return
val timeNs = collectedBuildTimes[buildTime] ?: return
p.println("${buildTime.name}: ${formatTime(timeNs)}")
p.withIndent {
BuildTime.children[buildTime]?.forEach { printBuildTime(it) }
}
}
for (buildTime in BuildTime.values()) {
if (buildTime.parent != null) continue
printBuildTime(buildTime)
}
}
p.println()
}
private fun printBuildAttributes(buildAttributes: BuildAttributes) {
val allAttributes = buildAttributes.asMap()
if (allAttributes.isEmpty()) return
p.withIndent("Build attributes:") {
val attributesByKind = allAttributes.entries.groupBy { it.key.kind }.toSortedMap()
for ((kind, attributesCounts) in attributesByKind) {
printMap(p, kind.name, attributesCounts.map { (k, v) -> k.name to v }.toMap())
}
}
p.println()
}
private fun printTaskOverview(build: BuildExecutionData) {
var allTasksTimeNs = 0L
var kotlinTotalTimeNs = 0L
val kotlinTasks = ArrayList<TaskExecutionData>()
for (task in build.taskExecutionData) {
val taskTimeNs = task.totalTimeNs
allTasksTimeNs += taskTimeNs
if (task.isKotlinTask) {
kotlinTotalTimeNs += taskTimeNs
kotlinTasks.add(task)
}
}
if (kotlinTasks.isEmpty()) {
p.println("No Kotlin task was run")
return
}
val ktTaskPercent = (kotlinTotalTimeNs.toDouble() / allTasksTimeNs * 100).asString(1)
p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeNs)} ($ktTaskPercent % of all tasks time)")
val table = TextTable("Time", "% of Kotlin time", "Task")
for (task in kotlinTasks.sortedByDescending { it.totalTimeNs }) {
val timeNs = task.totalTimeNs
val percent = (timeNs.toDouble() / kotlinTotalTimeNs * 100).asString(1)
table.addRow(formatTime(timeNs), "$percent %", task.task.path)
}
table.printTo(p)
p.println()
}
private fun printTasksLog(build: BuildExecutionData) {
for (task in build.taskExecutionData) {
printTaskLog(task)
p.println()
}
}
private fun printTaskLog(task: TaskExecutionData) {
val skipMessage = task.resultState.skipMessage
if (skipMessage != null) {
p.println("$task was skipped: $skipMessage")
} else {
p.println("$task finished in ${formatTime(task.totalTimeNs)}")
}
if (task.icLogLines.isNotEmpty()) {
p.withIndent("Compilation log for $task:") {
task.icLogLines.forEach { p.println(it) }
}
}
printMetrics(task.buildMetrics)
}
}
private fun printMap(p: Printer, name: String, mapping: Map<String, Int>) {
if (mapping.isEmpty()) return
if (mapping.size == 1) {
p.println("$name: ${mapping.keys.single()}")
return
}
p.withIndent("$name:") {
val sortedEnumMap = mapping.toSortedMap()
for ((k, v) in sortedEnumMap) {
p.println("$k($v)")
}
}
}
private class TextTable(vararg columnNames: String) {
private val rows = ArrayList<List<String>>()
private val columnsCount = columnNames.size
private val maxLengths = IntArray(columnsCount) { columnNames[it].length }
init {
rows.add(columnNames.toList())
}
fun addRow(vararg row: String) {
check(row.size == columnsCount) { "Row size ${row.size} differs from columns count $columnsCount" }
rows.add(row.toList())
for ((i, col) in row.withIndex()) {
maxLengths[i] = max(maxLengths[i], col.length)
}
}
fun printTo(p: Printer) {
for (row in rows) {
val rowStr = row.withIndex().joinToString("|") { (i, col) -> col.padEnd(maxLengths[i], ' ') }
p.println(rowStr)
}
}
}
@@ -0,0 +1,21 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import java.io.File
import java.io.Serializable
data class ReportingSettings(
val metricsOutputFile: File? = null,
val buildReportDir: File? = null,
val reportMetrics: Boolean = false,
val includeMetricsInReport: Boolean = false,
val buildReportMode: BuildReportMode = BuildReportMode.NONE
) : Serializable {
companion object {
const val serialVersionUID: Long = 0
}
}
@@ -5,7 +5,9 @@
package org.jetbrains.kotlin.gradle.report
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
internal class TaskExecutionResult(
val executionStrategy: String,
val icLogLines: List<String>?
val buildMetrics: BuildMetrics,
val icLogLines: List<String> = emptyList()
)
@@ -0,0 +1,83 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import org.gradle.api.Project
import org.gradle.api.invocation.Gradle
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
import java.text.SimpleDateFormat
import java.util.*
internal fun configureReporting(gradle: Gradle) {
val buildDataProcessors = ArrayList<BuildExecutionDataProcessor>()
val rootProject = gradle.rootProject
val reportingSettings = reportingSettings(rootProject)
gradle.taskGraph.whenReady { graph ->
graph.allTasks.asSequence()
.filterIsInstance<AbstractKotlinCompile<*>>()
.forEach { it.reportingSettings = reportingSettings }
}
if (reportingSettings.buildReportMode != BuildReportMode.NONE && reportingSettings.buildReportDir != null) {
configurePlainTextReportWriter(gradle, reportingSettings)?.let {
buildDataProcessors.add(it)
}
}
if (reportingSettings.metricsOutputFile != null) {
buildDataProcessors.add(MetricsWriter(reportingSettings.metricsOutputFile.absoluteFile, rootProject.logger))
}
if (buildDataProcessors.isNotEmpty() && !isConfigurationCacheAvailable(gradle)) {
val listener = BuildDataRecorder(gradle, buildDataProcessors)
gradle.addBuildListener(listener)
}
}
private fun reportingSettings(rootProject: Project): ReportingSettings {
val properties = PropertiesProvider(rootProject)
val buildReportMode =
when {
!properties.buildReportEnabled -> BuildReportMode.NONE
properties.buildReportVerbose -> BuildReportMode.VERBOSE
else -> BuildReportMode.SIMPLE
}
val metricsOutputFile = properties.singleBuildMetricsFile
val buildReportDir = properties.buildReportDir ?: rootProject.buildDir.resolve("reports/kotlin-build")
val includeMetricsInReport = properties.buildReportMetrics || buildReportMode == BuildReportMode.VERBOSE
return ReportingSettings(
metricsOutputFile = metricsOutputFile,
buildReportDir = buildReportDir,
reportMetrics = metricsOutputFile != null || includeMetricsInReport,
includeMetricsInReport = includeMetricsInReport,
buildReportMode = buildReportMode
)
}
private fun configurePlainTextReportWriter(
gradle: Gradle,
reportingSettings: ReportingSettings
): BuildExecutionDataProcessor? {
val log = gradle.rootProject.logger
val reportDir = reportingSettings.buildReportDir!!.apply { mkdirs() }
if (reportDir.isFile) {
log.error("Kotlin build report cannot be created: '$reportDir' is a file")
return null
}
reportDir.mkdirs()
val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time)
val reportFile = reportDir.resolve("${gradle.rootProject.name}-build-$ts.txt")
return PlainTextBuildReportWriter(
outputFile = reportFile,
printMetrics = reportingSettings.includeMetricsInReport,
log = log
)
}
@@ -0,0 +1,20 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report.data
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
internal class BuildExecutionData(
val startParameters: Collection<String>,
val failure: Throwable?,
val taskExecutionData: Collection<TaskExecutionData>
) {
val aggregatedMetrics by lazy {
BuildMetrics().also { acc ->
taskExecutionData.forEach { acc.addAll(it.buildMetrics) }
}
}
}
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report.data
internal interface BuildExecutionDataProcessor {
fun process(build: BuildExecutionData)
}
@@ -0,0 +1,22 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report.data
import org.gradle.api.Task
import org.gradle.api.tasks.TaskState
import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
internal interface TaskExecutionData {
val task: Task
val startNs: Long
val endNs: Long
val totalTimeNs: Long
val resultState: TaskState
val icLogLines: List<String>
val buildMetrics: BuildMetrics
val isKotlinTask: Boolean
}
@@ -99,7 +99,7 @@ open class KotlinCompileCommon : AbstractKotlinCompile<K2MetadataCompilerArgumen
val compilerRunner = compilerRunner
val environment = GradleCompilerEnvironment(
computedCompilerClasspath, messageCollector, outputItemCollector,
buildReportMode = buildReportMode,
reportingSettings = reportingSettings,
outputFiles = allOutputFiles()
)
compilerRunner.runMetadataCompilerAsync(sourceRoots.kotlinSourceFiles, args, environment)
@@ -16,6 +16,10 @@ import org.gradle.api.tasks.compile.JavaCompile
import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.CommonToolArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
@@ -38,7 +42,7 @@ import org.jetbrains.kotlin.gradle.plugin.PLUGIN_CLASSPATH_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinCompilation
import org.jetbrains.kotlin.gradle.plugin.mpp.associateWithTransitiveClosure
import org.jetbrains.kotlin.gradle.plugin.mpp.ownModuleName
import org.jetbrains.kotlin.gradle.report.BuildReportMode
import org.jetbrains.kotlin.gradle.report.ReportingSettings
import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.gradle.utils.isParentOf
import org.jetbrains.kotlin.gradle.utils.pathsAsStringRelativeTo
@@ -92,6 +96,10 @@ abstract class AbstractKotlinCompileTool<T : CommonToolArguments>
@get:Input
internal var useFallbackCompilerSearch: Boolean = false
@get:Internal
override val metrics: BuildMetricsReporter =
BuildMetricsReporterImpl()
@get:Classpath
@get:InputFiles
internal val computedCompilerClasspath: List<File> by lazy {
@@ -195,7 +203,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
incremental
@get:Internal
internal var buildReportMode: BuildReportMode? = null
internal var reportingSettings = ReportingSettings()
@get:Internal
internal val taskData: KotlinCompileTaskData = KotlinCompileTaskData.get(project, name)
@@ -327,7 +335,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
// To prevent this, we backup outputs before incremental build and restore when exception is thrown
val outputsBackup: TaskOutputsBackup? =
if (isIncrementalCompilationEnabled() && inputs.isIncremental)
kotlinLogger.logTime("Backing up outputs for incremental build") {
metrics.measure(BuildTime.BACKUP_OUTPUT) {
TaskOutputsBackup(allOutputFiles())
}
else null
@@ -342,7 +350,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
executeImpl(inputs)
} catch (t: Throwable) {
if (outputsBackup != null) {
kotlinLogger.logTime("Restoring previous outputs on error") {
metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
outputsBackup.restoreOutputs()
}
}
@@ -525,7 +533,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
val environment = GradleCompilerEnvironment(
computedCompilerClasspath, messageCollector, outputItemCollector,
outputFiles = allOutputFiles(),
buildReportMode = buildReportMode,
reportingSettings = reportingSettings,
incrementalCompilationEnvironment = icEnv,
kotlinScriptExtensions = sourceFilesExtensions.toTypedArray()
)
@@ -752,7 +760,7 @@ open class Kotlin2JsCompile : AbstractKotlinCompile<K2JSCompilerArguments>(), Ko
val environment = GradleCompilerEnvironment(
computedCompilerClasspath, messageCollector, outputItemCollector,
outputFiles = allOutputFiles(),
buildReportMode = buildReportMode,
reportingSettings = reportingSettings,
incrementalCompilationEnvironment = icEnv
)
compilerRunner.runJsCompilerAsync(sourceRoots.kotlinSourceFiles, commonSourceSet.toList(), args, environment)
@@ -1,6 +1,9 @@
package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.GradleException
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.compilerRunner.KotlinLogger
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
@@ -22,26 +25,33 @@ fun throwGradleExceptionIfError(exitCode: ExitCode) {
internal fun TaskWithLocalState.clearLocalState(reason: String? = null) {
val log = GradleKotlinLogger(logger)
clearLocalState(allOutputFiles(), log, reason)
clearLocalState(allOutputFiles(), log, metrics, reason)
}
internal fun clearLocalState(outputFiles: Iterable<File>, log: KotlinLogger, reason: String? = null) {
internal fun clearLocalState(
outputFiles: Iterable<File>,
log: KotlinLogger,
metrics: BuildMetricsReporter,
reason: String? = null
) {
log.kotlinDebug {
val suffix = reason?.let { " ($it)" }.orEmpty()
"Clearing output$suffix:"
}
for (file in outputFiles) {
if (!file.exists()) continue
when {
file.isDirectory -> {
log.debug("Deleting output directory: $file")
file.deleteRecursively()
file.mkdirs()
}
file.isFile -> {
log.debug("Deleting output file: $file")
file.delete()
metrics.measure(BuildTime.CLEAR_OUTPUT) {
for (file in outputFiles) {
if (!file.exists()) continue
when {
file.isDirectory -> {
log.debug("Deleting output directory: $file")
file.deleteRecursively()
file.mkdirs()
}
file.isFile -> {
log.debug("Deleting output file: $file")
file.delete()
}
}
}
}
@@ -0,0 +1,81 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.gradle.utils
import java.io.IOException
private val LINE_SEPARATOR = System.getProperty("line.separator")
internal class Printer(
private val out: Appendable,
private val indentUnit: String = " ",
private var indent: String = ""
) {
private fun append(s: String) {
try {
out.append(s)
} catch (e: IOException) { // Do nothing
}
}
fun println(vararg strings: String) {
this.print(*strings)
printLineSeparator()
}
private fun printLineSeparator() {
append(LINE_SEPARATOR)
}
fun print(vararg strings: String) {
if (strings.isNotEmpty()) {
this.printIndent()
}
this.printWithNoIndent(*strings)
}
private fun printIndent() {
append(indent)
}
private fun printWithNoIndent(vararg strings: String) {
for (s in strings) {
append(s)
}
}
fun pushIndent() {
indent += indentUnit
}
fun popIndent() {
check(indent.length >= indentUnit.length) { "No indentation to pop" }
indent = indent.substring(indentUnit.length)
}
inline fun <T> withIndent(headLine: String? = null, fn: () -> T): T {
if (headLine != null) {
this.println(headLine)
}
pushIndent()
return try {
fn()
} finally {
popIndent()
}
}
}
@@ -20,7 +20,7 @@ import kotlin.jvm.functions.Function0;
import org.apache.maven.plugin.logging.Log;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.kotlin.cli.common.ExitCode;
import org.jetbrains.kotlin.incremental.ICReporterBase;
import org.jetbrains.kotlin.build.report.ICReporterBase;
import java.io.File;
import java.util.Collection;
+2
View File
@@ -241,6 +241,7 @@ include ":benchmarks",
":kotlin-gradle-plugin-dsl-codegen",
":kotlin-gradle-plugin-npm-versions-codegen",
":kotlin-gradle-statistics",
":kotlin-gradle-build-metrics",
":kotlin-gradle-plugin",
":kotlin-gradle-plugin-model",
":kotlin-gradle-plugin-test-utils-embeddable",
@@ -500,6 +501,7 @@ project(':kotlin-gradle-plugin-api').projectDir = "$rootDir/libraries/tools/kotl
project(':kotlin-gradle-plugin-dsl-codegen').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-dsl-codegen" as File
project(':kotlin-gradle-plugin-npm-versions-codegen').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-npm-versions-codegen" as File
project(':kotlin-gradle-statistics').projectDir = "$rootDir/libraries/tools/kotlin-gradle-statistics" as File
project(':kotlin-gradle-build-metrics').projectDir = "$rootDir/libraries/tools/kotlin-gradle-build-metrics" as File
project(':kotlin-gradle-plugin').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin" as File
project(':kotlin-gradle-plugin-model').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-model" as File
project(':kotlin-gradle-plugin-test-utils-embeddable').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-test-utils-embeddable" as File