diff --git a/build-common/src/org/jetbrains/kotlin/build/ExecutionStrategy.kt b/build-common/src/org/jetbrains/kotlin/build/ExecutionStrategy.kt new file mode 100644 index 00000000000..f3cb6cb302b --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/ExecutionStrategy.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build + +import java.io.Serializable + +enum class ExecutionStrategy : Serializable { + DAEMON, IN_PROCESS, OUT_OF_PROCESS +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt new file mode 100644 index 00000000000..c6547bff350 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report + +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter + +open class BuildReporter( + protected open val icReporter: ICReporter, + protected open val buildMetricsReporter: BuildMetricsReporter +) : ICReporter by icReporter, BuildMetricsReporter by buildMetricsReporter + +class RemoteBuildReporter( + override val icReporter: RemoteICReporter, + override val buildMetricsReporter: RemoteBuildMetricsReporter +) : BuildReporter(icReporter, buildMetricsReporter), RemoteReporter { + override fun flush() { + icReporter.flush() + buildMetricsReporter.flush() + } +} diff --git a/build-common/src/org/jetbrains/kotlin/incremental/ICReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt similarity index 63% rename from build-common/src/org/jetbrains/kotlin/incremental/ICReporter.kt rename to build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt index 3986e888924..b08e2feef41 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/ICReporter.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt @@ -3,7 +3,7 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.incremental +package org.jetbrains.kotlin.build.report import org.jetbrains.kotlin.cli.common.ExitCode import java.io.File @@ -16,21 +16,6 @@ interface ICReporter { fun reportMarkDirtyClass(affectedFiles: Iterable, classFqName: String) fun reportMarkDirtyMember(affectedFiles: Iterable, scope: String, name: String) fun reportMarkDirty(affectedFiles: Iterable, reason: String) - - fun startMeasure(metric: String, startNs: Long) - fun endMeasure(metric: String, endNs: Long) } -fun ICReporter?.measure(metric: String, fn: () -> T): T { - if (this == null) return fn() - val start = System.nanoTime() - startMeasure(metric, start) - - try { - return fn() - } finally { - val end = System.nanoTime() - endMeasure(metric, end) - } -} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/ICReporterBase.kt b/build-common/src/org/jetbrains/kotlin/build/report/ICReporterBase.kt similarity index 84% rename from build-common/src/org/jetbrains/kotlin/incremental/ICReporterBase.kt rename to build-common/src/org/jetbrains/kotlin/build/report/ICReporterBase.kt index dab297b6087..db03207f8b2 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/ICReporterBase.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/ICReporterBase.kt @@ -1,9 +1,9 @@ /* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.incremental +package org.jetbrains.kotlin.build.report import java.io.File @@ -33,10 +33,4 @@ abstract class ICReporterBase(private val pathsBase: File? = null) : ICReporter protected fun File.relativeOrCanonical(): File = pathsBase?.let { relativeToOrNull(it) } ?: canonicalFile - - override fun startMeasure(metric: String, startNs: Long) { - } - - override fun endMeasure(metric: String, endNs: Long) { - } } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/RemoteICReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/RemoteICReporter.kt new file mode 100644 index 00000000000..ae1993da9eb --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/RemoteICReporter.kt @@ -0,0 +1,9 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report + +interface RemoteICReporter : ICReporter, + RemoteReporter \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/RemoteReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/RemoteReporter.kt new file mode 100644 index 00000000000..464e70b7044 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/RemoteReporter.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report + +interface RemoteReporter { + fun flush() +} + diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt new file mode 100644 index 00000000000..31793ed66d4 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.io.Serializable + +enum class BuildAttributeKind : Serializable { + REBUILD_REASON; + + companion object { + const val serialVersionUID = 0L + } +} + +enum class BuildAttribute(val kind: BuildAttributeKind) : Serializable { + NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON), + CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON), + UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON), + JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON), + JAVA_CHANGE_UNEXPECTED_PSI(BuildAttributeKind.REBUILD_REASON), + JAVA_CHANGE_UNKNOWN_QUALIFIER(BuildAttributeKind.REBUILD_REASON), + DEP_CHANGE_REMOVED_ENTRY(BuildAttributeKind.REBUILD_REASON), + DEP_CHANGE_HISTORY_IS_NOT_FOUND(BuildAttributeKind.REBUILD_REASON), + DEP_CHANGE_HISTORY_CANNOT_BE_READ(BuildAttributeKind.REBUILD_REASON), + DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS(BuildAttributeKind.REBUILD_REASON), + DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP(BuildAttributeKind.REBUILD_REASON), + IN_PROCESS_EXECUTION(BuildAttributeKind.REBUILD_REASON), + OUT_OF_PROCESS_EXECUTION(BuildAttributeKind.REBUILD_REASON), + IC_IS_NOT_ENABLED(BuildAttributeKind.REBUILD_REASON); + + companion object { + const val serialVersionUID = 0L + } +} diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt new file mode 100644 index 00000000000..867d2adbcc7 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.io.Serializable +import java.util.* + +class BuildAttributes : Serializable { + private val myAttributes = + EnumMap( + BuildAttribute::class.java + ) + + fun add(attr: BuildAttribute, count: Int = 1) { + myAttributes[attr] = myAttributes.getOrDefault(attr, 0) + count + } + + fun addAll(other: BuildAttributes) { + other.myAttributes.forEach { (attr, n) -> add(attr, n) } + } + + fun asMap(): Map = myAttributes + + companion object { + const val serialVersionUID = 0L + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt new file mode 100644 index 00000000000..94c48e07579 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetrics.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.io.Serializable + +data class BuildMetrics( + val buildTimes: BuildTimes = BuildTimes(), + val buildAttributes: BuildAttributes = BuildAttributes() +) : Serializable { + fun addAll(other: BuildMetrics) { + buildTimes.addAll(other.buildTimes) + buildAttributes.addAll(other.buildAttributes) + } + + companion object { + const val serialVersionUID = 0L + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt new file mode 100644 index 00000000000..e27d5a70223 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporter.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +interface BuildMetricsReporter { + fun startMeasure(metric: BuildTime, startNs: Long) + fun endMeasure(metric: BuildTime, endNs: Long) + + fun addAttribute(attribute: BuildAttribute) + + fun getMetrics(): BuildMetrics + fun addMetrics(metrics: BuildMetrics?) +} + +inline fun BuildMetricsReporter.measure(metric: BuildTime, fn: () -> T): T { + val start = System.nanoTime() + startMeasure(metric, start) + + try { + return fn() + } finally { + val end = System.nanoTime() + endMeasure(metric, end) + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt new file mode 100644 index 00000000000..be6a28b28cc --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildMetricsReporterImpl.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.util.* + +class BuildMetricsReporterImpl : BuildMetricsReporter { + private val myBuildTimeStartNs: EnumMap = + EnumMap( + BuildTime::class.java + ) + private val myBuildTimes = BuildTimes() + private val myBuildAttributes = BuildAttributes() + + override fun startMeasure(metric: BuildTime, startNs: Long) { + if (metric in myBuildTimeStartNs) { + error("$metric was restarted before it finished") + } + myBuildTimeStartNs[metric] = startNs + } + + override fun endMeasure(metric: BuildTime, endNs: Long) { + val startNs = myBuildTimeStartNs.remove(metric) ?: error("$metric finished before it started") + val durationNs = endNs - startNs + myBuildTimes.add(metric, durationNs) + } + + override fun addAttribute(attribute: BuildAttribute) { + myBuildAttributes.add(attribute) + } + + override fun getMetrics(): BuildMetrics = + BuildMetrics( + buildTimes = myBuildTimes, + buildAttributes = myBuildAttributes + ) + + override fun addMetrics(metrics: BuildMetrics?) { + if (metrics == null) return + + myBuildAttributes.addAll(metrics.buildAttributes) + myBuildTimes.addAll(metrics.buildTimes) + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt new file mode 100644 index 00000000000..cd7b22996c7 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.io.Serializable + +@Suppress("Reformat") +enum class BuildTime(val parent: BuildTime? = null) : Serializable { + GRADLE_TASK, + CLEAR_OUTPUT(GRADLE_TASK), + BACKUP_OUTPUT(GRADLE_TASK), + RESTORE_OUTPUT_FROM_BACKUP(GRADLE_TASK), + CONNECT_TO_DAEMON(GRADLE_TASK), + CLEAR_JAR_CACHE(GRADLE_TASK), + RUN_COMPILER(GRADLE_TASK), + NON_INCREMENTAL_COMPILATION_IN_PROCESS(RUN_COMPILER), + NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILER), + NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILER), + INCREMENTAL_COMPILATION(RUN_COMPILER), + IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION), + IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET), + IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES), + IC_ANALYZE_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES), + IC_ANALYZE_CHANGES_IN_JAVA_SOURCES(IC_CALCULATE_INITIAL_DIRTY_SET), + IC_ANALYZE_CHANGES_IN_ANDROID_LAYOUTS(IC_CALCULATE_INITIAL_DIRTY_SET), + IC_DETECT_REMOVED_CLASSES(IC_CALCULATE_INITIAL_DIRTY_SET), + CLEAR_OUTPUT_ON_REBUILD(INCREMENTAL_COMPILATION), + IC_UPDATE_CACHES(INCREMENTAL_COMPILATION), + INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION), + NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION), + IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION); + + companion object { + const val serialVersionUID = 0L + + val children by lazy { + values().filter { it.parent != null }.groupBy { it.parent } + } + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt new file mode 100644 index 00000000000..a052c1af2c3 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTimes.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import java.io.Serializable +import java.util.* + +class BuildTimes : Serializable { + private val myBuildTimes = EnumMap(BuildTime::class.java) + + fun addAll(other: BuildTimes) { + for ((bt, timeNs) in other.myBuildTimes) { + add(bt, timeNs) + } + } + + fun add(buildTime: BuildTime, timeNs: Long) { + myBuildTimes[buildTime] = myBuildTimes.getOrDefault(buildTime, 0) + timeNs + } + + fun asMap(): Map = myBuildTimes + + companion object { + const val serialVersionUID = 0L + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt new file mode 100644 index 00000000000..500e6db1238 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/DoNothingBuildMetricsReporter.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +object DoNothingBuildMetricsReporter : BuildMetricsReporter { + override fun startMeasure(metric: BuildTime, startNs: Long) { + } + + override fun endMeasure(metric: BuildTime, endNs: Long) { + } + + override fun addAttribute(attribute: BuildAttribute) { + } + + override fun getMetrics(): BuildMetrics = + BuildMetrics( + BuildTimes(), + BuildAttributes() + ) + + override fun addMetrics(metrics: BuildMetrics?) {} +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/RemoteBuildMetricsReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/RemoteBuildMetricsReporter.kt new file mode 100644 index 00000000000..6ec8561954b --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/RemoteBuildMetricsReporter.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.metrics + +import org.jetbrains.kotlin.build.report.RemoteReporter + +interface RemoteBuildMetricsReporter : BuildMetricsReporter, + RemoteReporter \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt index 0f400f9fd99..74e2cd840a3 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt @@ -20,6 +20,7 @@ import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.build.JvmSourceRoot import org.jetbrains.kotlin.build.isModuleMappingFile +import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt index 60876993244..ef684aa0e9d 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt @@ -29,6 +29,7 @@ enum class CompilationResultCategory(val code: Int) { IC_COMPILE_ITERATION(0), BUILD_REPORT_LINES(1), VERBOSE_BUILD_REPORT_LINES(2), + BUILD_METRICS(3) } interface CompilationResultsAsync { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index 11e32d1147a..faa86159517 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -21,6 +21,9 @@ import com.intellij.openapi.util.Disposer import com.intellij.openapi.vfs.impl.ZipHandler import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.RemoteBuildReporter +import org.jetbrains.kotlin.build.report.RemoteReporter import org.jetbrains.kotlin.cli.common.CLICompiler import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY @@ -286,7 +289,7 @@ abstract class CompileServiceImplBase( createMessageCollector: (ServicesFacadeT, CompilationOptions) -> MessageCollector, createReporter: (ServicesFacadeT, CompilationOptions) -> DaemonMessageReporter, createServices: (JpsServicesFacadeT, EventManager, Profiler) -> Services, - getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteICReporter + getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteBuildReporter ) = kotlin.run { val messageCollector = createMessageCollector(servicesFacade, compilationOptions) val daemonReporter = createReporter(servicesFacade, compilationOptions) @@ -514,7 +517,7 @@ abstract class CompileServiceImplBase( args: K2JSCompilerArguments, incrementalCompilationOptions: IncrementalCompilationOptions, compilerMessageCollector: MessageCollector, - reporter: RemoteICReporter + reporter: RemoteBuildReporter ): ExitCode { val allKotlinFiles = arrayListOf() val freeArgsWithoutKotlinFiles = arrayListOf() @@ -554,7 +557,7 @@ abstract class CompileServiceImplBase( k2jvmArgs: K2JVMCompilerArguments, incrementalCompilationOptions: IncrementalCompilationOptions, compilerMessageCollector: MessageCollector, - reporter: RemoteICReporter + reporter: RemoteBuildReporter ): ExitCode { val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS + (incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct() @@ -784,7 +787,7 @@ class CompileServiceImpl( createMessageCollector = ::CompileServicesFacadeMessageCollector, createReporter = ::DaemonMessageReporter, createServices = this::createCompileServices, - getICReporter = { a, b, c -> getICReporter(a, b!!, c)} + getICReporter = { a, b, c -> getBuildReporter(a, b!!, c)} ) override fun leaseReplSession( diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/BuildReportICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/BuildReportICReporter.kt index 9dee507a3f0..e60c68fdeb0 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/BuildReportICReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/BuildReportICReporter.kt @@ -8,23 +8,19 @@ package org.jetbrains.kotlin.daemon.report import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.daemon.common.CompilationResultCategory import org.jetbrains.kotlin.daemon.common.CompilationResults -import org.jetbrains.kotlin.incremental.ICReporterBase +import org.jetbrains.kotlin.build.report.ICReporterBase +import org.jetbrains.kotlin.build.report.RemoteICReporter import java.io.File import java.util.* -import kotlin.collections.ArrayList +// todo: sync BuildReportICReporterAsync internal class BuildReportICReporter( private val compilationResults: CompilationResults, rootDir: File, - private val isVerbose: Boolean = false, - // todo: default value - // todo: sync BuildReportICReporterAsync - private val reportMetrics: Boolean = true + private val isVerbose: Boolean = false ) : ICReporterBase(rootDir), RemoteICReporter { private val icLogLines = arrayListOf() private val recompilationReason = HashMap() - private val rootMetric = Metric("", 0) - private val metrics = ArrayDeque().apply { add(rootMetric) } override fun report(message: () -> String) { icLogLines.add(message()) @@ -36,30 +32,6 @@ internal class BuildReportICReporter( } } - override fun startMeasure(metric: String, startNs: Long) { - if (!reportMetrics) return - - val newMetric = Metric(metric, startNs) - if (metrics.isNotEmpty()) { - metrics.peekLast().children.add(newMetric) - } - metrics.addLast(newMetric) - } - - override fun endMeasure(metric: String, endNs: Long) { - if (!reportMetrics) return - - while (metrics.isNotEmpty()) { - val lastMetric = metrics.peekLast() - if (lastMetric.name == metric) { - lastMetric.endNs = endNs - break - } else { - metrics.removeLast() - } - } - } - override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) { if (!incremental) return @@ -76,25 +48,6 @@ internal class BuildReportICReporter( } override fun flush() { - if (reportMetrics) { - icLogLines.add("Performance metrics:") - reportMetric(rootMetric) - } - compilationResults.add(CompilationResultCategory.BUILD_REPORT_LINES.code, icLogLines) } - - private fun reportMetric(metric: Metric, level: Int = 0) { - if (level > 0) { - val timeMs = metric.endNs?.let { (it - metric.startNs) / 1_000_000L } - icLogLines.add(" ".repeat(level) + "{perf_metric:${metric.name}} ${timeMs ?: ""} ms") - } - - metric.children.forEach { reportMetric(it, level + 1) } - } -} - -private class Metric(val name: String, val startNs: Long) { - var endNs: Long? = null - val children = ArrayList() } \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompileIterationICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompileIterationICReporter.kt index fe3bf2d6ba6..dd67aae6e91 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompileIterationICReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompileIterationICReporter.kt @@ -9,7 +9,8 @@ import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.daemon.common.CompilationResultCategory import org.jetbrains.kotlin.daemon.common.CompilationResults import org.jetbrains.kotlin.daemon.common.CompileIterationResult -import org.jetbrains.kotlin.incremental.ICReporterBase +import org.jetbrains.kotlin.build.report.ICReporterBase +import org.jetbrains.kotlin.build.report.RemoteICReporter import java.io.File internal class CompileIterationICReporter( diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompositeICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompositeICReporter.kt index 0aeb8c62b83..89e59dabe87 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompositeICReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompositeICReporter.kt @@ -5,11 +5,11 @@ package org.jetbrains.kotlin.daemon.report +import org.jetbrains.kotlin.build.report.RemoteICReporter import org.jetbrains.kotlin.cli.common.ExitCode import java.io.File -internal class CompositeICReporter(private val reporters: Iterable) : - RemoteICReporter { +internal class CompositeICReporter(private val reporters: Iterable) : RemoteICReporter { override fun report(message: () -> String) { reporters.forEach { it.report(message) } } @@ -34,14 +34,6 @@ internal class CompositeICReporter(private val reporters: Iterable() @@ -120,19 +126,26 @@ fun getICReporterAsync( .mapNotNullTo(HashSet()) { resultCode -> CompilationResultCategory.values().getOrNull(resultCode) } - requestedResults.mapTo(reporters) { requestedResult -> + for (requestedResult in requestedResults) { when (requestedResult) { CompilationResultCategory.IC_COMPILE_ITERATION -> { - CompileIterationICReporterAsync(compilationResults) + reporters.add(CompileIterationICReporterAsync(compilationResults)) } CompilationResultCategory.BUILD_REPORT_LINES -> { - BuildReportICReporterAsync(compilationResults, root) + reporters.add(BuildReportICReporterAsync(compilationResults, root)) } CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> { - BuildReportICReporterAsync(compilationResults, root, isVerbose = true) + reporters.add(BuildReportICReporterAsync(compilationResults, root, isVerbose = true)) } } } - return CompositeICReporter(reporters) + val icReporter = CompositeICReporter(reporters) + val metricsReporter = DoNothingRemoteBuildMetricsReporter + return RemoteBuildReporter(icReporter, metricsReporter) +} + +object DoNothingRemoteBuildMetricsReporter : BuildMetricsReporter by DoNothingBuildMetricsReporter, RemoteBuildMetricsReporter { + override fun flush() { + } } \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt index 02292783073..bdb583cd6c5 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt @@ -16,15 +16,18 @@ package org.jetbrains.kotlin.daemon.report +import org.jetbrains.kotlin.build.report.RemoteBuildReporter +import org.jetbrains.kotlin.build.report.RemoteICReporter +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter import org.jetbrains.kotlin.daemon.common.* -import java.io.File import java.util.* -fun getICReporter( +fun getBuildReporter( servicesFacade: CompilerServicesFacadeBase, compilationResults: CompilationResults, compilationOptions: IncrementalCompilationOptions -): RemoteICReporter { +): RemoteBuildReporter { val root = compilationOptions.modulesInfo.projectRoot val reporters = ArrayList() @@ -38,21 +41,25 @@ fun getICReporter( .mapNotNullTo(HashSet()) { resultCode -> CompilationResultCategory.values().getOrNull(resultCode) } - requestedResults.mapTo(reporters) { requestedResult -> + for (requestedResult in requestedResults) { when (requestedResult) { CompilationResultCategory.IC_COMPILE_ITERATION -> { - CompileIterationICReporter(compilationResults) + reporters.add(CompileIterationICReporter(compilationResults)) } CompilationResultCategory.BUILD_REPORT_LINES -> { - BuildReportICReporter(compilationResults, root) + reporters.add(BuildReportICReporter(compilationResults, root)) } CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> { - BuildReportICReporter(compilationResults, root, isVerbose = true) + reporters.add(BuildReportICReporter(compilationResults, root, isVerbose = true)) } } } + val areBuildMetricsNeeded = CompilationResultCategory.BUILD_METRICS in requestedResults + val metricsReporter = + (if (areBuildMetricsNeeded) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter) + .let { RemoteBuildMetricsReporterAdapter(it, areBuildMetricsNeeded, compilationResults) } - return CompositeICReporter(reporters) + return RemoteBuildReporter(CompositeICReporter(reporters), metricsReporter) } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt index 08b303bcf3b..0e45f818432 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt @@ -17,6 +17,7 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.annotations.TestOnly +import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.name.FqName import java.io.File import java.io.IOException diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt index 97f1b101549..307ecffee2f 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt @@ -19,6 +19,8 @@ package org.jetbrains.kotlin.incremental import com.intellij.psi.PsiClass import com.intellij.psi.PsiFile import com.intellij.psi.PsiJavaFile +import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import java.io.File import java.util.* @@ -37,7 +39,7 @@ internal class ChangedJavaFilesProcessor( if (removedJava.any()) { reporter.report { "Some java files are removed: [${removedJava.joinToString()}]" } - return ChangesEither.Unknown() + return ChangesEither.Unknown(BuildAttribute.JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED) } val symbols = HashSet() @@ -47,7 +49,7 @@ internal class ChangedJavaFilesProcessor( val psiFile = psiFileFactory(javaFile) if (psiFile !is PsiJavaFile) { reporter.report { "Expected PsiJavaFile, got ${psiFile?.javaClass}" } - return ChangesEither.Unknown() + return ChangesEither.Unknown(BuildAttribute.JAVA_CHANGE_UNEXPECTED_PSI) } psiFile.classes.forEach { it.addLookupSymbols(symbols) } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt index d1cd22bc253..cca15fc1d2b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.name.FqName internal sealed class ChangesEither { @@ -24,5 +25,5 @@ internal sealed class ChangesEither { val fqNames: Collection = emptyList() ) : ChangesEither() - internal class Unknown(val reason: String? = null) : ChangesEither() + internal class Unknown(val reason: BuildAttribute) : ChangesEither() } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/DirtyFilesContainer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/DirtyFilesContainer.kt index 7c38ae6ddf6..050851d5ff5 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/DirtyFilesContainer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/DirtyFilesContainer.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.name.FqName import java.io.File diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index d66f7a92a7e..c529463ac24 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index fef27c91c2b..ecdbf02d6b2 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -18,6 +18,10 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS import org.jetbrains.kotlin.build.GeneratedFile +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.cli.common.messages.MessageCollector @@ -41,7 +45,7 @@ abstract class IncrementalCompilerRunner< >( private val workingDir: File, cacheDirName: String, - protected val reporter: ICReporter, + protected val reporter: BuildReporter, private val buildHistoryFile: File, // there might be some additional output directories (e.g. for generated java in kapt) // to remove them correctly on rebuild, we pass them as additional argument @@ -65,20 +69,33 @@ abstract class IncrementalCompilerRunner< // otherwise we track source files changes ourselves. providedChangedFiles: ChangedFiles?, projectDir: File? = null + ): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION) { + compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir) + } + + private fun compileImpl( + allSourceFiles: List, + args: Args, + messageCollector: MessageCollector, + providedChangedFiles: ChangedFiles?, + projectDir: File? = null ): ExitCode { assert(isICEnabled()) { "Incremental compilation is not enabled" } var caches = createCacheManager(args, projectDir) - fun rebuild(reason: () -> String): ExitCode { - reporter.report(reason) + fun rebuild(reason: BuildAttribute): ExitCode { + reporter.report { "Non-incremental compilation will be performed: $reason" } caches.close(false) - clearLocalStateOnRebuild(args) + // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate + reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) { + clearLocalStateOnRebuild(args) + } caches = createCacheManager(args, projectDir) if (providedChangedFiles == null) { caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) } val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } - return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(), messageCollector) + return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector) } return try { @@ -90,16 +107,17 @@ abstract class IncrementalCompilerRunner< compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector) } is CompilationMode.Rebuild -> { - rebuild { "Non-incremental compilation will be performed: ${compilationMode.reason}" } + rebuild(compilationMode.reason) } } if (!caches.close(flush = true)) throw RuntimeException("Could not flush caches") return exitCode - } catch (e: Exception) { + } catch (e: Exception) { // todo: catch only cache corruption // todo: warn? - rebuild { "Possible cache corruption. Rebuilding. $e" } + reporter.report { "Rebuilding because of possible caches corruption: $e" } + rebuild(BuildAttribute.CACHE_CORRUPTION) } } @@ -139,10 +157,17 @@ abstract class IncrementalCompilerRunner< ): CompilationMode = when (changedFiles) { is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector) - is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" } + is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) } - protected abstract fun calculateSourcesToCompile( + private fun calculateSourcesToCompile( + caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector + ): CompilationMode = + reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) { + calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector) + } + + protected abstract fun calculateSourcesToCompileImpl( caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, @@ -161,9 +186,7 @@ abstract class IncrementalCompilerRunner< protected sealed class CompilationMode { class Incremental(val dirtyFiles: DirtyFilesContainer) : CompilationMode() - class Rebuild(getReason: () -> String = { "" }) : CompilationMode() { - val reason: String by lazy(getReason) - } + class Rebuild(val reason: BuildAttribute) : CompilationMode() } protected abstract fun updateCaches( @@ -211,9 +234,17 @@ abstract class IncrementalCompilerRunner< ): ExitCode { preBuildHook(args, compilationMode) + val buildTimeMode: BuildTime val dirtySources = when (compilationMode) { - is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableList() - is CompilationMode.Rebuild -> allKotlinSources.toMutableList() + is CompilationMode.Incremental -> { + buildTimeMode = BuildTime.INCREMENTAL_ITERATION + compilationMode.dirtyFiles.toMutableList() + } + is CompilationMode.Rebuild -> { + buildTimeMode = BuildTime.NON_INCREMENTAL_ITERATION + reporter.addAttribute(compilationMode.reason) + allKotlinSources.toMutableList() + } } val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis()) @@ -247,7 +278,9 @@ abstract class IncrementalCompilerRunner< val bufferingMessageCollector = BufferingMessageCollector() val messageCollectorAdapter = MessageCollectorToOutputItemsCollectorAdapter(bufferingMessageCollector, outputItemsCollector) - exitCode = runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter) + exitCode = reporter.measure(buildTimeMode) { + runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter) + } val generatedFiles = outputItemsCollector.outputs.map(SimpleOutputItem::toGeneratedFile) if (compilationMode is CompilationMode.Incremental) { @@ -268,12 +301,13 @@ abstract class IncrementalCompilerRunner< dirtySourcesSinceLastTimeFile.delete() - caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker) - caches.inputsCache.registerOutputForSourceFiles(generatedFiles) - caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) val changesCollector = ChangesCollector() - updateCaches(services, caches, generatedFiles, changesCollector) - + reporter.measure(BuildTime.IC_UPDATE_CACHES) { + caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker) + caches.inputsCache.registerOutputForSourceFiles(generatedFiles) + caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) + updateCaches(services, caches, generatedFiles, changesCollector) + } if (compilationMode is CompilationMode.Rebuild) break val (dirtyLookupSymbols, dirtyClassFqNames) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter) @@ -339,7 +373,7 @@ abstract class IncrementalCompilerRunner< compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData - ) { + ) = reporter.measure(BuildTime.IC_WRITE_HISTORY_FILE) { val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList() val newDiff = if (compilationMode is CompilationMode.Incremental) { BuildDifference(currentBuildInfo.startTS, true, dirtyData) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 51d66a2d573..09c17ebfd2f 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -17,6 +17,10 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.build.GeneratedFile +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.isIrBackendEnabled @@ -45,9 +49,10 @@ fun makeJsIncrementally( .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList() val buildHistoryFile = File(cachesDir, "build-history.bin") + val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter) withJsIC { val compiler = IncrementalJsCompilerRunner( - cachesDir, reporter, + cachesDir, buildReporter, buildHistoryFile = buildHistoryFile, modulesApiHistory = EmptyModulesApiHistory, scopeExpansion = scopeExpansion @@ -69,7 +74,7 @@ inline fun withJsIC(fn: () -> R): R { class IncrementalJsCompilerRunner( private val workingDir: File, - reporter: ICReporter, + reporter: BuildReporter, buildHistoryFile: File, private val modulesApiHistory: ModulesApiHistory, private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER @@ -90,14 +95,14 @@ class IncrementalJsCompilerRunner( override fun destinationDir(args: K2JSCompilerArguments): File = File(args.outputFile).parentFile - override fun calculateSourcesToCompile( + override fun calculateSourcesToCompileImpl( caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments, messageCollector: MessageCollector ): CompilationMode { val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) - ?: return CompilationMode.Rebuild { "No information on previous build" } + ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) initDirtyFiles(dirtyFiles, changedFiles) @@ -107,9 +112,9 @@ class IncrementalJsCompilerRunner( @Suppress("UNUSED_VARIABLE") // for sealed when val unused = when (classpathChanges) { - is ChangesEither.Unknown -> return CompilationMode.Rebuild { - // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate - "Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}" + is ChangesEither.Unknown -> { + reporter.report { "Could not get classpath's changes: ${classpathChanges.reason}" } + return CompilationMode.Rebuild(classpathChanges.reason) } is ChangesEither.Known -> { dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols) @@ -117,7 +122,6 @@ class IncrementalJsCompilerRunner( } } - val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles) dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols) dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNames) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 5ba09291e97..c85f8e6d573 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -25,6 +25,10 @@ import com.intellij.psi.PsiJavaFile import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.GeneratedJvmClass +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.build.report.ICReporterBase +import org.jetbrains.kotlin.build.report.metrics.* import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments @@ -64,11 +68,12 @@ fun makeIncrementally( val sourceFiles = files.filter { it.extension.toLowerCase() in allExtensions }.toList() val buildHistoryFile = File(cachesDir, "build-history.bin") args.javaSourceRoots = sourceRoots.map { it.absolutePath }.toTypedArray() + val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter) withIC { val compiler = IncrementalJvmCompilerRunner( cachesDir, - reporter, + buildReporter, // Use precise setting in case of non-Gradle build usePreciseJavaTracking = true, outputFiles = emptyList(), @@ -104,7 +109,7 @@ inline fun withIC(enabled: Boolean = true, fn: () -> R): R { class IncrementalJvmCompilerRunner( workingDir: File, - reporter: ICReporter, + reporter: BuildReporter, private val usePreciseJavaTracking: Boolean, buildHistoryFile: File, outputFiles: Collection, @@ -154,7 +159,7 @@ class IncrementalJvmCompilerRunner( else null - override fun calculateSourcesToCompile( + override fun calculateSourcesToCompileImpl( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments, @@ -176,16 +181,20 @@ class IncrementalJvmCompilerRunner( val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) initDirtyFiles(dirtyFiles, changedFiles) - val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild { "No information on previous build" } + val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" } - val classpathChanges = getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter) + val classpathChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { + getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter) + } @Suppress("UNUSED_VARIABLE") // for sealed when val unused = when (classpathChanges) { - is ChangesEither.Unknown -> return CompilationMode.Rebuild { - // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate - "Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}" + is ChangesEither.Unknown -> { + reporter.report { + "Could not get classpath's changes: ${classpathChanges.reason}" + } + return CompilationMode.Rebuild(classpathChanges.reason) } is ChangesEither.Known -> { dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols) @@ -194,21 +203,26 @@ class IncrementalJvmCompilerRunner( } } - if (!usePreciseJavaTracking) { - val javaFilesChanges = javaFilesProcessor!!.process(changedFiles) - val affectedJavaSymbols = when (javaFilesChanges) { - is ChangesEither.Known -> javaFilesChanges.lookupSymbols - is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" } - } - dirtyFiles.addByDirtySymbols(affectedJavaSymbols) - } else { - if (!processChangedJava(changedFiles, caches)) { - return CompilationMode.Rebuild { "Could not get changes for java files" } + reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_JAVA_SOURCES) { + if (!usePreciseJavaTracking) { + val javaFilesChanges = javaFilesProcessor!!.process(changedFiles) + val affectedJavaSymbols = when (javaFilesChanges) { + is ChangesEither.Known -> javaFilesChanges.lookupSymbols + is ChangesEither.Unknown -> return CompilationMode.Rebuild(javaFilesChanges.reason) + } + dirtyFiles.addByDirtySymbols(affectedJavaSymbols) + } else { + val rebuildReason = processChangedJava(changedFiles, caches) + if (rebuildReason != null) return CompilationMode.Rebuild(rebuildReason) } } - val androidLayoutChanges = processLookupSymbolsForAndroidLayouts(changedFiles) - val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles) + val androidLayoutChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_ANDROID_LAYOUTS) { + processLookupSymbolsForAndroidLayouts(changedFiles) + } + val removedClassesChanges = reporter.measure(BuildTime.IC_DETECT_REMOVED_CLASSES) { + getRemovedClassesChanges(caches, changedFiles) + } dirtyFiles.addByDirtySymbols(androidLayoutChanges) dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols) @@ -217,7 +231,7 @@ class IncrementalJvmCompilerRunner( return CompilationMode.Incremental(dirtyFiles) } - private fun processChangedJava(changedFiles: ChangedFiles.Known, caches: IncrementalJvmCachesManager): Boolean { + private fun processChangedJava(changedFiles: ChangedFiles.Known, caches: IncrementalJvmCachesManager): BuildAttribute? { val javaFiles = (changedFiles.modified + changedFiles.removed).filter(File::isJavaFile) for (javaFile in javaFiles) { @@ -225,20 +239,20 @@ class IncrementalJvmCompilerRunner( if (!javaFile.exists()) { // todo: can we do this more optimal? reporter.report { "Could not get changed for untracked removed java file $javaFile" } - return false + return BuildAttribute.JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED } val psiFile = psiFileProvider.javaFile(javaFile) if (psiFile !is PsiJavaFile) { reporter.report { "[Precise Java tracking] Expected PsiJavaFile, got ${psiFile?.javaClass}" } - return false + return BuildAttribute.JAVA_CHANGE_UNEXPECTED_PSI } for (psiClass in psiFile.classes) { val qualifiedName = psiClass.qualifiedName if (qualifiedName == null) { reporter.report { "[Precise Java tracking] Class with unknown qualified name in $javaFile" } - return false + return BuildAttribute.JAVA_CHANGE_UNKNOWN_QUALIFIER } processChangedUntrackedJavaClass(psiClass, ClassId.topLevel(FqName(qualifiedName))) @@ -247,7 +261,7 @@ class IncrementalJvmCompilerRunner( } caches.platformCache.markDirty(javaFiles) - return true + return null } private fun processChangedUntrackedJavaClass(psiClass: PsiClass, classId: ClassId) { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt index 0bc29062d6b..fc4bb78c93c 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt @@ -18,6 +18,7 @@ package org.jetbrains.kotlin.incremental import com.intellij.util.containers.MultiMap import org.jetbrains.kotlin.build.GeneratedFile +import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner import org.jetbrains.kotlin.incremental.storage.SourceToOutputFilesMap diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt index 4b7257a3bdf..290712bf01b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt @@ -5,6 +5,10 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory import org.jetbrains.kotlin.incremental.util.Either import org.jetbrains.kotlin.name.FqName @@ -15,7 +19,7 @@ internal fun getClasspathChanges( changedFiles: ChangedFiles.Known, lastBuildInfo: BuildInfo, modulesApiHistory: ModulesApiHistory, - reporter: ICReporter? + reporter: BuildReporter ): ChangesEither { val classpathSet = HashSet() for (file in classpath) { @@ -29,7 +33,10 @@ internal fun getClasspathChanges( val removedClasspath = changedFiles.removed.filterTo(HashSet()) { it in classpathSet } // todo: removed classes could be processed normally - if (removedClasspath.isNotEmpty()) return ChangesEither.Unknown("Some files are removed from classpath $removedClasspath") + if (removedClasspath.isNotEmpty()) { + reporter.report { "Some files are removed from classpath: $removedClasspath" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_REMOVED_ENTRY) + } if (modifiedClasspath.isEmpty()) return ChangesEither.Known() @@ -38,28 +45,47 @@ internal fun getClasspathChanges( val symbols = HashSet() val fqNames = HashSet() - val historyFilesEither = modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath) + val historyFilesEither = + reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) { + modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath) + } val historyFiles = when (historyFilesEither) { is Either.Success> -> historyFilesEither.value - is Either.Error -> return ChangesEither.Unknown(historyFilesEither.reason) - } - - for (historyFile in historyFiles) { - val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter) - ?: return ChangesEither.Unknown("Could not read diffs from $historyFile") - val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS } - if (knownBuilds.isEmpty()) { - return ChangesEither.Unknown("No previously known builds for $historyFile") - } - - for (buildDiff in newBuilds) { - if (!buildDiff.isIncremental) return ChangesEither.Unknown("Non-incremental build from dependency $historyFile") - - val dirtyData = buildDiff.dirtyData - symbols.addAll(dirtyData.dirtyLookupSymbols) - fqNames.addAll(dirtyData.dirtyClassesFqNames) + is Either.Error -> { + reporter.report { "Could not find history files: ${historyFilesEither.reason}" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND) } } - return ChangesEither.Known(symbols, fqNames) + fun analyzeHistoryFiles(): ChangesEither { + for (historyFile in historyFiles) { + val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter) + ?: return run { + reporter.report { "Could not read diffs from $historyFile" } + ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ) + } + val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS } + if (knownBuilds.isEmpty()) { + reporter.report { "No previously known builds for $historyFile" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS) + } + + for (buildDiff in newBuilds) { + if (!buildDiff.isIncremental) { + reporter.report { "Non-incremental build from dependency $historyFile" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP) + + } + val dirtyData = buildDiff.dirtyData + symbols.addAll(dirtyData.dirtyLookupSymbols) + fqNames.addAll(dirtyData.dirtyClassesFqNames) + } + } + + return ChangesEither.Known(symbols, fqNames) + } + + return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) { + analyzeHistoryFiles() + } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/utils/TestICReporter.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/utils/TestICReporter.kt index 68b39eccfb7..2c224ec38f7 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/utils/TestICReporter.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/utils/TestICReporter.kt @@ -17,7 +17,7 @@ package org.jetbrains.kotlin.incremental.utils import org.jetbrains.kotlin.cli.common.ExitCode -import org.jetbrains.kotlin.incremental.ICReporterBase +import org.jetbrains.kotlin.build.report.ICReporterBase import java.io.File class TestICReporter : ICReporterBase() { diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 887425b0058..d0b97301a60 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -42,7 +42,7 @@ import org.jetbrains.kotlin.daemon.common.isDaemonEnabled import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.incremental.ICReporterBase +import org.jetbrains.kotlin.build.report.ICReporterBase import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager import org.jetbrains.kotlin.jps.model.kotlinKind diff --git a/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt b/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt index caa6b693e16..c180357c285 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt @@ -18,6 +18,8 @@ import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel import org.jetbrains.kotlin.backend.common.serialization.KlibIrVersion import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles @@ -316,7 +318,7 @@ class GenerateIrRuntime { withJsIC { val buildHistoryFile = File(cachesDir, "build-history.bin") val compiler = IncrementalJsCompilerRunner( - cachesDir, EmptyICReporter, + cachesDir, BuildReporter(EmptyICReporter, DoNothingBuildMetricsReporter), buildHistoryFile = buildHistoryFile, modulesApiHistory = EmptyModulesApiHistory ) @@ -360,7 +362,7 @@ class GenerateIrRuntime { withJsIC { val buildHistoryFile = File(cachesDir, "build-history.bin") val compiler = IncrementalJsCompilerRunner( - cachesDir, EmptyICReporter, + cachesDir, BuildReporter(EmptyICReporter, DoNothingBuildMetricsReporter), buildHistoryFile = buildHistoryFile, modulesApiHistory = EmptyModulesApiHistory ) diff --git a/libraries/tools/kotlin-gradle-build-metrics/build.gradle.kts b/libraries/tools/kotlin-gradle-build-metrics/build.gradle.kts new file mode 100644 index 00000000000..dfb8ccb401c --- /dev/null +++ b/libraries/tools/kotlin-gradle-build-metrics/build.gradle.kts @@ -0,0 +1,24 @@ +description = "kotlin-gradle-statistics" + +plugins { + kotlin("jvm") + id("jps-compatible") +} + +dependencies { + compileOnly(kotlinStdlib()) + + testImplementation(project(":kotlin-test:kotlin-test-junit")) + testImplementation("junit:junit:4.12") +} + +sourceSets { + "main" { projectDefault() } + "test" { projectDefault() } +} + +projectTest { + workingDir = rootDir +} + +publish() \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt b/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt new file mode 100644 index 00000000000..b7596751395 --- /dev/null +++ b/libraries/tools/kotlin-gradle-build-metrics/src/org/jetbrains/kotlin/gradle/internal/build/metrics/GradleBuildMetricsData.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.build.metrics + +import java.io.Serializable + +class GradleBuildMetricsData : Serializable { + val parentMetric: MutableMap = LinkedHashMap() + val buildAttributeKind: MutableMap = LinkedHashMap() + val taskData: MutableMap = LinkedHashMap() + + companion object { + const val serialVersionUID = 0L + } +} + +data class TaskData( + val path: String, + val typeFqName: String, + val timeMetrics: Map, + val buildAttributes: Map, + val didWork: Boolean +) : Serializable { + + companion object { + const val serialVersionUID = 0L + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index 1dfdc7ef9ff..df29e83b625 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -52,6 +52,8 @@ dependencies { compileOnly(project(":kotlin-scripting-compiler")) compileOnly(project(":kotlin-gradle-statistics")) embedded(project(":kotlin-gradle-statistics")) + compileOnly(project(":kotlin-gradle-build-metrics")) + embedded(project(":kotlin-gradle-build-metrics")) compile("com.google.code.gson:gson:${rootProject.extra["versions.jar.gson"]}") compile("de.undercouch:gradle-download-task:4.0.2") diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt index d2a102af802..710b625d8b6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt @@ -1,5 +1,7 @@ package org.jetbrains.kotlin.compilerRunner +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl import org.jetbrains.kotlin.daemon.common.CompilationResultCategory import org.jetbrains.kotlin.daemon.common.CompilationResults import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface @@ -22,7 +24,10 @@ internal class GradleCompilationResults( LoopbackNetworkInterface.serverLoopbackSocketFactory ) { - var icLogLines: List? = null + var icLogLines: List = emptyList() + private val buildMetricsReporter = BuildMetricsReporterImpl() + val buildMetrics: BuildMetrics + get() = buildMetricsReporter.getMetrics() @Throws(RemoteException::class) override fun add(compilationResultCategory: Int, value: Serializable) { @@ -42,7 +47,10 @@ internal class GradleCompilationResults( CompilationResultCategory.BUILD_REPORT_LINES.code, CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES.code -> { @Suppress("UNCHECKED_CAST") - icLogLines = value as? List + (value as? List)?.let { icLogLines = it } + } + CompilationResultCategory.BUILD_METRICS.code -> { + buildMetricsReporter.addMetrics(value as? BuildMetrics) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt index a731708da43..78b76bc1270 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt @@ -8,7 +8,7 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.file.FileCollection import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector -import org.jetbrains.kotlin.gradle.report.BuildReportMode +import org.jetbrains.kotlin.gradle.report.ReportingSettings import org.jetbrains.kotlin.gradle.tasks.findToolsJar import java.io.File @@ -17,7 +17,7 @@ internal class GradleCompilerEnvironment( messageCollector: GradlePrintingMessageCollector, outputItemsCollector: OutputItemsCollector, val outputFiles: FileCollection, - val buildReportMode: BuildReportMode?, + val reportingSettings: ReportingSettings, val incrementalCompilationEnvironment: IncrementalCompilationEnvironment? = null, val kotlinScriptExtensions: Array = emptyArray() ) : CompilerEnvironment(Services.EMPTY, messageCollector, outputItemsCollector) { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt index d0706b9f030..5bbb9eacf11 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt @@ -160,7 +160,7 @@ internal open class GradleCompilerRunner(protected val taskProvider: GradleCompi incrementalModuleInfo = modulesInfo, outputFiles = environment.outputFiles.toList(), taskPath = pathProvider, - buildReportMode = environment.buildReportMode, + reportingSettings = environment.reportingSettings, kotlinScriptExtensions = environment.kotlinScriptExtensions, allWarningsAsErrors = compilerArgs.allWarningsAsErrors ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index e728e2bc51d..bd601c26920 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.Project import org.gradle.api.logging.Logger +import org.jetbrains.kotlin.build.ExecutionStrategy +import org.jetbrains.kotlin.build.report.metrics.* import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.config.Services @@ -15,6 +17,7 @@ import org.jetbrains.kotlin.gradle.logging.* import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.report.BuildReportMode +import org.jetbrains.kotlin.gradle.report.ReportingSettings import org.jetbrains.kotlin.gradle.report.TaskExecutionResult import org.jetbrains.kotlin.gradle.tasks.clearLocalState import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError @@ -59,7 +62,7 @@ internal class GradleKotlinCompilerWorkArguments( val incrementalModuleInfo: IncrementalModuleInfo?, val outputFiles: List, val taskPath: String, - val buildReportMode: BuildReportMode?, + val reportingSettings: ReportingSettings, val kotlinScriptExtensions: Array, val allWarningsAsErrors: Boolean ) : Serializable { @@ -98,10 +101,12 @@ internal class GradleKotlinCompilerWork @Inject constructor( private val incrementalModuleInfo = config.incrementalModuleInfo private val outputFiles = config.outputFiles private val taskPath = config.taskPath - private val buildReportMode = config.buildReportMode + private val reportingSettings = config.reportingSettings private val kotlinScriptExtensions = config.kotlinScriptExtensions private val allWarningsAsErrors = config.allWarningsAsErrors private val buildDir = config.projectFiles.buildDir + private val metrics = if (reportingSettings.reportMetrics) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter + private var icLogLines: List = emptyList() private val log: KotlinLogger = TaskLoggers.get(taskPath)?.let { GradleKotlinLogger(it).apply { debug("Using '$taskPath' logger") } } @@ -120,13 +125,18 @@ internal class GradleKotlinCompilerWork @Inject constructor( get() = incrementalCompilationEnvironment != null override fun run() { - val messageCollector = GradlePrintingMessageCollector(log, allWarningsAsErrors) - val exitCode = compileWithDaemonOrFallbackImpl(messageCollector) - if (incrementalCompilationEnvironment?.disableMultiModuleIC == true) { - incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete() - } + try { + val messageCollector = GradlePrintingMessageCollector(log, allWarningsAsErrors) + val exitCode = compileWithDaemonOrFallbackImpl(messageCollector) + if (incrementalCompilationEnvironment?.disableMultiModuleIC == true) { + incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete() + } - throwGradleExceptionIfError(exitCode) + throwGradleExceptionIfError(exitCode) + } finally { + val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines) + TaskExecutionResults[taskPath] = result + } } private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): ExitCode { @@ -159,20 +169,21 @@ internal class GradleKotlinCompilerWork @Inject constructor( val isDebugEnabled = log.isDebugEnabled || System.getProperty("kotlin.daemon.debug.log")?.toBoolean() ?: true val daemonMessageCollector = if (isDebugEnabled) messageCollector else MessageCollector.NONE - val connection = - try { - GradleCompilerRunner.getDaemonConnectionImpl( - clientIsAliveFlagFile, - sessionFlagFile, - compilerFullClasspath, - daemonMessageCollector, - isDebugEnabled = isDebugEnabled - ) - } catch (e: Throwable) { - log.error("Caught an exception trying to connect to Kotlin Daemon:") - log.error(e.stackTraceAsString()) - null + metrics.measure(BuildTime.CONNECT_TO_DAEMON) { + try { + GradleCompilerRunner.getDaemonConnectionImpl( + clientIsAliveFlagFile, + sessionFlagFile, + compilerFullClasspath, + daemonMessageCollector, + isDebugEnabled = isDebugEnabled + ) + } catch (e: Throwable) { + log.error("Caught an exception trying to connect to Kotlin Daemon:") + log.error(e.stackTraceAsString()) + null + } } if (connection == null) { if (isIncremental) { @@ -215,7 +226,9 @@ internal class GradleKotlinCompilerWork @Inject constructor( // often source of the NoSuchObjectException and UnmarshalException, probably caused by the failed/crashed/exited daemon // TODO: implement a proper logic to avoid remote calls in such cases try { - daemon.clearJarCache() + metrics.measure(BuildTime.CLEAR_JAR_CACHE) { + daemon.clearJarCache() + } } catch (e: RemoteException) { log.warn("Unable to clear jar cache after compilation, maybe daemon is already down: $e") } @@ -229,6 +242,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( targetPlatform: CompileService.TargetPlatform, bufferingMessageCollector: GradleBufferingMessageCollector ): CompileService.CallResult { + metrics.addAttribute(BuildAttribute.IC_IS_NOT_ENABLED) val compilationOptions = CompilationOptions( compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER, targetPlatform = targetPlatform, @@ -238,15 +252,8 @@ internal class GradleKotlinCompilerWork @Inject constructor( kotlinScriptExtensions = kotlinScriptExtensions ) val servicesFacade = GradleCompilerServicesFacadeImpl(log, bufferingMessageCollector) - return try { + return metrics.measure(BuildTime.NON_INCREMENTAL_COMPILATION_DAEMON) { daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults = null) - } finally { - reportExecutionResultIfNeeded { - TaskExecutionResult( - executionStrategy = DAEMON_EXECUTION_STRATEGY, - icLogLines = nonIcBuildLog("incremental compilation is not enabled for '$taskPath'") - ) - } } } @@ -258,14 +265,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( ): CompileService.CallResult { val icEnv = incrementalCompilationEnvironment ?: error("incrementalCompilationEnvironment is null!") val knownChangedFiles = icEnv.changedFiles as? ChangedFiles.Known - - val requestedCompilationResults = EnumSet.of(CompilationResultCategory.IC_COMPILE_ITERATION) - when (buildReportMode) { - BuildReportMode.SIMPLE -> CompilationResultCategory.BUILD_REPORT_LINES - BuildReportMode.VERBOSE -> CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES - null -> null - }?.let { requestedCompilationResults.add(it) } - + val requestedCompilationResults = requestedCompilationResults() val compilationOptions = IncrementalCompilationOptions( areFileChangesKnown = knownChangedFiles != null, modifiedFiles = knownChangedFiles?.modified, @@ -286,36 +286,28 @@ internal class GradleKotlinCompilerWork @Inject constructor( log.info("Options for KOTLIN DAEMON: $compilationOptions") val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(log, bufferingMessageCollector) val compilationResults = GradleCompilationResults(log, projectRootFile) - val result = daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults) - - reportExecutionResultIfNeeded { - TaskExecutionResult( - executionStrategy = DAEMON_EXECUTION_STRATEGY, - icLogLines = compilationResults.icLogLines - ) + return metrics.measure(BuildTime.RUN_COMPILER) { + daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults) + }.also { + metrics.addMetrics(compilationResults.buildMetrics) + icLogLines = compilationResults.icLogLines } - - return result } private fun compileOutOfProcess(): ExitCode { - clearLocalState(outputFiles, log, reason = "out-of-process execution strategy is non-incremental") + metrics.addAttribute(BuildAttribute.OUT_OF_PROCESS_EXECUTION) + clearLocalState(outputFiles, log, metrics, reason = "out-of-process execution strategy is non-incremental") - return try { + return metrics.measure(BuildTime.NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS) { runToolInSeparateProcess(compilerArgs, compilerClassName, compilerFullClasspath, log, buildDir) - } finally { - reportExecutionResultIfNeeded { - TaskExecutionResult( - executionStrategy = OUT_OF_PROCESS_EXECUTION_STRATEGY, - icLogLines = nonIcBuildLog("$OUT_OF_PROCESS_EXECUTION_STRATEGY execution strategy does not support incremental compilation") - ) - } } } private fun compileInProcess(messageCollector: MessageCollector): ExitCode { - clearLocalState(outputFiles, log, reason = "in-process execution strategy is non-incremental") + metrics.addAttribute(BuildAttribute.IN_PROCESS_EXECUTION) + clearLocalState(outputFiles, log, metrics, reason = "in-process execution strategy is non-incremental") + metrics.startMeasure(BuildTime.NON_INCREMENTAL_COMPILATION_IN_PROCESS, System.nanoTime()) // in-process compiler should always be run in a different thread // to avoid leaking thread locals from compiler (see KT-28037) val threadPool = Executors.newSingleThreadExecutor() @@ -329,12 +321,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( bufferingMessageCollector.flush(messageCollector) threadPool.shutdown() - reportExecutionResultIfNeeded { - TaskExecutionResult( - executionStrategy = IN_PROCESS_EXECUTION_STRATEGY, - icLogLines = nonIcBuildLog("$IN_PROCESS_EXECUTION_STRATEGY execution strategy does not support incremental compilation") - ) - } + metrics.endMeasure(BuildTime.NON_INCREMENTAL_COMPILATION_IN_PROCESS, System.nanoTime()) } } @@ -366,6 +353,19 @@ internal class GradleKotlinCompilerWork @Inject constructor( return exitCode } + private fun requestedCompilationResults(): EnumSet { + val requestedCompilationResults = EnumSet.of(CompilationResultCategory.IC_COMPILE_ITERATION) + when (reportingSettings.buildReportMode) { + BuildReportMode.NONE -> null + BuildReportMode.SIMPLE -> CompilationResultCategory.BUILD_REPORT_LINES + BuildReportMode.VERBOSE -> CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES + }?.let { requestedCompilationResults.add(it) } + if (reportingSettings.reportMetrics) { + requestedCompilationResults.add(CompilationResultCategory.BUILD_METRICS) + } + return requestedCompilationResults + } + private fun reportCategories(verbose: Boolean): Array = if (!verbose) { arrayOf(ReportCategory.COMPILER_MESSAGE.code) @@ -379,14 +379,4 @@ internal class GradleKotlinCompilerWork @Inject constructor( } else { ReportSeverity.DEBUG.code } - - private inline fun reportExecutionResultIfNeeded(fn: () -> TaskExecutionResult) { - if (buildReportMode != null) { - val result = fn() - TaskExecutionResults[taskPath] = result - } - } - - private fun nonIcBuildLog(reason: String): List = - listOf("Performing non-incremental build: $reason") } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt index 178c2f1aa0c..f32602b40af 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt @@ -5,6 +5,8 @@ import org.gradle.api.file.FileCollection import org.gradle.api.internal.ConventionTask import org.gradle.api.tasks.* import org.gradle.api.tasks.incremental.IncrementalTaskInputs +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl import org.jetbrains.kotlin.gradle.internal.kapt.incremental.ClasspathSnapshot import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptClasspathChanges import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges @@ -137,6 +139,10 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState { protected val javaSourceRoots: Set get() = unfilteredJavaSourceRoots.filterTo(HashSet(), ::isRootAllowed) + @get:Internal + override val metrics: BuildMetricsReporter = + BuildMetricsReporterImpl() + private fun isRootAllowed(file: File): Boolean = file.exists() && !isAncestor(destinationDir, file) && diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt index b1d21e872fa..4a110c6667e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt @@ -84,7 +84,7 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput +) : BuildAdapter(), TaskExecutionListener { + private val taskRecords = HashMap() + + @Synchronized + override fun beforeExecute(task: Task) { + val startNs = System.nanoTime() + taskRecords[task] = TaskRecord(task, startNs) + } + + @Synchronized + override fun afterExecute(task: Task, state: TaskState) { + taskRecords[task]?.apply { processResult(state, TaskExecutionResults[task.path]) } + } + + override fun buildFinished(result: BuildResult) { + val startParams = arrayListOf() + gradle.startParameter.apply { + startParams.add("tasks = ${taskRequests.joinToString { it.args.toString() }}") + startParams.add("excluded tasks = $excludedTaskNames") + startParams.add("current dir = $currentDir") + startParams.add("project properties args = $projectProperties") + startParams.add("system properties args = $systemPropertiesArgs") + } + val buildData = BuildExecutionData( + startParameters = startParams, + failure = result.failure, + taskExecutionData = taskRecords.values.sortedBy { it.startNs } + ) + + buildDataProcessors.forEach { it.process(buildData) } + } +} + +private class TaskRecord( + override val task: Task, + override val startNs: Long +) : TaskExecutionData { + private var myEndNs: Long = 0 + private lateinit var myTaskState: TaskState + private var myIcLogLines: List = emptyList() + private val myBuildMetrics = BuildMetrics() + + override val isKotlinTask by lazy { + task.javaClass.name.startsWith("org.jetbrains.kotlin") + } + + override val endNs: Long + get() = myEndNs + + override val totalTimeNs: Long + get() = endNs - startNs + + override val resultState: TaskState + get() = myTaskState + + override val icLogLines: List + get() = myIcLogLines + + override val buildMetrics: BuildMetrics + get() = myBuildMetrics + + fun processResult(state: TaskState, executionResult: TaskExecutionResult?) { + myEndNs = System.nanoTime() + myTaskState = state + myBuildMetrics.buildTimes.add(BuildTime.GRADLE_TASK, totalTimeNs) + + if (executionResult != null) { + myBuildMetrics.addAll(executionResult.buildMetrics) + myIcLogLines = executionResult.icLogLines + } + + if (task is TaskWithLocalState) { + myBuildMetrics.addAll(task.metrics.getMetrics()) + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportMode.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportMode.kt index 4891949e858..5f5ea2e5611 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportMode.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportMode.kt @@ -5,7 +5,14 @@ package org.jetbrains.kotlin.gradle.report -enum class BuildReportMode { +import java.io.Serializable + +enum class BuildReportMode : Serializable { + NONE, SIMPLE, - VERBOSE -} \ No newline at end of file + VERBOSE; + + companion object { + const val serialVersionUID: Long = 0 + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporter.kt deleted file mode 100644 index c819d6668ed..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporter.kt +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.report - -import org.gradle.BuildAdapter -import org.gradle.BuildResult -import org.gradle.api.Task -import org.gradle.api.execution.TaskExecutionListener -import org.gradle.api.invocation.Gradle -import org.gradle.api.logging.Logger -import org.gradle.api.tasks.TaskState -import org.jetbrains.kotlin.gradle.logging.kotlinDebug -import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider -import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults -import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile -import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable -import java.io.File -import java.text.SimpleDateFormat -import java.util.* - -internal fun configureBuildReporter(gradle: Gradle, log: Logger) { - val rootProject = gradle.rootProject - val properties = PropertiesProvider(rootProject) - - if (!properties.buildReportEnabled) return - - val perfLogDir = properties.buildReportDir - ?: rootProject.buildDir.resolve("reports/kotlin-build").apply { mkdirs() } - - if (perfLogDir.isFile) { - log.error("Kotlin build report cannot be created: '$perfLogDir' is a file") - return - } - - perfLogDir.mkdirs() - val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time) - - val perfReportFile = perfLogDir.resolve("${gradle.rootProject.name}-build-$ts.txt") - - if (!isConfigurationCacheAvailable(gradle)) { - val reporter = KotlinBuildReporter(gradle, perfReportFile) - gradle.addBuildListener(reporter) - } - - val buildReportMode = if (properties.buildReportVerbose) BuildReportMode.VERBOSE else BuildReportMode.SIMPLE - - gradle.taskGraph.whenReady { graph -> - graph.allTasks.asSequence() - .filterIsInstance>() - .forEach { it.buildReportMode = buildReportMode } - } - - log.kotlinDebug { "Configured Kotlin build reporter" } -} - -internal class KotlinBuildReporter( - private val gradle: Gradle, - private val perfReportFile: File -) : BuildAdapter(), TaskExecutionListener { - init { - val dir = perfReportFile.parentFile - check(dir.isDirectory) { "$dir does not exist or is a file" } - check(!perfReportFile.isFile) { "Build report log file $perfReportFile exists already" } - } - - private val taskStartNs = HashMap() - private val kotlinTaskTimeNs = HashMap() - private val tasksSb = StringBuilder() - - @Volatile - private var allTasksTimeNs: Long = 0L - - @Synchronized - override fun beforeExecute(task: Task) { - taskStartNs[task] = System.nanoTime() - } - - @Synchronized - override fun afterExecute(task: Task, state: TaskState) { - val startNs = taskStartNs[task] ?: return - - val endNs = System.nanoTime() - val timeNs = endNs - startNs - allTasksTimeNs += timeNs - - if (!task.javaClass.name.startsWith("org.jetbrains.kotlin")) { - return - } - - kotlinTaskTimeNs[task] = timeNs - tasksSb.appendln() - - val skipMessage = state.skipMessage - if (skipMessage != null) { - tasksSb.appendln("$task was skipped: $skipMessage") - } else { - tasksSb.appendln("$task finished in ${formatTime(timeNs)}") - } - - val path = task.path - val executionResult = TaskExecutionResults[path] - if (executionResult != null) { - tasksSb.appendln("Execution strategy: ${executionResult.executionStrategy}") - - executionResult.icLogLines?.let { lines -> - tasksSb.appendln("Compilation log for $task:") - lines.forEach { tasksSb.appendln(" $it") } - } - } - } - - @Synchronized - override fun buildFinished(result: BuildResult) { - KotlinBuildReporterHandler().buildFinished(gradle, perfReportFile, kotlinTaskTimeNs.mapKeys{it.key.path}, allTasksTimeNs, result.failure) - } -} - diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterListener.kt index 6d95799f3f9..9c9e06b7fdb 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterListener.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.gradle.report -import com.intellij.util.text.DateFormatUtil.formatTime import org.gradle.api.invocation.Gradle import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.OperationCompletionListener diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt new file mode 100644 index 00000000000..3067d9e9a29 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/MetricsWriter.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import org.gradle.api.logging.Logger +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData +import org.jetbrains.kotlin.gradle.internal.build.metrics.TaskData +import org.jetbrains.kotlin.gradle.logging.kotlinDebug +import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData +import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor +import java.io.File +import java.io.ObjectOutputStream + +internal class MetricsWriter( + private val outputFile: File, + private val log: Logger +) : BuildExecutionDataProcessor { + override fun process(build: BuildExecutionData) { + if (build.failure != null) return + + try { + outputFile.parentFile?.apply { mkdirs() } + + val buildMetricsData = GradleBuildMetricsData() + for (metric in BuildTime.values()) { + buildMetricsData.parentMetric[metric.name] = metric.parent?.name + } + for (attr in BuildAttribute.values()) { + buildMetricsData.buildAttributeKind[attr.name] = attr.kind.name + } + + for (data in build.taskExecutionData) { + val path = data.task.path + val type = data.task::class.java.canonicalName + val buildTimes = data.buildMetrics.buildTimes.asMap().mapKeys { (k, _) -> k.name } + val buildAttributes = data.buildMetrics.buildAttributes.asMap().mapKeys { (k, _) -> k.name } + buildMetricsData.taskData[path] = + TaskData( + path = path, + typeFqName = type, + timeMetrics = buildTimes, + buildAttributes = buildAttributes, + didWork = data.task.didWork + ) + } + + ObjectOutputStream(outputFile.outputStream().buffered()).use { out -> + out.writeObject(buildMetricsData) + } + } catch (e: Exception) { + log.kotlinDebug { "Could not write metrics to $outputFile: $e" } + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt new file mode 100644 index 00000000000..d349e63da7f --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt @@ -0,0 +1,206 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import org.gradle.api.logging.Logger +import org.jetbrains.kotlin.build.report.metrics.BuildAttributes +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.BuildTimes +import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData +import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor +import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData +import org.jetbrains.kotlin.gradle.utils.Printer +import java.io.File +import java.util.* +import kotlin.math.max + +internal class PlainTextBuildReportWriter( + private val outputFile: File, + private val printMetrics: Boolean, + private val log: Logger +) : BuildExecutionDataProcessor { + + private lateinit var p: Printer + + override fun process(build: BuildExecutionData) { + try { + outputFile.bufferedWriter().use { writer -> + p = Printer(writer) + printBuildReport(build) + } + + log.lifecycle("Kotlin build report is written to ${outputFile.canonicalPath}") + } catch (e: Exception) { + log.error("Could not write Kotlin build report to ${outputFile.canonicalPath}", e) + } + } + + private fun printBuildReport(build: BuildExecutionData) { + printBuildInfo(build) + printMetrics(build.aggregatedMetrics) + printTaskOverview(build) + printTasksLog(build) + } + + private fun printBuildInfo(build: BuildExecutionData) { + p.withIndent("Gradle start parameters:") { + build.startParameters.forEach { p.println(it) } + } + + p.println() + if (build.failure != null) { + p.println("Build failed: ${build.failure}") + } + p.println() + } + + private fun printMetrics(buildMetrics: BuildMetrics) { + if (!printMetrics) return + + printBuildTimes(buildMetrics.buildTimes) + printBuildAttributes(buildMetrics.buildAttributes) + } + + private fun printBuildTimes(buildTimes: BuildTimes) { + val collectedBuildTimes = buildTimes.asMap() + + if (collectedBuildTimes.isEmpty()) return + + p.println("Time metrics:") + p.withIndent { + val visitedBuildTimes = HashSet() + fun printBuildTime(buildTime: BuildTime) { + if (!visitedBuildTimes.add(buildTime)) return + + val timeNs = collectedBuildTimes[buildTime] ?: return + p.println("${buildTime.name}: ${formatTime(timeNs)}") + p.withIndent { + BuildTime.children[buildTime]?.forEach { printBuildTime(it) } + } + } + + for (buildTime in BuildTime.values()) { + if (buildTime.parent != null) continue + + printBuildTime(buildTime) + } + } + p.println() + } + + private fun printBuildAttributes(buildAttributes: BuildAttributes) { + val allAttributes = buildAttributes.asMap() + if (allAttributes.isEmpty()) return + + p.withIndent("Build attributes:") { + val attributesByKind = allAttributes.entries.groupBy { it.key.kind }.toSortedMap() + for ((kind, attributesCounts) in attributesByKind) { + printMap(p, kind.name, attributesCounts.map { (k, v) -> k.name to v }.toMap()) + } + } + p.println() + } + + private fun printTaskOverview(build: BuildExecutionData) { + var allTasksTimeNs = 0L + var kotlinTotalTimeNs = 0L + val kotlinTasks = ArrayList() + + for (task in build.taskExecutionData) { + val taskTimeNs = task.totalTimeNs + allTasksTimeNs += taskTimeNs + + if (task.isKotlinTask) { + kotlinTotalTimeNs += taskTimeNs + kotlinTasks.add(task) + } + } + + if (kotlinTasks.isEmpty()) { + p.println("No Kotlin task was run") + return + } + + val ktTaskPercent = (kotlinTotalTimeNs.toDouble() / allTasksTimeNs * 100).asString(1) + p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeNs)} ($ktTaskPercent % of all tasks time)") + + val table = TextTable("Time", "% of Kotlin time", "Task") + for (task in kotlinTasks.sortedByDescending { it.totalTimeNs }) { + val timeNs = task.totalTimeNs + val percent = (timeNs.toDouble() / kotlinTotalTimeNs * 100).asString(1) + table.addRow(formatTime(timeNs), "$percent %", task.task.path) + } + table.printTo(p) + p.println() + } + + private fun printTasksLog(build: BuildExecutionData) { + for (task in build.taskExecutionData) { + printTaskLog(task) + p.println() + } + } + + private fun printTaskLog(task: TaskExecutionData) { + val skipMessage = task.resultState.skipMessage + if (skipMessage != null) { + p.println("$task was skipped: $skipMessage") + } else { + p.println("$task finished in ${formatTime(task.totalTimeNs)}") + } + + if (task.icLogLines.isNotEmpty()) { + p.withIndent("Compilation log for $task:") { + task.icLogLines.forEach { p.println(it) } + } + } + + printMetrics(task.buildMetrics) + } +} + +private fun printMap(p: Printer, name: String, mapping: Map) { + if (mapping.isEmpty()) return + + if (mapping.size == 1) { + p.println("$name: ${mapping.keys.single()}") + return + } + + p.withIndent("$name:") { + val sortedEnumMap = mapping.toSortedMap() + for ((k, v) in sortedEnumMap) { + p.println("$k($v)") + } + } +} + +private class TextTable(vararg columnNames: String) { + private val rows = ArrayList>() + private val columnsCount = columnNames.size + private val maxLengths = IntArray(columnsCount) { columnNames[it].length } + + init { + rows.add(columnNames.toList()) + } + + fun addRow(vararg row: String) { + check(row.size == columnsCount) { "Row size ${row.size} differs from columns count $columnsCount" } + rows.add(row.toList()) + + for ((i, col) in row.withIndex()) { + maxLengths[i] = max(maxLengths[i], col.length) + } + } + + fun printTo(p: Printer) { + for (row in rows) { + val rowStr = row.withIndex().joinToString("|") { (i, col) -> col.padEnd(maxLengths[i], ' ') } + p.println(rowStr) + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt new file mode 100644 index 00000000000..7e289797a92 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import java.io.File +import java.io.Serializable + +data class ReportingSettings( + val metricsOutputFile: File? = null, + val buildReportDir: File? = null, + val reportMetrics: Boolean = false, + val includeMetricsInReport: Boolean = false, + val buildReportMode: BuildReportMode = BuildReportMode.NONE +) : Serializable { + companion object { + const val serialVersionUID: Long = 0 + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt index f92e2a23fa0..5522601c2e9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt @@ -5,7 +5,9 @@ package org.jetbrains.kotlin.gradle.report +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics + internal class TaskExecutionResult( - val executionStrategy: String, - val icLogLines: List? + val buildMetrics: BuildMetrics, + val icLogLines: List = emptyList() ) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt new file mode 100644 index 00000000000..f07adcb59b5 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt @@ -0,0 +1,83 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import org.gradle.api.Project +import org.gradle.api.invocation.Gradle +import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider +import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor +import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile +import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable +import java.text.SimpleDateFormat +import java.util.* + +internal fun configureReporting(gradle: Gradle) { + val buildDataProcessors = ArrayList() + + val rootProject = gradle.rootProject + val reportingSettings = reportingSettings(rootProject) + gradle.taskGraph.whenReady { graph -> + graph.allTasks.asSequence() + .filterIsInstance>() + .forEach { it.reportingSettings = reportingSettings } + } + + if (reportingSettings.buildReportMode != BuildReportMode.NONE && reportingSettings.buildReportDir != null) { + configurePlainTextReportWriter(gradle, reportingSettings)?.let { + buildDataProcessors.add(it) + } + } + + if (reportingSettings.metricsOutputFile != null) { + buildDataProcessors.add(MetricsWriter(reportingSettings.metricsOutputFile.absoluteFile, rootProject.logger)) + } + + if (buildDataProcessors.isNotEmpty() && !isConfigurationCacheAvailable(gradle)) { + val listener = BuildDataRecorder(gradle, buildDataProcessors) + gradle.addBuildListener(listener) + } +} + +private fun reportingSettings(rootProject: Project): ReportingSettings { + val properties = PropertiesProvider(rootProject) + val buildReportMode = + when { + !properties.buildReportEnabled -> BuildReportMode.NONE + properties.buildReportVerbose -> BuildReportMode.VERBOSE + else -> BuildReportMode.SIMPLE + } + val metricsOutputFile = properties.singleBuildMetricsFile + val buildReportDir = properties.buildReportDir ?: rootProject.buildDir.resolve("reports/kotlin-build") + val includeMetricsInReport = properties.buildReportMetrics || buildReportMode == BuildReportMode.VERBOSE + return ReportingSettings( + metricsOutputFile = metricsOutputFile, + buildReportDir = buildReportDir, + reportMetrics = metricsOutputFile != null || includeMetricsInReport, + includeMetricsInReport = includeMetricsInReport, + buildReportMode = buildReportMode + ) +} + +private fun configurePlainTextReportWriter( + gradle: Gradle, + reportingSettings: ReportingSettings +): BuildExecutionDataProcessor? { + val log = gradle.rootProject.logger + val reportDir = reportingSettings.buildReportDir!!.apply { mkdirs() } + if (reportDir.isFile) { + log.error("Kotlin build report cannot be created: '$reportDir' is a file") + return null + } + reportDir.mkdirs() + val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time) + val reportFile = reportDir.resolve("${gradle.rootProject.name}-build-$ts.txt") + + return PlainTextBuildReportWriter( + outputFile = reportFile, + printMetrics = reportingSettings.includeMetricsInReport, + log = log + ) +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt new file mode 100644 index 00000000000..f286e1c41ad --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report.data + +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics + +internal class BuildExecutionData( + val startParameters: Collection, + val failure: Throwable?, + val taskExecutionData: Collection +) { + val aggregatedMetrics by lazy { + BuildMetrics().also { acc -> + taskExecutionData.forEach { acc.addAll(it.buildMetrics) } + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionDataProcessor.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionDataProcessor.kt new file mode 100644 index 00000000000..3437939a7a2 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionDataProcessor.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report.data + +internal interface BuildExecutionDataProcessor { + fun process(build: BuildExecutionData) +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/TaskExecutionData.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/TaskExecutionData.kt new file mode 100644 index 00000000000..c521f682512 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/data/TaskExecutionData.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report.data + +import org.gradle.api.Task +import org.gradle.api.tasks.TaskState +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics + +internal interface TaskExecutionData { + val task: Task + val startNs: Long + val endNs: Long + val totalTimeNs: Long + val resultState: TaskState + val icLogLines: List + val buildMetrics: BuildMetrics + val isKotlinTask: Boolean +} + diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt index 96fc0cd5a87..27efaac61d0 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt @@ -99,7 +99,7 @@ open class KotlinCompileCommon : AbstractKotlinCompile @get:Input internal var useFallbackCompilerSearch: Boolean = false + @get:Internal + override val metrics: BuildMetricsReporter = + BuildMetricsReporterImpl() + @get:Classpath @get:InputFiles internal val computedCompilerClasspath: List by lazy { @@ -195,7 +203,7 @@ abstract class AbstractKotlinCompile() : AbstractKo incremental @get:Internal - internal var buildReportMode: BuildReportMode? = null + internal var reportingSettings = ReportingSettings() @get:Internal internal val taskData: KotlinCompileTaskData = KotlinCompileTaskData.get(project, name) @@ -327,7 +335,7 @@ abstract class AbstractKotlinCompile() : AbstractKo // To prevent this, we backup outputs before incremental build and restore when exception is thrown val outputsBackup: TaskOutputsBackup? = if (isIncrementalCompilationEnabled() && inputs.isIncremental) - kotlinLogger.logTime("Backing up outputs for incremental build") { + metrics.measure(BuildTime.BACKUP_OUTPUT) { TaskOutputsBackup(allOutputFiles()) } else null @@ -342,7 +350,7 @@ abstract class AbstractKotlinCompile() : AbstractKo executeImpl(inputs) } catch (t: Throwable) { if (outputsBackup != null) { - kotlinLogger.logTime("Restoring previous outputs on error") { + metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) { outputsBackup.restoreOutputs() } } @@ -525,7 +533,7 @@ open class KotlinCompile : AbstractKotlinCompile(), Kotl val environment = GradleCompilerEnvironment( computedCompilerClasspath, messageCollector, outputItemCollector, outputFiles = allOutputFiles(), - buildReportMode = buildReportMode, + reportingSettings = reportingSettings, incrementalCompilationEnvironment = icEnv, kotlinScriptExtensions = sourceFilesExtensions.toTypedArray() ) @@ -752,7 +760,7 @@ open class Kotlin2JsCompile : AbstractKotlinCompile(), Ko val environment = GradleCompilerEnvironment( computedCompilerClasspath, messageCollector, outputItemCollector, outputFiles = allOutputFiles(), - buildReportMode = buildReportMode, + reportingSettings = reportingSettings, incrementalCompilationEnvironment = icEnv ) compilerRunner.runJsCompilerAsync(sourceRoots.kotlinSourceFiles, commonSourceSet.toList(), args, environment) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt index 5cfc78f35bf..9921c40e322 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt @@ -1,6 +1,9 @@ package org.jetbrains.kotlin.gradle.tasks import org.gradle.api.GradleException +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.compilerRunner.KotlinLogger import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger @@ -22,26 +25,33 @@ fun throwGradleExceptionIfError(exitCode: ExitCode) { internal fun TaskWithLocalState.clearLocalState(reason: String? = null) { val log = GradleKotlinLogger(logger) - clearLocalState(allOutputFiles(), log, reason) + clearLocalState(allOutputFiles(), log, metrics, reason) } -internal fun clearLocalState(outputFiles: Iterable, log: KotlinLogger, reason: String? = null) { +internal fun clearLocalState( + outputFiles: Iterable, + log: KotlinLogger, + metrics: BuildMetricsReporter, + reason: String? = null +) { log.kotlinDebug { val suffix = reason?.let { " ($it)" }.orEmpty() "Clearing output$suffix:" } - for (file in outputFiles) { - if (!file.exists()) continue - when { - file.isDirectory -> { - log.debug("Deleting output directory: $file") - file.deleteRecursively() - file.mkdirs() - } - file.isFile -> { - log.debug("Deleting output file: $file") - file.delete() + metrics.measure(BuildTime.CLEAR_OUTPUT) { + for (file in outputFiles) { + if (!file.exists()) continue + when { + file.isDirectory -> { + log.debug("Deleting output directory: $file") + file.deleteRecursively() + file.mkdirs() + } + file.isFile -> { + log.debug("Deleting output file: $file") + file.delete() + } } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/Printer.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/Printer.kt new file mode 100644 index 00000000000..71ef31d50de --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/Printer.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jetbrains.kotlin.gradle.utils + +import java.io.IOException + +private val LINE_SEPARATOR = System.getProperty("line.separator") + +internal class Printer( + private val out: Appendable, + private val indentUnit: String = " ", + private var indent: String = "" +) { + private fun append(s: String) { + try { + out.append(s) + } catch (e: IOException) { // Do nothing + } + } + + fun println(vararg strings: String) { + this.print(*strings) + printLineSeparator() + } + + private fun printLineSeparator() { + append(LINE_SEPARATOR) + } + + fun print(vararg strings: String) { + if (strings.isNotEmpty()) { + this.printIndent() + } + this.printWithNoIndent(*strings) + } + + private fun printIndent() { + append(indent) + } + + private fun printWithNoIndent(vararg strings: String) { + for (s in strings) { + append(s) + } + } + + fun pushIndent() { + indent += indentUnit + } + + fun popIndent() { + check(indent.length >= indentUnit.length) { "No indentation to pop" } + indent = indent.substring(indentUnit.length) + } + + inline fun withIndent(headLine: String? = null, fn: () -> T): T { + if (headLine != null) { + this.println(headLine) + } + pushIndent() + + return try { + fn() + } finally { + popIndent() + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-maven-plugin/src/main/java/org/jetbrains/kotlin/maven/incremental/MavenICReporter.java b/libraries/tools/kotlin-maven-plugin/src/main/java/org/jetbrains/kotlin/maven/incremental/MavenICReporter.java index 05809d2ba83..8031b21606a 100644 --- a/libraries/tools/kotlin-maven-plugin/src/main/java/org/jetbrains/kotlin/maven/incremental/MavenICReporter.java +++ b/libraries/tools/kotlin-maven-plugin/src/main/java/org/jetbrains/kotlin/maven/incremental/MavenICReporter.java @@ -20,7 +20,7 @@ import kotlin.jvm.functions.Function0; import org.apache.maven.plugin.logging.Log; import org.jetbrains.annotations.NotNull; import org.jetbrains.kotlin.cli.common.ExitCode; -import org.jetbrains.kotlin.incremental.ICReporterBase; +import org.jetbrains.kotlin.build.report.ICReporterBase; import java.io.File; import java.util.Collection; diff --git a/settings.gradle b/settings.gradle index 1ebbde809b9..5d949923667 100644 --- a/settings.gradle +++ b/settings.gradle @@ -241,6 +241,7 @@ include ":benchmarks", ":kotlin-gradle-plugin-dsl-codegen", ":kotlin-gradle-plugin-npm-versions-codegen", ":kotlin-gradle-statistics", + ":kotlin-gradle-build-metrics", ":kotlin-gradle-plugin", ":kotlin-gradle-plugin-model", ":kotlin-gradle-plugin-test-utils-embeddable", @@ -500,6 +501,7 @@ project(':kotlin-gradle-plugin-api').projectDir = "$rootDir/libraries/tools/kotl project(':kotlin-gradle-plugin-dsl-codegen').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-dsl-codegen" as File project(':kotlin-gradle-plugin-npm-versions-codegen').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-npm-versions-codegen" as File project(':kotlin-gradle-statistics').projectDir = "$rootDir/libraries/tools/kotlin-gradle-statistics" as File +project(':kotlin-gradle-build-metrics').projectDir = "$rootDir/libraries/tools/kotlin-gradle-build-metrics" as File project(':kotlin-gradle-plugin').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin" as File project(':kotlin-gradle-plugin-model').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-model" as File project(':kotlin-gradle-plugin-test-utils-embeddable').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-test-utils-embeddable" as File