azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add IC metrics reporting

Browse Source
This commit is contained in:
Alexey Tsvetkov
2020-03-02 14:26:49 +03:00
committed by Alexander Likhachev
parent 5bde6457b1
commit 36387d97ad
69 changed files with 1396 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt
+1
View File
@@ -29,6 +29,7 @@ enum class CompilationResultCategory(val code: Int) {
IC_COMPILE_ITERATION(0),
BUILD_REPORT_LINES(1),
VERBOSE_BUILD_REPORT_LINES(2),
BUILD_METRICS(3)
}
interface CompilationResultsAsync {
compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt
+7 -4
View File
@@ -21,6 +21,9 @@ import com.intellij.openapi.util.Disposer
import com.intellij.openapi.vfs.impl.ZipHandler
import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.RemoteBuildReporter
import org.jetbrains.kotlin.build.report.RemoteReporter
import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY
@@ -286,7 +289,7 @@ abstract class CompileServiceImplBase(
createMessageCollector: (ServicesFacadeT, CompilationOptions) -> MessageCollector,
createReporter: (ServicesFacadeT, CompilationOptions) -> DaemonMessageReporter,
createServices: (JpsServicesFacadeT, EventManager, Profiler) -> Services,
getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteICReporter
getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteBuildReporter
) = kotlin.run {
val messageCollector = createMessageCollector(servicesFacade, compilationOptions)
val daemonReporter = createReporter(servicesFacade, compilationOptions)
@@ -514,7 +517,7 @@ abstract class CompileServiceImplBase(
args: K2JSCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: RemoteICReporter
reporter: RemoteBuildReporter
): ExitCode {
val allKotlinFiles = arrayListOf<File>()
val freeArgsWithoutKotlinFiles = arrayListOf<String>()
@@ -554,7 +557,7 @@ abstract class CompileServiceImplBase(
k2jvmArgs: K2JVMCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: RemoteICReporter
reporter: RemoteBuildReporter
): ExitCode {
val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS +
(incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct()
@@ -784,7 +787,7 @@ class CompileServiceImpl(
createMessageCollector = ::CompileServicesFacadeMessageCollector,
createReporter = ::DaemonMessageReporter,
createServices = this::createCompileServices,
getICReporter = { a, b, c -> getICReporter(a, b!!, c)}
getICReporter = { a, b, c -> getBuildReporter(a, b!!, c)}
)
override fun leaseReplSession(
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/BuildReportICReporter.kt
+4 -51
View File
@@ -8,23 +8,19 @@ package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteICReporter
import java.io.File
import java.util.*
import kotlin.collections.ArrayList
// todo: sync BuildReportICReporterAsync
internal class BuildReportICReporter(
private val compilationResults: CompilationResults,
rootDir: File,
private val isVerbose: Boolean = false,
// todo: default value
// todo: sync BuildReportICReporterAsync
private val reportMetrics: Boolean = true
private val isVerbose: Boolean = false
) : ICReporterBase(rootDir), RemoteICReporter {
private val icLogLines = arrayListOf<String>()
private val recompilationReason = HashMap<File, String>()
private val rootMetric = Metric("<root>", 0)
private val metrics = ArrayDeque<Metric>().apply { add(rootMetric) }
override fun report(message: () -> String) {
icLogLines.add(message())
@@ -36,30 +32,6 @@ internal class BuildReportICReporter(
}
}
override fun startMeasure(metric: String, startNs: Long) {
if (!reportMetrics) return
val newMetric = Metric(metric, startNs)
if (metrics.isNotEmpty()) {
metrics.peekLast().children.add(newMetric)
}
metrics.addLast(newMetric)
}
override fun endMeasure(metric: String, endNs: Long) {
if (!reportMetrics) return
while (metrics.isNotEmpty()) {
val lastMetric = metrics.peekLast()
if (lastMetric.name == metric) {
lastMetric.endNs = endNs
break
} else {
metrics.removeLast()
}
}
}
override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection<File>, exitCode: ExitCode) {
if (!incremental) return
@@ -76,25 +48,6 @@ internal class BuildReportICReporter(
}
override fun flush() {
if (reportMetrics) {
icLogLines.add("Performance metrics:")
reportMetric(rootMetric)
}
compilationResults.add(CompilationResultCategory.BUILD_REPORT_LINES.code, icLogLines)
}
private fun reportMetric(metric: Metric, level: Int = 0) {
if (level > 0) {
val timeMs = metric.endNs?.let { (it - metric.startNs) / 1_000_000L }
icLogLines.add(" ".repeat(level) + "{perf_metric:${metric.name}} ${timeMs ?: "<unknown>"} ms")
}
metric.children.forEach { reportMetric(it, level + 1) }
}
}
private class Metric(val name: String, val startNs: Long) {
var endNs: Long? = null
val children = ArrayList<Metric>()
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompileIterationICReporter.kt
+2 -1
View File
@@ -9,7 +9,8 @@ import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
import org.jetbrains.kotlin.daemon.common.CompileIterationResult
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteICReporter
import java.io.File
internal class CompileIterationICReporter(
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/CompositeICReporter.kt
+2 -10
View File
@@ -5,11 +5,11 @@
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.build.report.RemoteICReporter
import org.jetbrains.kotlin.cli.common.ExitCode
import java.io.File
internal class CompositeICReporter(private val reporters: Iterable<RemoteICReporter>) :
RemoteICReporter {
internal class CompositeICReporter(private val reporters: Iterable<RemoteICReporter>) : RemoteICReporter {
override fun report(message: () -> String) {
reporters.forEach { it.report(message) }
}
@@ -34,14 +34,6 @@ internal class CompositeICReporter(private val reporters: Iterable<RemoteICRepor
reporters.forEach { it.reportMarkDirty(affectedFiles, reason) }
}
override fun startMeasure(metric: String, startNs: Long) {
reporters.forEach { it.startMeasure(metric, startNs) }
}
override fun endMeasure(metric: String, endNs: Long) {
reporters.forEach { it.endMeasure(metric, endNs) }
}
override fun flush() {
reporters.forEach { it.flush() }
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/DebugMessagesICReporter.kt
+2 -1
View File
@@ -7,7 +7,8 @@ package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteICReporter
import java.io.File
internal class DebugMessagesICReporter(
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/RemoteBuildMetricsReporterAdapter.kt
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
class RemoteBuildMetricsReporterAdapter(
private val delegate: BuildMetricsReporter,
private val shouldReport: Boolean,
private val compilationResults: CompilationResults
) :
BuildMetricsReporter by delegate,
RemoteBuildMetricsReporter {
override fun flush() {
if (shouldReport) {
val metrics = delegate.getMetrics()
compilationResults.add(CompilationResultCategory.BUILD_METRICS.code, metrics)
}
}
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/RemoteICReporter.kt
-12
View File
@@ -1,12 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.incremental.ICReporter
interface RemoteICReporter : ICReporter {
fun flush()
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/RemoteICReporter.kt
+21 -8
View File
@@ -9,8 +9,14 @@ import kotlinx.coroutines.*
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.report.CompositeICReporter
import org.jetbrains.kotlin.daemon.report.RemoteICReporter
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.RemoteBuildReporter
import org.jetbrains.kotlin.build.report.RemoteICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter
import org.jetbrains.kotlin.daemon.report.BuildReportICReporter
import org.jetbrains.kotlin.daemon.report.CompileIterationICReporter
import java.io.File
internal class DebugMessagesICReporterAsync(
@@ -106,7 +112,7 @@ fun getICReporterAsync(
servicesFacade: CompilerServicesFacadeBaseAsync,
compilationResults: CompilationResultsAsync?,
compilationOptions: IncrementalCompilationOptions
): RemoteICReporter {
): RemoteBuildReporter {
val root = compilationOptions.modulesInfo.projectRoot
val reporters = ArrayList<RemoteICReporter>()
@@ -120,19 +126,26 @@ fun getICReporterAsync(
.mapNotNullTo(HashSet()) { resultCode ->
CompilationResultCategory.values().getOrNull(resultCode)
}
requestedResults.mapTo(reporters) { requestedResult ->
for (requestedResult in requestedResults) {
when (requestedResult) {
CompilationResultCategory.IC_COMPILE_ITERATION -> {
CompileIterationICReporterAsync(compilationResults)
reporters.add(CompileIterationICReporterAsync(compilationResults))
}
CompilationResultCategory.BUILD_REPORT_LINES -> {
BuildReportICReporterAsync(compilationResults, root)
reporters.add(BuildReportICReporterAsync(compilationResults, root))
}
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> {
BuildReportICReporterAsync(compilationResults, root, isVerbose = true)
reporters.add(BuildReportICReporterAsync(compilationResults, root, isVerbose = true))
}
}
}
return CompositeICReporter(reporters)
val icReporter = CompositeICReporter(reporters)
val metricsReporter = DoNothingRemoteBuildMetricsReporter
return RemoteBuildReporter(icReporter, metricsReporter)
}
object DoNothingRemoteBuildMetricsReporter : BuildMetricsReporter by DoNothingBuildMetricsReporter, RemoteBuildMetricsReporter {
override fun flush() {
}
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt
+15 -8
View File
@@ -16,15 +16,18 @@
package org.jetbrains.kotlin.daemon.report
import org.jetbrains.kotlin.build.report.RemoteBuildReporter
import org.jetbrains.kotlin.build.report.RemoteICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.daemon.common.*
import java.io.File
import java.util.*
fun getICReporter(
fun getBuildReporter(
servicesFacade: CompilerServicesFacadeBase,
compilationResults: CompilationResults,
compilationOptions: IncrementalCompilationOptions
): RemoteICReporter {
): RemoteBuildReporter {
val root = compilationOptions.modulesInfo.projectRoot
val reporters = ArrayList<RemoteICReporter>()
@@ -38,21 +41,25 @@ fun getICReporter(
.mapNotNullTo(HashSet()) { resultCode ->
CompilationResultCategory.values().getOrNull(resultCode)
}
requestedResults.mapTo(reporters) { requestedResult ->
for (requestedResult in requestedResults) {
when (requestedResult) {
CompilationResultCategory.IC_COMPILE_ITERATION -> {
CompileIterationICReporter(compilationResults)
reporters.add(CompileIterationICReporter(compilationResults))
}
CompilationResultCategory.BUILD_REPORT_LINES -> {
BuildReportICReporter(compilationResults, root)
reporters.add(BuildReportICReporter(compilationResults, root))
}
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> {
BuildReportICReporter(compilationResults, root, isVerbose = true)
reporters.add(BuildReportICReporter(compilationResults, root, isVerbose = true))
}
}
}
val areBuildMetricsNeeded = CompilationResultCategory.BUILD_METRICS in requestedResults
val metricsReporter =
(if (areBuildMetricsNeeded) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter)
.let { RemoteBuildMetricsReporterAdapter(it, areBuildMetricsNeeded, compilationResults) }
return CompositeICReporter(reporters)
return RemoteBuildReporter(CompositeICReporter(reporters), metricsReporter)
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt
+1
View File
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.name.FqName
import java.io.File
import java.io.IOException
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt
+4 -2
View File
@@ -19,6 +19,8 @@ package org.jetbrains.kotlin.incremental
import com.intellij.psi.PsiClass
import com.intellij.psi.PsiFile
import com.intellij.psi.PsiJavaFile
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import java.io.File
import java.util.*
@@ -37,7 +39,7 @@ internal class ChangedJavaFilesProcessor(
if (removedJava.any()) {
reporter.report { "Some java files are removed: [${removedJava.joinToString()}]" }
return ChangesEither.Unknown()
return ChangesEither.Unknown(BuildAttribute.JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED)
}
val symbols = HashSet<LookupSymbol>()
@@ -47,7 +49,7 @@ internal class ChangedJavaFilesProcessor(
val psiFile = psiFileFactory(javaFile)
if (psiFile !is PsiJavaFile) {
reporter.report { "Expected PsiJavaFile, got ${psiFile?.javaClass}" }
return ChangesEither.Unknown()
return ChangesEither.Unknown(BuildAttribute.JAVA_CHANGE_UNEXPECTED_PSI)
}
psiFile.classes.forEach { it.addLookupSymbols(symbols) }
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt
+2 -1
View File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.name.FqName
internal sealed class ChangesEither {
@@ -24,5 +25,5 @@ internal sealed class ChangesEither {
val fqNames: Collection<FqName> = emptyList()
) : ChangesEither()
internal class Unknown(val reason: String? = null) : ChangesEither()
internal class Unknown(val reason: BuildAttribute) : ChangesEither()
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/DirtyFilesContainer.kt
+1
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.name.FqName
import java.io.File
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt
+1
View File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt
+56 -22
View File
@@ -18,6 +18,10 @@ package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -41,7 +45,7 @@ abstract class IncrementalCompilerRunner<
>(
private val workingDir: File,
cacheDirName: String,
protected val reporter: ICReporter,
protected val reporter: BuildReporter,
private val buildHistoryFile: File,
// there might be some additional output directories (e.g. for generated java in kapt)
// to remove them correctly on rebuild, we pass them as additional argument
@@ -65,20 +69,33 @@ abstract class IncrementalCompilerRunner<
// otherwise we track source files changes ourselves.
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION) {
compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir)
}
private fun compileImpl(
allSourceFiles: List<File>,
args: Args,
messageCollector: MessageCollector,
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode {
assert(isICEnabled()) { "Incremental compilation is not enabled" }
var caches = createCacheManager(args, projectDir)
fun rebuild(reason: () -> String): ExitCode {
reporter.report(reason)
fun rebuild(reason: BuildAttribute): ExitCode {
reporter.report { "Non-incremental compilation will be performed: $reason" }
caches.close(false)
clearLocalStateOnRebuild(args)
// todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
clearLocalStateOnRebuild(args)
}
caches = createCacheManager(args, projectDir)
if (providedChangedFiles == null) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
}
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(), messageCollector)
return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector)
}
return try {
@@ -90,16 +107,17 @@ abstract class IncrementalCompilerRunner<
compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector)
}
is CompilationMode.Rebuild -> {
rebuild { "Non-incremental compilation will be performed: ${compilationMode.reason}" }
rebuild(compilationMode.reason)
}
}
if (!caches.close(flush = true)) throw RuntimeException("Could not flush caches")
return exitCode
} catch (e: Exception) {
} catch (e: Exception) { // todo: catch only cache corruption
// todo: warn?
rebuild { "Possible cache corruption. Rebuilding. $e" }
reporter.report { "Rebuilding because of possible caches corruption: $e" }
rebuild(BuildAttribute.CACHE_CORRUPTION)
}
}
@@ -139,10 +157,17 @@ abstract class IncrementalCompilerRunner<
): CompilationMode =
when (changedFiles) {
is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector)
is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" }
is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
}
protected abstract fun calculateSourcesToCompile(
private fun calculateSourcesToCompile(
caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector
): CompilationMode =
reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector)
}
protected abstract fun calculateSourcesToCompileImpl(
caches: CacheManager,
changedFiles: ChangedFiles.Known,
args: Args,
@@ -161,9 +186,7 @@ abstract class IncrementalCompilerRunner<
protected sealed class CompilationMode {
class Incremental(val dirtyFiles: DirtyFilesContainer) : CompilationMode()
class Rebuild(getReason: () -> String = { "" }) : CompilationMode() {
val reason: String by lazy(getReason)
}
class Rebuild(val reason: BuildAttribute) : CompilationMode()
}
protected abstract fun updateCaches(
@@ -211,9 +234,17 @@ abstract class IncrementalCompilerRunner<
): ExitCode {
preBuildHook(args, compilationMode)
val buildTimeMode: BuildTime
val dirtySources = when (compilationMode) {
is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableList()
is CompilationMode.Rebuild -> allKotlinSources.toMutableList()
is CompilationMode.Incremental -> {
buildTimeMode = BuildTime.INCREMENTAL_ITERATION
compilationMode.dirtyFiles.toMutableList()
}
is CompilationMode.Rebuild -> {
buildTimeMode = BuildTime.NON_INCREMENTAL_ITERATION
reporter.addAttribute(compilationMode.reason)
allKotlinSources.toMutableList()
}
}
val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
@@ -247,7 +278,9 @@ abstract class IncrementalCompilerRunner<
val bufferingMessageCollector = BufferingMessageCollector()
val messageCollectorAdapter = MessageCollectorToOutputItemsCollectorAdapter(bufferingMessageCollector, outputItemsCollector)
exitCode = runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter)
exitCode = reporter.measure(buildTimeMode) {
runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter)
}
val generatedFiles = outputItemsCollector.outputs.map(SimpleOutputItem::toGeneratedFile)
if (compilationMode is CompilationMode.Incremental) {
@@ -268,12 +301,13 @@ abstract class IncrementalCompilerRunner<
dirtySourcesSinceLastTimeFile.delete()
caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
val changesCollector = ChangesCollector()
updateCaches(services, caches, generatedFiles, changesCollector)
reporter.measure(BuildTime.IC_UPDATE_CACHES) {
caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
updateCaches(services, caches, generatedFiles, changesCollector)
}
if (compilationMode is CompilationMode.Rebuild) break
val (dirtyLookupSymbols, dirtyClassFqNames) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
@@ -339,7 +373,7 @@ abstract class IncrementalCompilerRunner<
compilationMode: CompilationMode,
currentBuildInfo: BuildInfo,
dirtyData: DirtyData
) {
) = reporter.measure(BuildTime.IC_WRITE_HISTORY_FILE) {
val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList()
val newDiff = if (compilationMode is CompilationMode.Incremental) {
BuildDifference(currentBuildInfo.startTS, true, dirtyData)
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt
+12 -8
View File
@@ -17,6 +17,10 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.isIrBackendEnabled
@@ -45,9 +49,10 @@ fun makeJsIncrementally(
.filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList()
val buildHistoryFile = File(cachesDir, "build-history.bin")
val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
withJsIC {
val compiler = IncrementalJsCompilerRunner(
cachesDir, reporter,
cachesDir, buildReporter,
buildHistoryFile = buildHistoryFile,
modulesApiHistory = EmptyModulesApiHistory,
scopeExpansion = scopeExpansion
@@ -69,7 +74,7 @@ inline fun <R> withJsIC(fn: () -> R): R {
class IncrementalJsCompilerRunner(
private val workingDir: File,
reporter: ICReporter,
reporter: BuildReporter,
buildHistoryFile: File,
private val modulesApiHistory: ModulesApiHistory,
private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER
@@ -90,14 +95,14 @@ class IncrementalJsCompilerRunner(
override fun destinationDir(args: K2JSCompilerArguments): File =
File(args.outputFile).parentFile
override fun calculateSourcesToCompile(
override fun calculateSourcesToCompileImpl(
caches: IncrementalJsCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JSCompilerArguments,
messageCollector: MessageCollector
): CompilationMode {
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
?: return CompilationMode.Rebuild { "No information on previous build" }
?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
initDirtyFiles(dirtyFiles, changedFiles)
@@ -107,9 +112,9 @@ class IncrementalJsCompilerRunner(
@Suppress("UNUSED_VARIABLE") // for sealed when
val unused = when (classpathChanges) {
is ChangesEither.Unknown -> return CompilationMode.Rebuild {
// todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
"Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}"
is ChangesEither.Unknown -> {
reporter.report { "Could not get classpath's changes: ${classpathChanges.reason}" }
return CompilationMode.Rebuild(classpathChanges.reason)
}
is ChangesEither.Known -> {
dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols)
@@ -117,7 +122,6 @@ class IncrementalJsCompilerRunner(
}
}
val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)
dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNames)
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt
+39 -25
View File
@@ -25,6 +25,10 @@ import com.intellij.psi.PsiJavaFile
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
@@ -64,11 +68,12 @@ fun makeIncrementally(
val sourceFiles = files.filter { it.extension.toLowerCase() in allExtensions }.toList()
val buildHistoryFile = File(cachesDir, "build-history.bin")
args.javaSourceRoots = sourceRoots.map { it.absolutePath }.toTypedArray()
val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
withIC {
val compiler = IncrementalJvmCompilerRunner(
cachesDir,
reporter,
buildReporter,
// Use precise setting in case of non-Gradle build
usePreciseJavaTracking = true,
outputFiles = emptyList(),
@@ -104,7 +109,7 @@ inline fun <R> withIC(enabled: Boolean = true, fn: () -> R): R {
class IncrementalJvmCompilerRunner(
workingDir: File,
reporter: ICReporter,
reporter: BuildReporter,
private val usePreciseJavaTracking: Boolean,
buildHistoryFile: File,
outputFiles: Collection<File>,
@@ -154,7 +159,7 @@ class IncrementalJvmCompilerRunner(
else
null
override fun calculateSourcesToCompile(
override fun calculateSourcesToCompileImpl(
caches: IncrementalJvmCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JVMCompilerArguments,
@@ -176,16 +181,20 @@ class IncrementalJvmCompilerRunner(
val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
initDirtyFiles(dirtyFiles, changedFiles)
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild { "No information on previous build" }
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
val classpathChanges = getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
val classpathChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
}
@Suppress("UNUSED_VARIABLE") // for sealed when
val unused = when (classpathChanges) {
is ChangesEither.Unknown -> return CompilationMode.Rebuild {
// todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
"Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}"
is ChangesEither.Unknown -> {
reporter.report {
"Could not get classpath's changes: ${classpathChanges.reason}"
}
return CompilationMode.Rebuild(classpathChanges.reason)
}
is ChangesEither.Known -> {
dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols)
@@ -194,21 +203,26 @@ class IncrementalJvmCompilerRunner(
}
}
if (!usePreciseJavaTracking) {
val javaFilesChanges = javaFilesProcessor!!.process(changedFiles)
val affectedJavaSymbols = when (javaFilesChanges) {
is ChangesEither.Known -> javaFilesChanges.lookupSymbols
is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" }
}
dirtyFiles.addByDirtySymbols(affectedJavaSymbols)
} else {
if (!processChangedJava(changedFiles, caches)) {
return CompilationMode.Rebuild { "Could not get changes for java files" }
reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_JAVA_SOURCES) {
if (!usePreciseJavaTracking) {
val javaFilesChanges = javaFilesProcessor!!.process(changedFiles)
val affectedJavaSymbols = when (javaFilesChanges) {
is ChangesEither.Known -> javaFilesChanges.lookupSymbols
is ChangesEither.Unknown -> return CompilationMode.Rebuild(javaFilesChanges.reason)
}
dirtyFiles.addByDirtySymbols(affectedJavaSymbols)
} else {
val rebuildReason = processChangedJava(changedFiles, caches)
if (rebuildReason != null) return CompilationMode.Rebuild(rebuildReason)
}
}
val androidLayoutChanges = processLookupSymbolsForAndroidLayouts(changedFiles)
val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)
val androidLayoutChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_ANDROID_LAYOUTS) {
processLookupSymbolsForAndroidLayouts(changedFiles)
}
val removedClassesChanges = reporter.measure(BuildTime.IC_DETECT_REMOVED_CLASSES) {
getRemovedClassesChanges(caches, changedFiles)
}
dirtyFiles.addByDirtySymbols(androidLayoutChanges)
dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
@@ -217,7 +231,7 @@ class IncrementalJvmCompilerRunner(
return CompilationMode.Incremental(dirtyFiles)
}
private fun processChangedJava(changedFiles: ChangedFiles.Known, caches: IncrementalJvmCachesManager): Boolean {
private fun processChangedJava(changedFiles: ChangedFiles.Known, caches: IncrementalJvmCachesManager): BuildAttribute? {
val javaFiles = (changedFiles.modified + changedFiles.removed).filter(File::isJavaFile)
for (javaFile in javaFiles) {
@@ -225,20 +239,20 @@ class IncrementalJvmCompilerRunner(
if (!javaFile.exists()) {
// todo: can we do this more optimal?
reporter.report { "Could not get changed for untracked removed java file $javaFile" }
return false
return BuildAttribute.JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED
}
val psiFile = psiFileProvider.javaFile(javaFile)
if (psiFile !is PsiJavaFile) {
reporter.report { "[Precise Java tracking] Expected PsiJavaFile, got ${psiFile?.javaClass}" }
return false
return BuildAttribute.JAVA_CHANGE_UNEXPECTED_PSI
}
for (psiClass in psiFile.classes) {
val qualifiedName = psiClass.qualifiedName
if (qualifiedName == null) {
reporter.report { "[Precise Java tracking] Class with unknown qualified name in $javaFile" }
return false
return BuildAttribute.JAVA_CHANGE_UNKNOWN_QUALIFIER
}
processChangedUntrackedJavaClass(psiClass, ClassId.topLevel(FqName(qualifiedName)))
@@ -247,7 +261,7 @@ class IncrementalJvmCompilerRunner(
}
caches.platformCache.markDirty(javaFiles)
return true
return null
}
private fun processChangedUntrackedJavaClass(psiClass: PsiClass, classId: ClassId) {
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt
+1
View File
@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.incremental
import com.intellij.util.containers.MultiMap
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.SourceToOutputFilesMap
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt
+47 -21
View File
@@ -5,6 +5,10 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
import org.jetbrains.kotlin.incremental.util.Either
import org.jetbrains.kotlin.name.FqName
@@ -15,7 +19,7 @@ internal fun getClasspathChanges(
changedFiles: ChangedFiles.Known,
lastBuildInfo: BuildInfo,
modulesApiHistory: ModulesApiHistory,
reporter: ICReporter?
reporter: BuildReporter
): ChangesEither {
val classpathSet = HashSet<File>()
for (file in classpath) {
@@ -29,7 +33,10 @@ internal fun getClasspathChanges(
val removedClasspath = changedFiles.removed.filterTo(HashSet()) { it in classpathSet }
// todo: removed classes could be processed normally
if (removedClasspath.isNotEmpty()) return ChangesEither.Unknown("Some files are removed from classpath $removedClasspath")
if (removedClasspath.isNotEmpty()) {
reporter.report { "Some files are removed from classpath: $removedClasspath" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_REMOVED_ENTRY)
}
if (modifiedClasspath.isEmpty()) return ChangesEither.Known()
@@ -38,28 +45,47 @@ internal fun getClasspathChanges(
val symbols = HashSet<LookupSymbol>()
val fqNames = HashSet<FqName>()
val historyFilesEither = modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
val historyFilesEither =
reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) {
modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
}
val historyFiles = when (historyFilesEither) {
is Either.Success<Set<File>> -> historyFilesEither.value
is Either.Error -> return ChangesEither.Unknown(historyFilesEither.reason)
}
for (historyFile in historyFiles) {
val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
?: return ChangesEither.Unknown("Could not read diffs from $historyFile")
val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
if (knownBuilds.isEmpty()) {
return ChangesEither.Unknown("No previously known builds for $historyFile")
}
for (buildDiff in newBuilds) {
if (!buildDiff.isIncremental) return ChangesEither.Unknown("Non-incremental build from dependency $historyFile")
val dirtyData = buildDiff.dirtyData
symbols.addAll(dirtyData.dirtyLookupSymbols)
fqNames.addAll(dirtyData.dirtyClassesFqNames)
is Either.Error -> {
reporter.report { "Could not find history files: ${historyFilesEither.reason}" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND)
}
}
return ChangesEither.Known(symbols, fqNames)
fun analyzeHistoryFiles(): ChangesEither {
for (historyFile in historyFiles) {
val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
?: return run {
reporter.report { "Could not read diffs from $historyFile" }
ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ)
}
val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
if (knownBuilds.isEmpty()) {
reporter.report { "No previously known builds for $historyFile" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS)
}
for (buildDiff in newBuilds) {
if (!buildDiff.isIncremental) {
reporter.report { "Non-incremental build from dependency $historyFile" }
return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP)
}
val dirtyData = buildDiff.dirtyData
symbols.addAll(dirtyData.dirtyLookupSymbols)
fqNames.addAll(dirtyData.dirtyClassesFqNames)
}
}
return ChangesEither.Known(symbols, fqNames)
}
return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) {
analyzeHistoryFiles()
}
}
compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/utils/TestICReporter.kt
+1 -1
View File
@@ -17,7 +17,7 @@
package org.jetbrains.kotlin.incremental.utils
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.build.report.ICReporterBase
import java.io.File
class TestICReporter : ICReporterBase() {