[KGP] Introduce Incremental Compilation Feature Toggles

Makes it easier to introduce a Gradle property for configuring
IncrementalCompilerRunner.

^KT-64513 Fixed
^KT-63837 In Progress


Merge-request: KT-MR-13671
Merged-by: Evgenii Mazhukin <evgenii.mazhukin@jetbrains.com>
This commit is contained in:
Evgenii Mazhukin
2023-12-27 13:43:54 +00:00
committed by Space Team
parent 97f8f7a734
commit ee3119e9d2
19 changed files with 157 additions and 70 deletions
@@ -23,12 +23,7 @@ class IncrementalCompilationContext(
* Controls whether changes in lookup cache should be tracked. Required for the classpath snapshots based IC approach * Controls whether changes in lookup cache should be tracked. Required for the classpath snapshots based IC approach
*/ */
val trackChangesInLookupCache: Boolean = false, val trackChangesInLookupCache: Boolean = false,
/** val icFeatures: IncrementalCompilationFeatures = IncrementalCompilationFeatures.DEFAULT_CONFIGURATION,
* Controls whether any changes should be propagated to FS until we decide that the compilation is successful or not
*
* Required for optimizing Gradle side outputs backup
*/
val keepIncrementalCompilationCachesInMemory: Boolean = false,
) { ) {
@Deprecated("This constructor is scheduled to be removed. KSP is using it") @Deprecated("This constructor is scheduled to be removed. KSP is using it")
constructor( constructor(
@@ -45,7 +40,9 @@ class IncrementalCompilationContext(
transaction, transaction,
reporter, reporter,
trackChangesInLookupCache, trackChangesInLookupCache,
keepIncrementalCompilationCachesInMemory IncrementalCompilationFeatures.DEFAULT_CONFIGURATION.copy(
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory
),
) )
val fileDescriptorForSourceFiles: KeyDescriptor<File> = pathConverterForSourceFiles.getFileDescriptor() val fileDescriptorForSourceFiles: KeyDescriptor<File> = pathConverterForSourceFiles.getFileDescriptor()
@@ -0,0 +1,58 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental
import java.io.Serializable
/**
* Lists configurable feature toggles for Incremental Compilation.
* Obsolete features might be removed, so it's not recommended
* to use this outside of Kotlin repository.
*
* It's intended to be used across all build systems and target platforms.
* Then, for example, if we're ready to port a feature from one platform to another,
* we just have to initialize the build system-dependent property, and the value
* would be available at the use site without any additional boilerplate.
*
* To add a new feature toggle, consider this checklist:
*
* 1. Gradle input: see [AbstractKotlinCompile.getIncrementalCompilationFeatures].
* Platform-agnostic properties should be the default, but you can extend one of the subclasses, if needed.
* If new property might affect the result of compilation, annotate it with @Input.
* 2. Gradle.properties support: declare the property in [org.jetbrains.kotlin.gradle.plugin.PropertiesProvider].
* Add its use in [org.jetbrains.kotlin.gradle.tasks.configuration.AbstractKotlinCompileConfig] or one of its subclasses (see step 1).
* 3. Build Tools Api, KGP to Interface part: see [BuildToolsApiCompilationWork.performCompilation]
* You need to update both the interface [IncrementalJvmCompilationConfiguration] and its implementation(s).
* 4. Build Tools Api, Interface to Implementation part: see [ICConfiguration.extractIncrementalCompilationFeatures] and its uses.
* Most likely you just need to update the `extract` method.
* 5. Maven - TODO(emazhukin) will do and describe in KT-63837
* 6. Gradle Integration Tests - add new option to [org.jetbrains.kotlin.gradle.testbase.BuildOptions]
*/
data class IncrementalCompilationFeatures(
/**
* Snapshot-based cross-module IC if true, BuildHistory-based if false.
* Snapshot-based IC is only available in JVM.
*/
val withAbiSnapshot: Boolean = false,
/**
* Enables [RecoverableCompilationTransaction] for better restoration of build outputs
* in case of a build error.
*/
val preciseCompilationResultsBackup: Boolean = false,
/**
* If enabled, IC caches are flushed to FS only when the compilation is successful.
* Optimizes outputs backup on Gradle side.
* Requires [preciseCompilationResultsBackup]
*/
val keepIncrementalCompilationCachesInMemory: Boolean = false,
) : Serializable {
companion object {
val DEFAULT_CONFIGURATION = IncrementalCompilationFeatures()
const val serialVersionUID: Long = 0
}
}
@@ -153,7 +153,7 @@ fun <KEY, VALUE> createPersistentStorage(
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
): PersistentStorage<KEY, VALUE> { ): PersistentStorage<KEY, VALUE> {
return LazyStorage(storageFile, keyDescriptor, valueExternalizer).let { storage -> return LazyStorage(storageFile, keyDescriptor, valueExternalizer).let { storage ->
if (icContext.keepIncrementalCompilationCachesInMemory) { if (icContext.icFeatures.keepIncrementalCompilationCachesInMemory) {
InMemoryStorage(storage).also { InMemoryStorage(storage).also {
icContext.transaction.registerInMemoryStorageWrapper(it) icContext.transaction.registerInMemoryStorageWrapper(it)
} }
@@ -170,7 +170,7 @@ fun <KEY, E> createAppendablePersistentStorage(
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
): AppendablePersistentStorage<KEY, E> { ): AppendablePersistentStorage<KEY, E> {
return AppendableLazyStorage(storageFile, keyDescriptor, elementExternalizer).let { storage -> return AppendableLazyStorage(storageFile, keyDescriptor, elementExternalizer).let { storage ->
if (icContext.keepIncrementalCompilationCachesInMemory) { if (icContext.icFeatures.keepIncrementalCompilationCachesInMemory) {
AppendableInMemoryStorage(storage).also { AppendableInMemoryStorage(storage).also {
icContext.transaction.registerInMemoryStorageWrapper(it) icContext.transaction.registerInMemoryStorageWrapper(it)
} }
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
import org.jetbrains.kotlin.incremental.runWithin import org.jetbrains.kotlin.incremental.runWithin
import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories
import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Assertions.*
@@ -126,7 +127,9 @@ class InMemoryStorageTest {
val fileToPathConverter = RelativeFileToPathConverter(storageFile) val fileToPathConverter = RelativeFileToPathConverter(storageFile)
val icContext = IncrementalCompilationContext( val icContext = IncrementalCompilationContext(
pathConverterForSourceFiles = fileToPathConverter, pathConverterForSourceFiles = fileToPathConverter,
keepIncrementalCompilationCachesInMemory = useInMemoryWrapper icFeatures = IncrementalCompilationFeatures(
keepIncrementalCompilationCachesInMemory = useInMemoryWrapper,
),
) )
icContext.transaction.runWithin { transaction -> icContext.transaction.runWithin { transaction ->
val lookupMap = LookupMap(storageFile, icContext) val lookupMap = LookupMap(storageFile, icContext)
@@ -29,6 +29,7 @@ import org.jetbrains.kotlin.daemon.client.BasicCompilerServicesWithResultsFacade
import org.jetbrains.kotlin.daemon.common.CompilerId import org.jetbrains.kotlin.daemon.common.CompilerId
import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions
import org.jetbrains.kotlin.daemon.common.filterExtractProps import org.jetbrains.kotlin.daemon.common.filterExtractProps
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
import org.jetbrains.kotlin.incremental.IncrementalJvmCompilerRunner import org.jetbrains.kotlin.incremental.IncrementalJvmCompilerRunner
import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter
import org.jetbrains.kotlin.incremental.extractKotlinSourcesFromFreeCompilerArguments import org.jetbrains.kotlin.incremental.extractKotlinSourcesFromFreeCompilerArguments
@@ -153,9 +154,7 @@ internal object CompilationServiceImpl : CompilationService {
outputDirs = options.outputDirs, outputDirs = options.outputDirs,
kotlinSourceFilesExtensions = kotlinFilenameExtensions, kotlinSourceFilesExtensions = kotlinFilenameExtensions,
classpathChanges = classpathChanges, classpathChanges = classpathChanges,
withAbiSnapshot = false, icFeatures = options.extractIncrementalCompilationFeatures(),
preciseCompilationResultsBackup = options.preciseCompilationResultsBackupEnabled,
keepIncrementalCompilationCachesInMemory = options.incrementalCompilationCachesKeptInMemory
) )
val rootProjectDir = options.rootProjectDir val rootProjectDir = options.rootProjectDir
val buildDir = options.buildDir val buildDir = options.buildDir
@@ -0,0 +1,25 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.buildtools.internal
import org.jetbrains.kotlin.buildtools.api.jvm.IncrementalJvmCompilationConfiguration
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
private typealias ICConfiguration = IncrementalJvmCompilationConfiguration<*>
/**
* IncrementalJvmCompilationConfiguration provides single-property API for forward-compatibility.
*
* configurationAdapters are there to regroup the properties and work with higher-level interfaces.
*/
internal fun ICConfiguration.extractIncrementalCompilationFeatures(): IncrementalCompilationFeatures {
return IncrementalCompilationFeatures(
withAbiSnapshot = false,
preciseCompilationResultsBackup = preciseCompilationResultsBackupEnabled,
keepIncrementalCompilationCachesInMemory = incrementalCompilationCachesKeptInMemory,
)
}
@@ -53,9 +53,7 @@ internal val JvmCompilationConfigurationImpl.asDaemonCompilationOptions: Compila
rootProjectDir = options.rootProjectDir, rootProjectDir = options.rootProjectDir,
buildDir = options.buildDir, buildDir = options.buildDir,
kotlinScriptExtensions = ktsExtensionsAsArray, kotlinScriptExtensions = ktsExtensionsAsArray,
withAbiSnapshot = false, icFeatures = options.extractIncrementalCompilationFeatures(),
preciseCompilationResultsBackup = options.preciseCompilationResultsBackupEnabled,
keepIncrementalCompilationCachesInMemory = options.incrementalCompilationCachesKeptInMemory,
) )
} }
else -> CompilationOptions( else -> CompilationOptions(
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.daemon.common package org.jetbrains.kotlin.daemon.common
import org.jetbrains.kotlin.incremental.ClasspathChanges import org.jetbrains.kotlin.incremental.ClasspathChanges
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
import java.io.File import java.io.File
import java.io.Serializable import java.io.Serializable
@@ -76,9 +77,7 @@ class IncrementalCompilationOptions(
val buildDir: File?, val buildDir: File?,
kotlinScriptExtensions: Array<String>? = null, kotlinScriptExtensions: Array<String>? = null,
val withAbiSnapshot: Boolean = false, val icFeatures: IncrementalCompilationFeatures = IncrementalCompilationFeatures.DEFAULT_CONFIGURATION,
val preciseCompilationResultsBackup: Boolean = false,
val keepIncrementalCompilationCachesInMemory: Boolean = false,
) : CompilationOptions( ) : CompilationOptions(
compilerMode, compilerMode,
targetPlatform, targetPlatform,
@@ -88,7 +87,7 @@ class IncrementalCompilationOptions(
kotlinScriptExtensions kotlinScriptExtensions
) { ) {
companion object { companion object {
const val serialVersionUID: Long = 3 const val serialVersionUID: Long = 4
} }
override fun toString(): String { override fun toString(): String {
@@ -101,6 +100,7 @@ class IncrementalCompilationOptions(
"workingDir=$workingDir, " + "workingDir=$workingDir, " +
"multiModuleICSettings=$multiModuleICSettings, " + "multiModuleICSettings=$multiModuleICSettings, " +
"usePreciseJavaTracking=$usePreciseJavaTracking, " + "usePreciseJavaTracking=$usePreciseJavaTracking, " +
"icFeatures=$icFeatures, " +
"outputFiles=$outputFiles" + "outputFiles=$outputFiles" +
")" ")"
} }
@@ -614,9 +614,7 @@ abstract class CompileServiceImplBase(
buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings?.buildHistoryFile, buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings?.buildHistoryFile,
scopeExpansion = if (args.isIrBackendEnabled()) CompileScopeExpansionMode.ALWAYS else CompileScopeExpansionMode.NEVER, scopeExpansion = if (args.isIrBackendEnabled()) CompileScopeExpansionMode.ALWAYS else CompileScopeExpansionMode.NEVER,
modulesApiHistory = modulesApiHistory, modulesApiHistory = modulesApiHistory,
withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot, icFeatures = incrementalCompilationOptions.icFeatures,
preciseCompilationResultsBackup = incrementalCompilationOptions.preciseCompilationResultsBackup,
keepIncrementalCompilationCachesInMemory = incrementalCompilationOptions.keepIncrementalCompilationCachesInMemory,
) )
return try { return try {
compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles) compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles)
@@ -683,9 +681,7 @@ abstract class CompileServiceImplBase(
modulesApiHistory = modulesApiHistory, modulesApiHistory = modulesApiHistory,
kotlinSourceFilesExtensions = allKotlinExtensions, kotlinSourceFilesExtensions = allKotlinExtensions,
classpathChanges = incrementalCompilationOptions.classpathChanges, classpathChanges = incrementalCompilationOptions.classpathChanges,
withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot, icFeatures = incrementalCompilationOptions.icFeatures,
preciseCompilationResultsBackup = incrementalCompilationOptions.preciseCompilationResultsBackup,
keepIncrementalCompilationCachesInMemory = incrementalCompilationOptions.keepIncrementalCompilationCachesInMemory,
) )
return try { return try {
compiler.compile( compiler.compile(
@@ -72,9 +72,11 @@ abstract class IncrementalCompilerRunner<
*/ */
private val outputDirs: Collection<File>?, private val outputDirs: Collection<File>?,
protected val withAbiSnapshot: Boolean = false, /**
private val preciseCompilationResultsBackup: Boolean = false, * Various options. Boolean flags, both stable and experimental, should be added there.
private val keepIncrementalCompilationCachesInMemory: Boolean = false, * Non-trivial configuration should NOT be added there.
*/
protected val icFeatures: IncrementalCompilationFeatures,
) { ) {
protected val cacheDirectory = File(workingDir, cacheDirName) protected val cacheDirectory = File(workingDir, cacheDirName)
@@ -96,7 +98,7 @@ abstract class IncrementalCompilerRunner<
reporter = reporter, reporter = reporter,
trackChangesInLookupCache = shouldTrackChangesInLookupCache, trackChangesInLookupCache = shouldTrackChangesInLookupCache,
storeFullFqNamesInLookupCache = shouldStoreFullFqNamesInLookupCache, storeFullFqNamesInLookupCache = shouldStoreFullFqNamesInLookupCache,
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory, icFeatures = icFeatures,
) )
protected abstract val shouldTrackChangesInLookupCache: Boolean protected abstract val shouldTrackChangesInLookupCache: Boolean
@@ -201,7 +203,7 @@ abstract class IncrementalCompilerRunner<
return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e) return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
} }
val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null val classpathAbiSnapshot = if (icFeatures.withAbiSnapshot) getClasspathAbiSnapshot(args) else null
// Step 2: Compute files to recompile // Step 2: Compute files to recompile
val compilationMode = try { val compilationMode = try {
@@ -216,7 +218,7 @@ abstract class IncrementalCompilerRunner<
return ICResult.RequiresRebuild(compilationMode.reason) return ICResult.RequiresRebuild(compilationMode.reason)
} }
val abiSnapshotData = if (withAbiSnapshot) { val abiSnapshotData = if (icFeatures.withAbiSnapshot) {
if (!abiSnapshotFile.exists()) { if (!abiSnapshotFile.exists()) {
reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" } reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT) return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
@@ -276,7 +278,7 @@ abstract class IncrementalCompilerRunner<
if (trackChangedFiles) { if (trackChangedFiles) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
} }
val abiSnapshotData = if (withAbiSnapshot) { val abiSnapshotData = if (icFeatures.withAbiSnapshot) {
AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args)) AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
} else null } else null
@@ -405,7 +407,7 @@ abstract class IncrementalCompilerRunner<
return exitCode return exitCode
} }
private fun createTransaction() = if (preciseCompilationResultsBackup) { private fun createTransaction() = if (icFeatures.preciseCompilationResultsBackup) {
RecoverableCompilationTransaction(reporter, Files.createTempDirectory("kotlin-backups")) RecoverableCompilationTransaction(reporter, Files.createTempDirectory("kotlin-backups"))
} else { } else {
NonRecoverableCompilationTransaction() NonRecoverableCompilationTransaction()
@@ -519,7 +521,7 @@ abstract class IncrementalCompilerRunner<
updateCaches(services, caches, generatedFiles, changesCollector) updateCaches(services, caches, generatedFiles, changesCollector)
} }
if (compilationMode is CompilationMode.Rebuild) { if (compilationMode is CompilationMode.Rebuild) {
if (withAbiSnapshot) { if (icFeatures.withAbiSnapshot) {
abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges()) abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges())
} }
break break
@@ -552,7 +554,7 @@ abstract class IncrementalCompilerRunner<
buildDirtyFqNames.addAll(dirtyClassFqNames) buildDirtyFqNames.addAll(dirtyClassFqNames)
//update //update
if (withAbiSnapshot) { if (icFeatures.withAbiSnapshot) {
//TODO(valtman) check method/ kts class remove //TODO(valtman) check method/ kts class remove
changesCollector.protoDataRemoved().forEach { abiSnapshotData!!.snapshot.protos.remove(it) } changesCollector.protoDataRemoved().forEach { abiSnapshotData!!.snapshot.protos.remove(it) }
abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges()) abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges())
@@ -564,7 +566,7 @@ abstract class IncrementalCompilerRunner<
BuildInfo.write(icContext, currentBuildInfo, lastBuildInfoFile) BuildInfo.write(icContext, currentBuildInfo, lastBuildInfoFile)
//write abi snapshot //write abi snapshot
if (withAbiSnapshot) { if (icFeatures.withAbiSnapshot) {
//TODO(valtman) check method/class remove //TODO(valtman) check method/class remove
AbiSnapshotImpl.write(icContext, abiSnapshotData!!.snapshot, abiSnapshotFile) AbiSnapshotImpl.write(icContext, abiSnapshotData!!.snapshot, abiSnapshotFile)
} }
@@ -89,24 +89,20 @@ class IncrementalJsCompilerRunner(
buildHistoryFile: File?, buildHistoryFile: File?,
private val modulesApiHistory: ModulesApiHistory, private val modulesApiHistory: ModulesApiHistory,
private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER, private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER,
withAbiSnapshot: Boolean = false, icFeatures: IncrementalCompilationFeatures = IncrementalCompilationFeatures.DEFAULT_CONFIGURATION,
preciseCompilationResultsBackup: Boolean = false,
keepIncrementalCompilationCachesInMemory: Boolean = false,
) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>( ) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>(
workingDir, workingDir,
"caches-js", "caches-js",
reporter, reporter,
buildHistoryFile = buildHistoryFile, buildHistoryFile = buildHistoryFile,
outputDirs = null, outputDirs = null,
withAbiSnapshot = withAbiSnapshot, icFeatures = icFeatures,
preciseCompilationResultsBackup = preciseCompilationResultsBackup,
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory,
) { ) {
override val shouldTrackChangesInLookupCache override val shouldTrackChangesInLookupCache
get() = false get() = false
override val shouldStoreFullFqNamesInLookupCache override val shouldStoreFullFqNamesInLookupCache
get() = withAbiSnapshot get() = icFeatures.withAbiSnapshot
override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JSCompilerArguments) = override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JSCompilerArguments) =
IncrementalJsCachesManager(icContext, if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol, cacheDirectory) IncrementalJsCachesManager(icContext, if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol, cacheDirectory)
@@ -128,7 +124,7 @@ class IncrementalJsCompilerRunner(
if (buildHistoryFile == null) { if (buildHistoryFile == null) {
error("The build is configured to use the build-history based IC approach, but doesn't specify the buildHistoryFile") error("The build is configured to use the build-history based IC approach, but doesn't specify the buildHistoryFile")
} }
if (!withAbiSnapshot && !buildHistoryFile.isFile) { if (!icFeatures.withAbiSnapshot && !buildHistoryFile.isFile) {
return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
} }
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile, messageCollector) ?: return CompilationMode.Rebuild(BuildAttribute.INVALID_LAST_BUILD_INFO) val lastBuildInfo = BuildInfo.read(lastBuildInfoFile, messageCollector) ?: return CompilationMode.Rebuild(BuildAttribute.INVALID_LAST_BUILD_INFO)
@@ -68,24 +68,20 @@ open class IncrementalJvmCompilerRunner(
private val modulesApiHistory: ModulesApiHistory, private val modulesApiHistory: ModulesApiHistory,
override val kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS, override val kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
private val classpathChanges: ClasspathChanges, private val classpathChanges: ClasspathChanges,
withAbiSnapshot: Boolean = false, icFeatures: IncrementalCompilationFeatures = IncrementalCompilationFeatures.DEFAULT_CONFIGURATION,
preciseCompilationResultsBackup: Boolean = false,
keepIncrementalCompilationCachesInMemory: Boolean = false,
) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>( ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
workingDir, workingDir,
"caches-jvm", "caches-jvm",
reporter, reporter,
buildHistoryFile = buildHistoryFile, buildHistoryFile = buildHistoryFile,
outputDirs = outputDirs, outputDirs = outputDirs,
withAbiSnapshot = withAbiSnapshot, icFeatures = icFeatures,
preciseCompilationResultsBackup = preciseCompilationResultsBackup,
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory,
) { ) {
override val shouldTrackChangesInLookupCache override val shouldTrackChangesInLookupCache
get() = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun get() = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
override val shouldStoreFullFqNamesInLookupCache override val shouldStoreFullFqNamesInLookupCache
get() = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled get() = icFeatures.withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled
override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments) = override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments) =
IncrementalJvmCachesManager(icContext, args.destination?.let { File(it) }, cacheDirectory) IncrementalJvmCachesManager(icContext, args.destination?.let { File(it) }, cacheDirectory)
@@ -133,7 +129,7 @@ open class IncrementalJvmCompilerRunner(
classpathAbiSnapshots: Map<String, AbiSnapshot> classpathAbiSnapshots: Map<String, AbiSnapshot>
): CompilationMode { ): CompilationMode {
return try { return try {
calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, classpathAbiSnapshots, withAbiSnapshot) calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, classpathAbiSnapshots, icFeatures.withAbiSnapshot)
} finally { } finally {
this.messageCollector.flush(messageCollector) this.messageCollector.flush(messageCollector)
this.messageCollector.clear() this.messageCollector.clear()
@@ -32,7 +32,9 @@ class IncrementalJvmCompilerTestRunner(
modulesApiHistory, modulesApiHistory,
kotlinSourceFilesExtensions, kotlinSourceFilesExtensions,
classpathChanges, classpathChanges,
withAbiSnapshot icFeatures = IncrementalCompilationFeatures(
withAbiSnapshot = withAbiSnapshot
),
) { ) {
override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments): IncrementalJvmCachesManager = override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments): IncrementalJvmCachesManager =
object : IncrementalJvmCachesManager( object : IncrementalJvmCachesManager(
@@ -132,7 +132,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
private fun collectStatTags(): Set<StatTag> { private fun collectStatTags(): Set<StatTag> {
val statTags = HashSet<StatTag>() val statTags = HashSet<StatTag>()
config.incrementalCompilationEnvironment?.withAbiSnapshot?.ifTrue { statTags.add(StatTag.ABI_SNAPSHOT) } config.incrementalCompilationEnvironment?.icFeatures?.withAbiSnapshot?.ifTrue { statTags.add(StatTag.ABI_SNAPSHOT) }
if (config.incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) { if (config.incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
statTags.add(StatTag.ARTIFACT_TRANSFORM) statTags.add(StatTag.ARTIFACT_TRANSFORM)
} }
@@ -320,9 +320,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
rootProjectDir = icEnv.rootProjectDir, rootProjectDir = icEnv.rootProjectDir,
buildDir = icEnv.buildDir, buildDir = icEnv.buildDir,
kotlinScriptExtensions = config.kotlinScriptExtensions, kotlinScriptExtensions = config.kotlinScriptExtensions,
withAbiSnapshot = icEnv.withAbiSnapshot, icFeatures = icEnv.icFeatures,
preciseCompilationResultsBackup = icEnv.preciseCompilationResultsBackup,
keepIncrementalCompilationCachesInMemory = icEnv.keepIncrementalCompilationCachesInMemory
) )
log.info("Options for KOTLIN DAEMON: $compilationOptions") log.info("Options for KOTLIN DAEMON: $compilationOptions")
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.buildtools.api.SourcesChanges import org.jetbrains.kotlin.buildtools.api.SourcesChanges
import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
import org.jetbrains.kotlin.incremental.ClasspathChanges import org.jetbrains.kotlin.incremental.ClasspathChanges
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
import java.io.File import java.io.File
import java.io.Serializable import java.io.Serializable
@@ -20,11 +21,9 @@ internal class IncrementalCompilationEnvironment(
val usePreciseJavaTracking: Boolean = false, val usePreciseJavaTracking: Boolean = false,
val disableMultiModuleIC: Boolean = false, val disableMultiModuleIC: Boolean = false,
val multiModuleICSettings: MultiModuleICSettings, val multiModuleICSettings: MultiModuleICSettings,
val withAbiSnapshot: Boolean = false, val icFeatures: IncrementalCompilationFeatures = IncrementalCompilationFeatures.DEFAULT_CONFIGURATION,
val preciseCompilationResultsBackup: Boolean = false,
val keepIncrementalCompilationCachesInMemory: Boolean = false,
) : Serializable { ) : Serializable {
companion object { companion object {
const val serialVersionUID: Long = 3 const val serialVersionUID: Long = 4
} }
} }
@@ -86,12 +86,14 @@ internal abstract class BuildToolsApiCompilationWork @Inject constructor(
val icEnv = workArguments.incrementalCompilationEnvironment val icEnv = workArguments.incrementalCompilationEnvironment
val classpathChanges = icEnv?.classpathChanges val classpathChanges = icEnv?.classpathChanges
if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) { if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
// important detail: by using primitive-type single-field setters,
// we maintain compatibility of this KGP code with future BuildToolsApi implementations
val classpathSnapshotsConfig = jvmCompilationConfig.makeClasspathSnapshotBasedIncrementalCompilationConfiguration() val classpathSnapshotsConfig = jvmCompilationConfig.makeClasspathSnapshotBasedIncrementalCompilationConfiguration()
.setRootProjectDir(icEnv.rootProjectDir) .setRootProjectDir(icEnv.rootProjectDir)
.setBuildDir(icEnv.buildDir) .setBuildDir(icEnv.buildDir)
.usePreciseJavaTracking(icEnv.usePreciseJavaTracking) .usePreciseJavaTracking(icEnv.usePreciseJavaTracking)
.usePreciseCompilationResultsBackup(icEnv.preciseCompilationResultsBackup) .usePreciseCompilationResultsBackup(icEnv.icFeatures.preciseCompilationResultsBackup)
.keepIncrementalCompilationCachesInMemory(icEnv.keepIncrementalCompilationCachesInMemory) .keepIncrementalCompilationCachesInMemory(icEnv.icFeatures.keepIncrementalCompilationCachesInMemory)
.useOutputDirs(workArguments.outputFiles) .useOutputDirs(workArguments.outputFiles)
.forceNonIncrementalMode(classpathChanges !is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun) .forceNonIncrementalMode(classpathChanges !is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun)
val classpathSnapshotsParameters = ClasspathSnapshotBasedIncrementalCompilationApproachParameters( val classpathSnapshotsParameters = ClasspathSnapshotBasedIncrementalCompilationApproachParameters(
@@ -6,7 +6,6 @@
package org.jetbrains.kotlin.gradle.tasks package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.file.* import org.gradle.api.file.*
import org.gradle.api.logging.Logging
import org.gradle.api.model.ObjectFactory import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.Property import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider import org.gradle.api.provider.Provider
@@ -38,6 +37,7 @@ import org.jetbrains.kotlin.gradle.plugin.internal.UsesBuildIdProviderService
import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService import org.jetbrains.kotlin.gradle.plugin.statistics.UsesBuildFusService
import org.jetbrains.kotlin.gradle.report.* import org.jetbrains.kotlin.gradle.report.*
import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import java.io.File import java.io.File
@@ -294,6 +294,17 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
commonSourceSet commonSourceSet
) )
/**
* Entry point for getting IC feature toggles in Gradle. Child classes should override it
* if they have a platform-specific Input.
*/
protected open fun makeIncrementalCompilationFeatures(): IncrementalCompilationFeatures {
return IncrementalCompilationFeatures(
preciseCompilationResultsBackup = preciseCompilationResultsBackup.get(),
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory.get(),
)
}
private fun executeImpl( private fun executeImpl(
inputChanges: InputChanges, inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup? taskOutputsBackup: TaskOutputsBackup?
@@ -267,8 +267,7 @@ abstract class Kotlin2JsCompile @Inject constructor(
rootProjectDir = rootProjectDir, rootProjectDir = rootProjectDir,
buildDir = projectLayout.buildDirectory.getFile(), buildDir = projectLayout.buildDirectory.getFile(),
multiModuleICSettings = multiModuleICSettings, multiModuleICSettings = multiModuleICSettings,
preciseCompilationResultsBackup = preciseCompilationResultsBackup.get(), icFeatures = makeIncrementalCompilationFeatures(),
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory.get(),
) )
} else null } else null
@@ -48,6 +48,7 @@ import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnable
import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot
import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun
import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles
import org.jetbrains.kotlin.incremental.IncrementalCompilationFeatures
import org.jetbrains.kotlin.utils.addToStdlib.cast import org.jetbrains.kotlin.utils.addToStdlib.cast
import javax.inject.Inject import javax.inject.Inject
@@ -326,9 +327,7 @@ abstract class KotlinCompile @Inject constructor(
usePreciseJavaTracking = usePreciseJavaTracking, usePreciseJavaTracking = usePreciseJavaTracking,
disableMultiModuleIC = disableMultiModuleIC, disableMultiModuleIC = disableMultiModuleIC,
multiModuleICSettings = multiModuleICSettings, multiModuleICSettings = multiModuleICSettings,
withAbiSnapshot = useKotlinAbiSnapshot.get(), icFeatures = makeIncrementalCompilationFeatures(),
preciseCompilationResultsBackup = preciseCompilationResultsBackup.get(),
keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory.get(),
) )
} else null } else null
@@ -466,6 +465,13 @@ abstract class KotlinCompile @Inject constructor(
super.setSource(*sources) super.setSource(*sources)
} }
// override incremental compilation features, while withAbiSnapshot is JVM-only
override fun makeIncrementalCompilationFeatures(): IncrementalCompilationFeatures {
return super.makeIncrementalCompilationFeatures().copy(
withAbiSnapshot = useKotlinAbiSnapshot.get(),
)
}
private fun getClasspathChanges(inputChanges: InputChanges): ClasspathChanges = when { private fun getClasspathChanges(inputChanges: InputChanges): ClasspathChanges = when {
!classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathSnapshotDisabled !classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathSnapshotDisabled
else -> { else -> {