[Gradle] Add a property to control if the IC caches in-memory wrapper is enabled

#KT-56052 In Progress

build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt

@@ -35,11 +35,13 @@ class IncrementalCompilationContext(
         transaction: CompilationTransaction = DummyCompilationTransaction(),
         reporter: ICReporter = DoNothingICReporter,
         trackChangesInLookupCache: Boolean = false,
+        keepIncrementalCompilationCachesInMemory: Boolean = false,
     ) : this(
         createDefaultPathConverter(rootProjectDir),
         storeFullFqNamesInLookupCache,
         transaction,
         reporter,
         trackChangesInLookupCache,
+        keepIncrementalCompilationCachesInMemory
     )
 }

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt

@@ -73,6 +73,7 @@ class IncrementalCompilationOptions(
     kotlinScriptExtensions: Array<String>? = null,
     val withAbiSnapshot: Boolean = false,
     val preciseCompilationResultsBackup: Boolean = false,
+    val keepIncrementalCompilationCachesInMemory: Boolean = false,
 ) : CompilationOptions(
     compilerMode,
     targetPlatform,
@@ -82,7 +83,7 @@ class IncrementalCompilationOptions(
     kotlinScriptExtensions
 ) {
     companion object {
-        const val serialVersionUID: Long = 1
+        const val serialVersionUID: Long = 2
     }
 
     override fun toString(): String {

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -559,6 +559,7 @@ abstract class CompileServiceImplBase(
             modulesApiHistory = modulesApiHistory,
             withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot,
             preciseCompilationResultsBackup = incrementalCompilationOptions.preciseCompilationResultsBackup,
+            keepIncrementalCompilationCachesInMemory = incrementalCompilationOptions.keepIncrementalCompilationCachesInMemory,
         )
         return try {
             compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles)
@@ -619,6 +620,7 @@ abstract class CompileServiceImplBase(
             classpathChanges = incrementalCompilationOptions.classpathChanges,
             withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot,
             preciseCompilationResultsBackup = incrementalCompilationOptions.preciseCompilationResultsBackup,
+            keepIncrementalCompilationCachesInMemory = incrementalCompilationOptions.keepIncrementalCompilationCachesInMemory,
         )
         return try {
             compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, projectRoot)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -72,6 +72,7 @@ abstract class IncrementalCompilerRunner<
 
     protected val withAbiSnapshot: Boolean = false,
     private val preciseCompilationResultsBackup: Boolean = false,
+    private val keepIncrementalCompilationCachesInMemory: Boolean = false,
 ) {
 
     protected val cacheDirectory = File(workingDir, cacheDirName)
@@ -83,10 +84,21 @@ abstract class IncrementalCompilerRunner<
     /**
      * Creates an instance of [IncrementalCompilationContext] that holds common incremental compilation context mostly required for [CacheManager]
      */
-    protected abstract fun createIncrementalCompilationContext(
+    private fun createIncrementalCompilationContext(
         projectDir: File?,
         transaction: CompilationTransaction
-    ): IncrementalCompilationContext
+    ) = IncrementalCompilationContext(
+        transaction = transaction,
+        rootProjectDir = projectDir,
+        reporter = reporter,
+        trackChangesInLookupCache = shouldTrackChangesInLookupCache,
+        storeFullFqNamesInLookupCache = shouldStoreFullFqNamesInLookupCache,
+        keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory,
+    )
+
+    protected abstract val shouldTrackChangesInLookupCache: Boolean
+
+    protected abstract val shouldStoreFullFqNamesInLookupCache: Boolean
 
     protected abstract fun createCacheManager(icContext: IncrementalCompilationContext, args: Args): CacheManager
     protected abstract fun destinationDir(args: Args): File

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -89,6 +89,7 @@ class IncrementalJsCompilerRunner(
     private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER,
     withAbiSnapshot: Boolean = false,
     preciseCompilationResultsBackup: Boolean = false,
+    keepIncrementalCompilationCachesInMemory: Boolean = false,
 ) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>(
     workingDir,
     "caches-js",
@@ -97,14 +98,13 @@ class IncrementalJsCompilerRunner(
     outputDirs = null,
     withAbiSnapshot = withAbiSnapshot,
     preciseCompilationResultsBackup = preciseCompilationResultsBackup,
+    keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory,
 ) {
-    override fun createIncrementalCompilationContext(projectDir: File?, transaction: CompilationTransaction) =
-        IncrementalCompilationContext(
-            transaction = transaction,
-            rootProjectDir = projectDir,
-            reporter = reporter,
-            storeFullFqNamesInLookupCache = withAbiSnapshot,
-        )
+    override val shouldTrackChangesInLookupCache
+        get() = false
+
+    override val shouldStoreFullFqNamesInLookupCache
+        get() = withAbiSnapshot
 
     override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JSCompilerArguments) =
         IncrementalJsCachesManager(icContext, if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol, cacheDirectory)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -142,6 +142,7 @@ open class IncrementalJvmCompilerRunner(
     private val classpathChanges: ClasspathChanges,
     withAbiSnapshot: Boolean = false,
     preciseCompilationResultsBackup: Boolean = false,
+    keepIncrementalCompilationCachesInMemory: Boolean = false,
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
     workingDir,
     "caches-jvm",
@@ -150,15 +151,13 @@ open class IncrementalJvmCompilerRunner(
     outputDirs = outputDirs,
     withAbiSnapshot = withAbiSnapshot,
     preciseCompilationResultsBackup = preciseCompilationResultsBackup,
+    keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory,
 ) {
-    override fun createIncrementalCompilationContext(projectDir: File?, transaction: CompilationTransaction) =
-        IncrementalCompilationContext(
-            transaction = transaction,
-            rootProjectDir = projectDir,
-            reporter = reporter,
-            trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun,
-            storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
-        )
+    override val shouldTrackChangesInLookupCache
+        get() = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
+
+    override val shouldStoreFullFqNamesInLookupCache
+        get() = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled
 
     override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments) =
         IncrementalJvmCachesManager(icContext, args.destination?.let { File(it) }, cacheDirectory)

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt

@@ -298,6 +298,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
             kotlinScriptExtensions = kotlinScriptExtensions,
             withAbiSnapshot = icEnv.withAbiSnapshot,
             preciseCompilationResultsBackup = icEnv.preciseCompilationResultsBackup,
+            keepIncrementalCompilationCachesInMemory = icEnv.keepIncrementalCompilationCachesInMemory
         )
 
         log.info("Options for KOTLIN DAEMON: $compilationOptions")

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt

@@ -20,8 +20,9 @@ internal class IncrementalCompilationEnvironment(
     val multiModuleICSettings: MultiModuleICSettings,
     val withAbiSnapshot: Boolean = false,
     val preciseCompilationResultsBackup: Boolean = false,
+    val keepIncrementalCompilationCachesInMemory: Boolean = false,
 ) : Serializable {
     companion object {
-        const val serialVersionUID: Long = 2
+        const val serialVersionUID: Long = 3
     }
 }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt

@@ -15,6 +15,7 @@ import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessageOutputStream
 import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.Companion.jsCompilerProperty
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_JS_STDLIB_DOM_API_INCLUDED
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_ABI_SNAPSHOT
+import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_COMPILER_KEEP_INCREMENTAL_COMPILATION_CACHES_IN_MEMORY
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_COMPILER_USE_PRECISE_COMPILATION_RESULTS_BACKUP
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_JS_KARMA_BROWSERS
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ANDROID_GRADLE_PLUGIN_COMPATIBILITY_NO_WARN
@@ -511,6 +512,12 @@ internal class PropertiesProvider private constructor(private val project: Proje
     val preciseCompilationResultsBackup: Boolean
         get() = booleanProperty(KOTLIN_COMPILER_USE_PRECISE_COMPILATION_RESULTS_BACKUP) ?: false
 
+    /**
+     * This property should be enabled together with [preciseCompilationResultsBackup]
+     */
+    val keepIncrementalCompilationCachesInMemory: Boolean
+        get() = booleanProperty(KOTLIN_COMPILER_KEEP_INCREMENTAL_COMPILATION_CACHES_IN_MEMORY) ?: false
+
     /**
      * Retrieves a comma-separated list of browsers to use when running karma tests for [target]
      * @see KOTLIN_JS_KARMA_BROWSERS
@@ -593,6 +600,7 @@ internal class PropertiesProvider private constructor(private val project: Proje
         const val KOTLIN_OPTIONS_SUPPRESS_FREEARGS_MODIFICATION_WARNING = "kotlin.options.suppressFreeCompilerArgsModificationWarning"
         const val KOTLIN_NATIVE_USE_XCODE_MESSAGE_STYLE = "kotlin.native.useXcodeMessageStyle"
         const val KOTLIN_COMPILER_USE_PRECISE_COMPILATION_RESULTS_BACKUP = "kotlin.compiler.preciseCompilationResultsBackup"
+        const val KOTLIN_COMPILER_KEEP_INCREMENTAL_COMPILATION_CACHES_IN_MEMORY = "kotlin.compiler.keepIncrementalCompilationCachesInMemory"
     }
 
     companion object {

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt

@@ -191,6 +191,9 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
     @get:Internal
     internal abstract val preciseCompilationResultsBackup: Property<Boolean>
 
+    @get:Internal
+    internal abstract val keepIncrementalCompilationCachesInMemory: Property<Boolean>
+
     /** Task outputs that we don't want to include in [TaskOutputsBackup] (see [TaskOutputsBackup.outputsToRestore] for more info). */
     @get:Internal
     internal abstract val taskOutputsBackupExcludes: SetProperty<File>

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/Kotlin2JsCompile.kt

@@ -300,6 +300,7 @@ abstract class Kotlin2JsCompile @Inject constructor(
                 taskBuildCacheableOutputDirectory.get().asFile,
                 multiModuleICSettings = multiModuleICSettings,
                 preciseCompilationResultsBackup = preciseCompilationResultsBackup.get(),
+                keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory.get(),
             )
         } else null
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompile.kt

@@ -278,6 +278,7 @@ abstract class KotlinCompile @Inject constructor(
                 multiModuleICSettings = multiModuleICSettings,
                 withAbiSnapshot = useKotlinAbiSnapshot.get(),
                 preciseCompilationResultsBackup = preciseCompilationResultsBackup.get(),
+                keepIncrementalCompilationCachesInMemory = keepIncrementalCompilationCachesInMemory.get(),
             )
         } else null
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt

@@ -100,6 +100,12 @@ internal abstract class AbstractKotlinCompileConfig<TASK : AbstractKotlinCompile
             task.taskOutputsBackupExcludes.addAll(task.preciseCompilationResultsBackup.map {
                 if (it) listOf(task.destinationDirectory.get().asFile, task.taskBuildLocalStateDirectory.get().asFile) else emptyList()
             })
+            task.keepIncrementalCompilationCachesInMemory
+                .convention(task.preciseCompilationResultsBackup.map { it && propertiesProvider.keepIncrementalCompilationCachesInMemory })
+                .finalizeValueOnRead()
+            task.taskOutputsBackupExcludes.addAll(task.keepIncrementalCompilationCachesInMemory.map {
+                if (it) listOf(task.taskBuildCacheableOutputDirectory.get().asFile) else emptyList()
+            })
 
             task.incremental = false
             task.useModuleDetection.convention(false)