[BT] Implement incremental compilation within the daemon

#KT-57398 In Progress

compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api

@@ -102,6 +102,7 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Classpat
 	public abstract fun forceNonIncrementalMode (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration;
 	public abstract fun getAssuredNoClasspathSnapshotsChanges ()Z
 	public abstract fun keepIncrementalCompilationCachesInMemory (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration;
+	public abstract fun useOutputDirs (Ljava/util/Collection;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration;
 	public abstract fun usePreciseCompilationResultsBackup (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration;
 	public abstract fun usePreciseJavaTracking (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration;
 	public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration;
@@ -115,10 +116,12 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Incremen
 	public static synthetic fun forceNonIncrementalMode$default (Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;ZILjava/lang/Object;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;
 	public abstract fun getForcedNonIncrementalMode ()Z
 	public abstract fun getIncrementalCompilationCachesKeptInMemory ()Z
+	public abstract fun getOutputDirs ()Ljava/util/Set;
 	public abstract fun getPreciseCompilationResultsBackupEnabled ()Z
 	public abstract fun getPreciseJavaTrackingEnabled ()Z
 	public abstract fun getProjectDir ()Ljava/io/File;
 	public abstract fun keepIncrementalCompilationCachesInMemory (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;
+	public abstract fun useOutputDirs (Ljava/util/Collection;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;
 	public abstract fun usePreciseCompilationResultsBackup (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;
 	public abstract fun usePreciseJavaTracking (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;
 	public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;

compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt

@@ -147,6 +147,21 @@ public interface IncrementalJvmCompilationConfiguration<P : IncrementalCompilati
      * @see [forcedNonIncrementalMode]
      */
     public fun forceNonIncrementalMode(value: Boolean = true): IncrementalJvmCompilationConfiguration<P>
+
+    /**
+     * The directories that the compiler will clean in the case of fallback to non-incremental compilation.
+     *
+     * The default ones are calculated in the case of a `null` value as a set of the incremental compilation working directory
+     * passed to [JvmCompilationConfiguration.useIncrementalCompilation] and the classes output directory from the compiler arguments.
+     *
+     * If the value is set explicitly, it must contain the above-mentioned default directories.
+     */
+    public val outputDirs: Set<File>?
+
+    /**
+     * @see [IncrementalJvmCompilationConfiguration.outputDirs]]
+     */
+    public fun useOutputDirs(outputDirs: Collection<File>): IncrementalJvmCompilationConfiguration<P>
 }
 
 /**
@@ -180,6 +195,8 @@ public interface ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration :
     override fun keepIncrementalCompilationCachesInMemory(value: Boolean): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration
 
     override fun forceNonIncrementalMode(value: Boolean): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration
+
+    override fun useOutputDirs(outputDirs: Collection<File>): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration
 }
 
 /**

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt

@@ -58,8 +58,7 @@ internal object CompilationServiceImpl : CompilationService {
             "Initial JVM compilation configuration object must be acquired from the `makeJvmCompilationConfiguration` method."
         }
         val loggerAdapter = KotlinLoggerMessageCollectorAdapter(compilationConfig.logger)
-        val selectedStrategy = strategyConfig.selectedStrategy
-        return when (selectedStrategy) {
+        return when (val selectedStrategy = strategyConfig.selectedStrategy) {
             is CompilerExecutionStrategy.InProcess -> compileInProcess(loggerAdapter, sources, arguments)
             is CompilerExecutionStrategy.Daemon -> compileWithinDaemon(
                 projectId,

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt

@@ -41,7 +41,9 @@ internal class JvmCompilationConfigurationImpl(
         sourcesChanges: SourcesChanges,
         approachParameters: P,
         options: IncrementalJvmCompilationConfiguration<P>,
-    ) = TODO("Incremental compilation is not yet supported to run via the Build Tools API")
+    ) {
+        aggregatedIcConfiguration = AggregatedIcConfiguration(options, approachParameters, sourcesChanges, workingDirectory)
+    }
 }
 
 internal abstract class JvmIncrementalCompilationConfigurationImpl<P : IncrementalCompilationApproachParameters>(
@@ -50,6 +52,7 @@ internal abstract class JvmIncrementalCompilationConfigurationImpl<P : Increment
     override var incrementalCompilationCachesKeptInMemory: Boolean = false,
     override var projectDir: File? = null,
     override var forcedNonIncrementalMode: Boolean = false,
+    override var outputDirs: Set<File> = emptySet(),
 ) : IncrementalJvmCompilationConfiguration<P> {
     override fun useProjectDir(projectDir: File): IncrementalJvmCompilationConfiguration<P> {
         this.projectDir = projectDir
@@ -75,6 +78,11 @@ internal abstract class JvmIncrementalCompilationConfigurationImpl<P : Increment
         forcedNonIncrementalMode = value
         return this
     }
+
+    override fun useOutputDirs(outputDirs: Collection<File>): IncrementalJvmCompilationConfiguration<P> {
+        this.outputDirs = outputDirs.toSet()
+        return this
+    }
 }
 
 internal class ClasspathSnapshotBasedIncrementalJvmCompilationConfigurationImpl(
@@ -111,4 +119,9 @@ internal class ClasspathSnapshotBasedIncrementalJvmCompilationConfigurationImpl(
         assuredNoClasspathSnapshotsChanges = value
         return this
     }
+
+    override fun useOutputDirs(outputDirs: Collection<File>): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration {
+        super.useOutputDirs(outputDirs)
+        return this
+    }
 }

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt

@@ -6,8 +6,12 @@
 package org.jetbrains.kotlin.buildtools.internal
 
 import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
+import org.jetbrains.kotlin.buildtools.api.SourcesChanges
+import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters
 import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration
 import org.jetbrains.kotlin.daemon.common.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges
+import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles
 import java.io.Serializable
 import java.rmi.server.UnicastRemoteObject
 
@@ -17,8 +21,48 @@ internal val JvmCompilationConfigurationImpl.asDaemonCompilationOptions: Compila
         val reportCategories = arrayOf(ReportCategory.COMPILER_MESSAGE.code, ReportCategory.IC_MESSAGE.code) // TODO: automagically compute the value, related to BasicCompilerServicesWithResultsFacadeServer
         val reportSeverity = ReportSeverity.INFO.code // TODO: automagically compute the value, related to BasicCompilerServicesWithResultsFacadeServer
         val requestedCompilationResults = emptyArray<Int>() // TODO: automagically compute the value, related to DaemonCompilationResults
-        return when (aggregatedIcConfiguration?.options) {
-            is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration -> TODO("Incremental compilation within the daemon is not yet supported")
+        val aggregatedIcConfiguration = aggregatedIcConfiguration
+        return when (val options = aggregatedIcConfiguration?.options) {
+            is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration -> {
+                val sourcesChanges = aggregatedIcConfiguration.sourcesChanges
+                val params = aggregatedIcConfiguration.parameters as ClasspathSnapshotBasedIncrementalCompilationApproachParameters
+                val snapshotFiles =
+                    ClasspathSnapshotFiles(params.newClasspathSnapshotFiles, params.shrunkClasspathSnapshot.parentFile)
+                IncrementalCompilationOptions(
+                    areFileChangesKnown = sourcesChanges is SourcesChanges.Known,
+                    modifiedFiles = if (sourcesChanges is SourcesChanges.Known) sourcesChanges.modifiedFiles else null,
+                    deletedFiles = if (sourcesChanges is SourcesChanges.Known) sourcesChanges.removedFiles else null,
+                    classpathChanges = when {
+                        !params.shrunkClasspathSnapshot.exists() -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot(
+                            snapshotFiles
+                        )
+                        options.forcedNonIncrementalMode -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun(
+                            snapshotFiles
+                        )
+                        options.assuredNoClasspathSnapshotsChanges -> ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges(
+                            snapshotFiles
+                        )
+                        else -> {
+                            ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler(snapshotFiles)
+                        }
+                    },
+                    workingDir = aggregatedIcConfiguration.workingDir,
+                    compilerMode = CompilerMode.INCREMENTAL_COMPILER,
+                    targetPlatform = CompileService.TargetPlatform.JVM,
+                    reportCategories = reportCategories,
+                    reportSeverity = reportSeverity,
+                    requestedCompilationResults = requestedCompilationResults,
+                    usePreciseJavaTracking = options.preciseJavaTrackingEnabled,
+                    outputFiles = options.outputDirs,
+                    multiModuleICSettings = null, // required only for the build history approach
+                    modulesInfo = null, // required only for the build history approach
+                    rootProjectDir = options.projectDir,
+                    kotlinScriptExtensions = ktsExtensionsAsArray,
+                    withAbiSnapshot = false,
+                    preciseCompilationResultsBackup = options.preciseCompilationResultsBackupEnabled,
+                    keepIncrementalCompilationCachesInMemory = options.incrementalCompilationCachesKeptInMemory,
+                )
+            }
             else -> CompilationOptions(
                 compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER,
                 targetPlatform = CompileService.TargetPlatform.JVM,

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt

@@ -67,7 +67,7 @@ class IncrementalCompilationOptions(
     /**
      * Directories that should be cleared when IC decides to rebuild
      */
-    val outputFiles: List<File>,
+    val outputFiles: Collection<File>? = null,
     val multiModuleICSettings: MultiModuleICSettings? = null,
     val modulesInfo: IncrementalModuleInfo? = null,
     /**
@@ -87,7 +87,7 @@ class IncrementalCompilationOptions(
     kotlinScriptExtensions
 ) {
     companion object {
-        const val serialVersionUID: Long = 2
+        const val serialVersionUID: Long = 3
     }
 
     override fun toString(): String {

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildToolsApiJvmCompilationIT.kt

@@ -57,7 +57,6 @@ class BuildToolsApiJvmCompilationIT : KGPBaseTest() {
     }
 
     @GradleTest
-    @Disabled
     @DisplayName("Simple project incremental compilation within the daemon")
     fun compileJvmIncrementalWithinDaemon(gradleVersion: GradleVersion) = testSimpleProject(
         gradleVersion, defaultBuildOptions.copy(

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt

@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
 import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
 import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
 import org.jetbrains.kotlin.buildtools.api.*
+import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.compilerRunner.GradleKotlinCompilerWorkArguments
 import org.jetbrains.kotlin.compilerRunner.asFinishLogMessage
@@ -26,9 +27,17 @@ import org.jetbrains.kotlin.gradle.plugin.internal.BuildIdService
 import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
 import org.jetbrains.kotlin.gradle.tasks.throwExceptionIfCompilationFailed
+import org.jetbrains.kotlin.incremental.ChangedFiles
+import org.jetbrains.kotlin.incremental.ClasspathChanges
 import org.slf4j.LoggerFactory
 import java.io.File
-import java.util.*
+
+@OptIn(ExperimentalBuildToolsApi::class)
+private val ChangedFiles.asSourcesChanges: SourcesChanges
+    get() = when (this) {
+        is ChangedFiles.Known -> SourcesChanges.Known(modified, removed)
+        is ChangedFiles.Unknown -> SourcesChanges.Unknown
+    }
 
 @OptIn(ExperimentalBuildToolsApi::class)
 internal abstract class BuildToolsApiCompilationWork : WorkAction<BuildToolsApiCompilationWork.BuildToolsApiCompilationParameters> {
@@ -81,6 +90,34 @@ internal abstract class BuildToolsApiCompilationWork : WorkAction<BuildToolsApiC
                 }
             }
             val jvmCompilationConfig = compilationService.makeJvmCompilationConfiguration()
+                .useLogger(log)
+                .useKotlinScriptFilenameExtensions(workArguments.kotlinScriptExtensions.toList())
+            val icEnv = workArguments.incrementalCompilationEnvironment
+            val classpathChanges = icEnv?.classpathChanges
+            if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
+                val classpathSnapshotsConfig = jvmCompilationConfig.makeClasspathSnapshotBasedIncrementalCompilationConfiguration()
+                    .useProjectDir(icEnv.rootProjectDir)
+                    .usePreciseJavaTracking(icEnv.usePreciseJavaTracking)
+                    .usePreciseCompilationResultsBackup(icEnv.preciseCompilationResultsBackup)
+                    .keepIncrementalCompilationCachesInMemory(icEnv.keepIncrementalCompilationCachesInMemory)
+                    .useOutputDirs(workArguments.outputFiles)
+                    .forceNonIncrementalMode(classpathChanges !is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun)
+                val classpathSnapshotsParameters = ClasspathSnapshotBasedIncrementalCompilationApproachParameters(
+                    classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles,
+                    classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
+                )
+                when (classpathChanges) {
+                    is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges -> classpathSnapshotsConfig.assureNoClasspathSnapshotsChanges()
+                    is ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun -> classpathSnapshotsConfig.forceNonIncrementalMode()
+                    else -> {}
+                }
+                jvmCompilationConfig.useIncrementalCompilation(
+                    icEnv.workingDir,
+                    icEnv.changedFiles.asSourcesChanges,
+                    classpathSnapshotsParameters,
+                    classpathSnapshotsConfig,
+                )
+            }
             val result = compilationService.compileJvm(
                 buildId,
                 executionConfig,