[BT] Add support for in-process incremental compilation

#KT-61865 Fixed
2023-09-13 05:45:00 +02:00
parent fb1d2278a0
commit b460c06907
14 changed files with 139 additions and 61 deletions
@@ -5,5 +5,5 @@

 package org.jetbrains.kotlin.build

-val DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS = listOf("kt", "kts")
+val DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS = setOf("kt", "kts")

@@ -22,7 +22,7 @@ import java.io.IOException
 fun File.isJavaFile() =
        extension.equals("java", ignoreCase = true)

-fun File.isKotlinFile(sourceFilesExtensions: List<String>): Boolean =
+fun File.isKotlinFile(sourceFilesExtensions: Collection<String>): Boolean =
    !isJavaFile() && sourceFilesExtensions.any { it.equals(extension, ignoreCase = true) }

 fun File.isClassFile(): Boolean =
@@ -5,6 +5,8 @@

 package org.jetbrains.kotlin.buildtools.internal

+import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
+import org.jetbrains.kotlin.build.report.DoNothingBuildReporter
 import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
 import org.jetbrains.kotlin.buildtools.api.*
 import org.jetbrains.kotlin.buildtools.api.jvm.*
@@ -21,7 +23,12 @@ import org.jetbrains.kotlin.daemon.client.BasicCompilerServicesWithResultsFacade
 import org.jetbrains.kotlin.daemon.common.CompilerId
 import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions
 import org.jetbrains.kotlin.daemon.common.filterExtractProps
+import org.jetbrains.kotlin.incremental.ClasspathChanges
+import org.jetbrains.kotlin.incremental.IncrementalJvmCompilerRunner
 import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter
+import org.jetbrains.kotlin.incremental.extractKotlinSourcesFromFreeCompilerArguments
+import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
+import org.jetbrains.kotlin.incremental.storage.FileLocations
 import org.jetbrains.kotlin.scripting.compiler.plugin.impl.reporter
 import org.jetbrains.kotlin.scripting.definitions.ScriptDefinitionsFromClasspathDiscoverySource
 import java.io.File
@@ -32,7 +39,7 @@ import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration
 private val ExitCode.asCompilationResult
    get() = when (this) {
        ExitCode.OK -> CompilationResult.COMPILATION_SUCCESS
-        ExitCode.COMPILATION_ERROR -> CompilationResult.COMPILER_INTERNAL_ERROR
+        ExitCode.COMPILATION_ERROR -> CompilationResult.COMPILATION_ERROR
        ExitCode.INTERNAL_ERROR -> CompilationResult.COMPILER_INTERNAL_ERROR
        ExitCode.OOM_ERROR -> CompilationResult.COMPILATION_OOM_ERROR
        else -> error("Unexpected exit code: $this")
@@ -55,7 +62,7 @@ internal object CompilationServiceImpl : CompilationService {
        strategyConfig: CompilerExecutionStrategyConfiguration,
        compilationConfig: JvmCompilationConfiguration,
        sources: List<File>,
-        arguments: List<String>
+        arguments: List<String>,
    ): CompilationResult {
        check(strategyConfig is CompilerExecutionStrategyConfigurationImpl) {
            "Initial strategy configuration object must be acquired from the `makeCompilerExecutionStrategyConfiguration` method."
@@ -65,7 +72,7 @@ internal object CompilationServiceImpl : CompilationService {
        }
        val loggerAdapter = KotlinLoggerMessageCollectorAdapter(compilationConfig.logger)
        return when (val selectedStrategy = strategyConfig.selectedStrategy) {
-            is CompilerExecutionStrategy.InProcess -> compileInProcess(loggerAdapter, sources, arguments)
+            is CompilerExecutionStrategy.InProcess -> compileInProcess(loggerAdapter, compilationConfig, sources, arguments)
            is CompilerExecutionStrategy.Daemon -> compileWithinDaemon(
                projectId,
                loggerAdapter,
@@ -94,8 +101,9 @@ internal object CompilationServiceImpl : CompilationService {

    private fun compileInProcess(
        loggerAdapter: KotlinLoggerMessageCollectorAdapter,
+        compilationConfiguration: JvmCompilationConfigurationImpl,
        sources: List<File>,
-        arguments: List<String>
+        arguments: List<String>,
    ): CompilationResult {
        val compiler = K2JVMCompiler()
        val parsedArguments = compiler.createArguments()
@@ -103,9 +111,47 @@ internal object CompilationServiceImpl : CompilationService {
        validateArguments(parsedArguments.errors)?.let {
            throw CompilerArgumentsParseException(it)
        }
-        parsedArguments.freeArgs += sources.map { it.absolutePath } // TODO: they're not explicitly passed yet
        loggerAdapter.report(CompilerMessageSeverity.INFO, arguments.toString())
-        return compiler.exec(loggerAdapter, Services.EMPTY, parsedArguments).asCompilationResult
+        val aggregatedIcConfiguration = compilationConfiguration.aggregatedIcConfiguration
+        return when (val options = aggregatedIcConfiguration?.options) {
+            is ClasspathSnapshotBasedIncrementalJvmCompilationConfigurationImpl -> {
+                val kotlinFilenameExtensions =
+                    (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS + compilationConfiguration.kotlinScriptFilenameExtensions)
+
+                @Suppress("DEPRECATION") // TODO: get rid of that parsing KT-62759
+                val kotlinSources = extractKotlinSourcesFromFreeCompilerArguments(parsedArguments, kotlinFilenameExtensions) + sources
+
+                @Suppress("UNCHECKED_CAST")
+                val classpathChanges =
+                    (aggregatedIcConfiguration as AggregatedIcConfiguration<ClasspathSnapshotBasedIncrementalCompilationApproachParameters>).classpathChanges
+                val incrementalCompiler = IncrementalJvmCompilerRunner(
+                    aggregatedIcConfiguration.workingDir,
+                    DoNothingBuildReporter,
+                    buildHistoryFile = null,
+                    modulesApiHistory = EmptyModulesApiHistory,
+                    usePreciseJavaTracking = options.preciseJavaTrackingEnabled,
+                    outputDirs = options.outputDirs,
+                    kotlinSourceFilesExtensions = kotlinFilenameExtensions,
+                    classpathChanges = classpathChanges,
+                    withAbiSnapshot = false,
+                    preciseCompilationResultsBackup = options.preciseCompilationResultsBackupEnabled,
+                    keepIncrementalCompilationCachesInMemory = options.incrementalCompilationCachesKeptInMemory
+                )
+                val rootProjectDir = options.rootProjectDir
+                val buildDir = options.buildDir
+                parsedArguments.incrementalCompilation = true
+                incrementalCompiler.compile(
+                    kotlinSources, parsedArguments, loggerAdapter, aggregatedIcConfiguration.sourcesChanges.asChangedFiles,
+                    fileLocations = if (rootProjectDir != null && buildDir != null) {
+                        FileLocations(rootProjectDir, buildDir)
+                    } else null
+                ).asCompilationResult
+            }
+            else -> {
+                parsedArguments.freeArgs += sources.map { it.absolutePath }
+                compiler.exec(loggerAdapter, Services.EMPTY, parsedArguments).asCompilationResult
+            }
+        }
    }

    private fun compileWithinDaemon(
@@ -114,7 +160,7 @@ internal object CompilationServiceImpl : CompilationService {
        daemonConfiguration: CompilerExecutionStrategy.Daemon,
        compilationConfiguration: JvmCompilationConfigurationImpl,
        sources: List<File>,
-        arguments: List<String>
+        arguments: List<String>,
    ): CompilationResult {
        val compilerId = CompilerId.makeCompilerId(getCurrentClasspath())
        val sessionIsAliveFlagFile = buildIdToSessionFlagFile.computeIfAbsent(projectId) {
@@ -144,7 +190,7 @@ internal object CompilationServiceImpl : CompilationService {
        val daemonCompileOptions = compilationConfiguration.asDaemonCompilationOptions
        val exitCode = daemon.compile(
            sessionId,
-            arguments.toTypedArray() + sources.map { it.absolutePath }, // TODO: the sources not explicitly passed yet
+            arguments.toTypedArray() + sources.map { it.absolutePath }, // TODO: pass the sources explicitly KT-62759
            daemonCompileOptions,
            BasicCompilerServicesWithResultsFacadeServer(loggerAdapter),
            DaemonCompilationResults()
@@ -25,27 +25,15 @@ internal val JvmCompilationConfigurationImpl.asDaemonCompilationOptions: Compila
        return when (val options = aggregatedIcConfiguration?.options) {
            is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration -> {
                val sourcesChanges = aggregatedIcConfiguration.sourcesChanges
-                val params = aggregatedIcConfiguration.parameters as ClasspathSnapshotBasedIncrementalCompilationApproachParameters
-                val snapshotFiles =
-                    ClasspathSnapshotFiles(params.newClasspathSnapshotFiles, params.shrunkClasspathSnapshot.parentFile)
+
+                @Suppress("UNCHECKED_CAST")
+                val classpathChanges =
+                    (aggregatedIcConfiguration as AggregatedIcConfiguration<ClasspathSnapshotBasedIncrementalCompilationApproachParameters>).classpathChanges
                IncrementalCompilationOptions(
                    areFileChangesKnown = sourcesChanges is SourcesChanges.Known,
                    modifiedFiles = if (sourcesChanges is SourcesChanges.Known) sourcesChanges.modifiedFiles else null,
                    deletedFiles = if (sourcesChanges is SourcesChanges.Known) sourcesChanges.removedFiles else null,
-                    classpathChanges = when {
-                        !params.shrunkClasspathSnapshot.exists() -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot(
-                            snapshotFiles
-                        )
-                        options.forcedNonIncrementalMode -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun(
-                            snapshotFiles
-                        )
-                        options.assuredNoClasspathSnapshotsChanges -> ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges(
-                            snapshotFiles
-                        )
-                        else -> {
-                            ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler(snapshotFiles)
-                        }
-                    },
+                    classpathChanges = classpathChanges,
                    workingDir = aggregatedIcConfiguration.workingDir,
                    compilerMode = CompilerMode.INCREMENTAL_COMPILER,
                    targetPlatform = CompileService.TargetPlatform.JVM,
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.internal
+
+import org.jetbrains.kotlin.buildtools.api.SourcesChanges
+import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters
+import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration
+import org.jetbrains.kotlin.incremental.ChangedFiles
+import org.jetbrains.kotlin.incremental.ClasspathChanges
+import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles
+
+internal val SourcesChanges.asChangedFiles
+    get() = when (this) {
+        is SourcesChanges.Known -> ChangedFiles.Known(modifiedFiles, removedFiles)
+        is SourcesChanges.ToBeCalculated -> ChangedFiles.Unknown() // TODO: add proper support for SourcesChanges.ToBeCalculated
+        is SourcesChanges.Unknown -> ChangedFiles.Unknown()
+    }
+
+internal val AggregatedIcConfiguration<ClasspathSnapshotBasedIncrementalCompilationApproachParameters>.classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled
+    get() {
+        check(options is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration) {
+            "options expected to be an instance of ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration"
+        }
+        val params = parameters
+        val snapshotFiles = ClasspathSnapshotFiles(params.newClasspathSnapshotFiles, params.shrunkClasspathSnapshot.parentFile)
+        return when {
+            !params.shrunkClasspathSnapshot.exists() -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot(
+                snapshotFiles
+            )
+            options.forcedNonIncrementalMode -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun(
+                snapshotFiles
+            )
+            options.assuredNoClasspathSnapshotsChanges -> ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges(
+                snapshotFiles
+            )
+            else -> {
+                ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler(snapshotFiles)
+            }
+        }
+    }
@@ -539,16 +539,8 @@ abstract class CompileServiceImplBase(
        reporter: RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>
    ): ExitCode {
        reporter.startMeasureGc()
-        val allKotlinFiles = arrayListOf<File>()
-        val freeArgsWithoutKotlinFiles = arrayListOf<String>()
-        args.freeArgs.forEach {
-            if (it.endsWith(".kt") && File(it).exists()) {
-                allKotlinFiles.add(File(it))
-            } else {
-                freeArgsWithoutKotlinFiles.add(it)
-            }
-        }
-        args.freeArgs = freeArgsWithoutKotlinFiles
+        @Suppress("DEPRECATION") // TODO: get rid of that parsing KT-62759
+        val allKotlinFiles = extractKotlinSourcesFromFreeCompilerArguments(args, setOf("kt"))

        val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) {
            ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!)
@@ -591,19 +583,10 @@ abstract class CompileServiceImplBase(
    ): ExitCode {
        reporter.startMeasureGc()
        val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS +
-                (incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct()
-        val dotExtensions = allKotlinExtensions.map { ".$it" }
-        val freeArgs = arrayListOf<String>()
-        val allKotlinFiles = arrayListOf<File>()
-        for (arg in k2jvmArgs.freeArgs) {
-            val file = File(arg)
-            if (file.isFile && dotExtensions.any { ext -> file.path.endsWith(ext, ignoreCase = true) }) {
-                allKotlinFiles.add(file)
-            } else {
-                freeArgs.add(arg)
-            }
-        }
-        k2jvmArgs.freeArgs = freeArgs
+                (incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).toSet()
+
+        @Suppress("DEPRECATION") // TODO: get rid of that parsing KT-62759
+        val allKotlinFiles = extractKotlinSourcesFromFreeCompilerArguments(k2jvmArgs, allKotlinExtensions)

        val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) {
            ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!)
@@ -12,7 +12,7 @@ import java.io.File
 class DirtyFilesContainer(
    private val caches: IncrementalCachesManager<*>,
    private val reporter: ICReporter,
-    private val sourceFilesExtensions: List<String>
+    private val sourceFilesExtensions: Set<String>
 ) {
    private val myDirtyFiles = HashSet<File>()

@@ -81,7 +81,7 @@ abstract class IncrementalCompilerRunner<
    private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
    protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
    private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
-    protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
+    protected open val kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS

    /**
     * Creates an instance of [IncrementalCompilationContext] that holds common incremental compilation context mostly required for [CacheManager]
@@ -663,3 +663,23 @@ abstract class IncrementalCompilerRunner<
        }
    }
 }
+
+@Deprecated("Temporary function to reuse the logic. KT-62759")
+fun extractKotlinSourcesFromFreeCompilerArguments(
+    compilerArguments: CommonCompilerArguments,
+    kotlinFilenameExtensions: Set<String>,
+): List<File> {
+    val dotExtensions = kotlinFilenameExtensions.map { ".$it" }
+    val freeArgs = arrayListOf<String>()
+    val allKotlinFiles = arrayListOf<File>()
+    for (arg in compilerArguments.freeArgs) {
+        val file = File(arg)
+        if (file.isFile && dotExtensions.any { ext -> file.path.endsWith(ext, ignoreCase = true) }) {
+            allKotlinFiles.add(file)
+        } else {
+            freeArgs.add(arg)
+        }
+    }
+    compilerArguments.freeArgs = freeArgs
+    return allKotlinFiles
+}
@@ -64,7 +64,7 @@ open class IncrementalFirJvmCompilerRunner(
    buildHistoryFile: File?,
    outputDirs: Collection<File>?,
    modulesApiHistory: ModulesApiHistory,
-    kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
+    kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
    classpathChanges: ClasspathChanges
 ) : IncrementalJvmCompilerRunner(
    workingDir,
@@ -66,7 +66,7 @@ open class IncrementalJvmCompilerRunner(
    buildHistoryFile: File?,
    outputDirs: Collection<File>?,
    private val modulesApiHistory: ModulesApiHistory,
-    override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
+    override val kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
    private val classpathChanges: ClasspathChanges,
    withAbiSnapshot: Boolean = false,
    preciseCompilationResultsBackup: Boolean = false,
@@ -18,7 +18,7 @@ class IncrementalFirJvmCompilerTestRunner(
    buildHistoryFile: File,
    outputDirs: Collection<File>?,
    modulesApiHistory: ModulesApiHistory,
-    kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
+    kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
    classpathChanges: ClasspathChanges,
    val testLookupTracker: TestLookupTracker
 ) : IncrementalFirJvmCompilerRunner(
@@ -19,7 +19,7 @@ class IncrementalJvmCompilerTestRunner(
    buildHistoryFile: File,
    outputDirs: Collection<File>?,
    modulesApiHistory: ModulesApiHistory,
-    override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
+    override val kotlinSourceFilesExtensions: Set<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
    classpathChanges: ClasspathChanges,
    withAbiSnapshot: Boolean = false,
    val testLookupTracker: TestLookupTracker
@@ -232,7 +232,7 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() {

        val testDir = File(path)
        val workToOriginalFileMap = HashMap(copyTestSources(testDir, srcDir, filePrefix = ""))
-        var dirtyFiles = srcDir.walk().filterTo(HashSet()) { it.isKotlinFile(listOf("kt", "kts")) }
+        var dirtyFiles = srcDir.walk().filterTo(HashSet()) { it.isKotlinFile(setOf("kt", "kts")) }
        val steps = getModificationsToPerform(
            testDir,
            moduleNames = null,
@@ -9,10 +9,9 @@ import org.gradle.testkit.runner.BuildResult
 import org.gradle.util.GradleVersion
 import org.jetbrains.kotlin.gradle.internals.asFinishLogMessage
 import org.jetbrains.kotlin.gradle.plugin.diagnostics.KotlinToolingDiagnostics
-import org.jetbrains.kotlin.gradle.testbase.*
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
 import org.jetbrains.kotlin.gradle.tasks.USING_JVM_INCREMENTAL_COMPILATION_MESSAGE
-import org.junit.jupiter.api.Disabled
+import org.jetbrains.kotlin.gradle.testbase.*
 import org.junit.jupiter.api.DisplayName

@DisplayName("JVM compilation via the Build Tools API")
@@ -69,12 +68,11 @@ class BuildToolsApiJvmCompilationIT : KGPBaseTest() {
    }

    @GradleTest
-    @Disabled
    @DisplayName("Simple project incremental in-process compilation")
    fun compileJvmInProcessIncremental(gradleVersion: GradleVersion) = testSimpleProject(
        gradleVersion, defaultBuildOptions.copy(
            compilerExecutionStrategy = KotlinCompilerExecutionStrategy.IN_PROCESS,
-            incremental = false,
+            incremental = true,
        )
    ) {
        assertOutputContains(KotlinCompilerExecutionStrategy.IN_PROCESS.asFinishLogMessage)