[BT] Implement non-incremental compilation within the daemon

#KT-57398 In Progress

compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api

@@ -26,7 +26,7 @@ public final class org/jetbrains/kotlin/buildtools/api/CompilerArgumentsParseExc
 }
 
 public abstract interface class org/jetbrains/kotlin/buildtools/api/CompilerExecutionStrategyConfiguration {
-	public abstract fun useDaemonStrategy (Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlin/buildtools/api/CompilerExecutionStrategyConfiguration;
+	public abstract fun useDaemonStrategy (Ljava/util/List;)Lorg/jetbrains/kotlin/buildtools/api/CompilerExecutionStrategyConfiguration;
 	public abstract fun useInProcessStrategy ()Lorg/jetbrains/kotlin/buildtools/api/CompilerExecutionStrategyConfiguration;
 }
 

compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/CompilerExecutionStrategyConfiguration.kt

@@ -23,11 +23,9 @@ public interface CompilerExecutionStrategyConfiguration {
     /**
      * Marks the compilation to be run in Kotlin daemon launched as a separate process and shared across similar compilation requests.
      * See Kotlin daemon documentation here: https://kotlinlang.org/docs/gradle-compilation-and-caches.html#the-kotlin-daemon-and-how-to-use-it-with-gradle
-     * @param sessionDir a directory for storing data related to the daemon work session (should not be shared with other compilations)
      * @param jvmArguments a list of JVM startup arguments for the daemon
      */
     public fun useDaemonStrategy(
-        sessionDir: File,
         jvmArguments: List<String>,
     ): CompilerExecutionStrategyConfiguration
 }

compiler/build-tools/kotlin-build-tools-impl/build.gradle.kts

@@ -8,7 +8,13 @@ dependencies {
     implementation(kotlinStdlib())
     compileOnly(project(":compiler:cli"))
     compileOnly(project(":compiler:cli-js"))
+    compileOnly(project(":kotlin-build-common"))
+    compileOnly(project(":daemon-common"))
+    compileOnly(project(":kotlin-daemon-client"))
+    compileOnly(project(":compiler:incremental-compilation-impl"))
+    compileOnly(project(":kotlin-compiler-runner-unshaded"))
     runtimeOnly(project(":kotlin-compiler-embeddable"))
+    runtimeOnly(project(":kotlin-compiler-runner"))
 }
 
 publish()

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt

@@ -12,8 +12,15 @@ import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments
 import org.jetbrains.kotlin.cli.common.arguments.validateArguments
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
+import org.jetbrains.kotlin.compilerRunner.KotlinCompilerRunnerUtils
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.daemon.client.BasicCompilerServicesWithResultsFacadeServer
+import org.jetbrains.kotlin.daemon.common.CompilerId
+import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions
+import org.jetbrains.kotlin.daemon.common.filterExtractProps
 import java.io.File
+import java.net.URLClassLoader
+import java.util.concurrent.ConcurrentHashMap
 
 private val ExitCode.asCompilationResult
     get() = when (this) {
@@ -24,7 +31,11 @@ private val ExitCode.asCompilationResult
         else -> error("Unexpected exit code: $this")
     }
 
+private fun getCurrentClasspath() = (CompilationServiceImpl::class.java.classLoader as URLClassLoader).urLs.map { File(it.file) }
+
 internal object CompilationServiceImpl : CompilationService {
+    private val buildIdToSessionFlagFile: MutableMap<ProjectId, File> = ConcurrentHashMap()
+
     override fun calculateClasspathSnapshot(classpathEntry: File): ClasspathEntrySnapshot {
         TODO("Calculating classpath snapshots via the Build Tools API is not yet implemented: KT-57565")
     }
@@ -47,14 +58,23 @@ internal object CompilationServiceImpl : CompilationService {
             "Initial JVM compilation configuration object must be acquired from the `makeJvmCompilationConfiguration` method."
         }
         val loggerAdapter = KotlinLoggerMessageCollectorAdapter(compilationConfig.logger)
-        return when (strategyConfig.selectedStrategy) {
+        val selectedStrategy = strategyConfig.selectedStrategy
+        return when (selectedStrategy) {
             is CompilerExecutionStrategy.InProcess -> compileInProcess(loggerAdapter, sources, arguments)
-            is CompilerExecutionStrategy.Daemon -> TODO("The daemon strategy is not yet supported in the Build Tools API")
+            is CompilerExecutionStrategy.Daemon -> compileWithinDaemon(
+                projectId,
+                loggerAdapter,
+                selectedStrategy,
+                compilationConfig,
+                sources,
+                arguments
+            )
         }
     }
 
     override fun finishProjectCompilation(projectId: ProjectId) {
-
+        val file = buildIdToSessionFlagFile.remove(projectId) ?: return
+        file.delete()
     }
 
     private fun compileInProcess(
@@ -68,10 +88,59 @@ internal object CompilationServiceImpl : CompilationService {
         validateArguments(parsedArguments.errors)?.let {
             throw CompilerArgumentsParseException(it)
         }
-        parsedArguments.freeArgs += sources.map { it.absolutePath } // TODO: they're not actually passed yet
+        parsedArguments.freeArgs += sources.map { it.absolutePath } // TODO: they're not explicitly passed yet
         loggerAdapter.report(CompilerMessageSeverity.INFO, arguments.toString())
         return compiler.exec(loggerAdapter, Services.EMPTY, parsedArguments).asCompilationResult
     }
+
+    private fun compileWithinDaemon(
+        projectId: ProjectId,
+        loggerAdapter: KotlinLoggerMessageCollectorAdapter,
+        daemonConfiguration: CompilerExecutionStrategy.Daemon,
+        compilationConfiguration: JvmCompilationConfigurationImpl,
+        sources: List<File>,
+        arguments: List<String>
+    ): CompilationResult {
+        val compilerId = CompilerId.makeCompilerId(getCurrentClasspath())
+        val sessionIsAliveFlagFile = buildIdToSessionFlagFile.computeIfAbsent(projectId) {
+            createSessionIsAliveFlagFile()
+        }
+
+        val jvmOptions = configureDaemonJVMOptions(
+            inheritMemoryLimits = true,
+            inheritOtherJvmOptions = false,
+            inheritAdditionalProperties = true
+        ).also { opts ->
+            if (daemonConfiguration.jvmArguments.isNotEmpty()) {
+                opts.jvmParams.addAll(
+                    daemonConfiguration.jvmArguments.filterExtractProps(opts.mappers, "", opts.restMapper)
+                )
+            }
+        }
+
+        val (daemon, sessionId) = KotlinCompilerRunnerUtils.newDaemonConnection(
+            compilerId,
+            clientIsAliveFile,
+            sessionIsAliveFlagFile,
+            loggerAdapter,
+            false,
+            daemonJVMOptions = jvmOptions
+        ) ?: error("Can't get connection")
+        val daemonCompileOptions = compilationConfiguration.asDaemonCompilationOptions
+        val exitCode = daemon.compile(
+            sessionId,
+            arguments.toTypedArray() + sources.map { it.absolutePath }, // TODO: the sources not explicitly passed yet
+            daemonCompileOptions,
+            BasicCompilerServicesWithResultsFacadeServer(loggerAdapter),
+            DaemonCompilationResults()
+        ).get()
+
+        return (ExitCode.entries.find { it.code == exitCode } ?: if (exitCode == 0) {
+            ExitCode.OK
+        } else {
+            ExitCode.COMPILATION_ERROR
+        }).asCompilationResult
+    }
 }
 
 internal class CompilationServiceProxy : CompilationService by CompilationServiceImpl

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilerExecutionStrategyConfigurationImpl.kt

@@ -11,19 +11,20 @@ import java.io.File
 internal sealed interface CompilerExecutionStrategy {
     data object InProcess : CompilerExecutionStrategy
 
-    data class Daemon(val sessionDir: File, val jvmArguments: List<String>) : CompilerExecutionStrategy
+    data class Daemon(val jvmArguments: List<String>) : CompilerExecutionStrategy
 }
 
 internal class CompilerExecutionStrategyConfigurationImpl : CompilerExecutionStrategyConfiguration {
     internal var selectedStrategy: CompilerExecutionStrategy = CompilerExecutionStrategy.InProcess
+        private set
 
     override fun useInProcessStrategy(): CompilerExecutionStrategyConfiguration {
         selectedStrategy = CompilerExecutionStrategy.InProcess
         return this
     }
 
-    override fun useDaemonStrategy(sessionDir: File, jvmArguments: List<String>): CompilerExecutionStrategyConfiguration {
-        selectedStrategy = CompilerExecutionStrategy.Daemon(sessionDir, jvmArguments)
+    override fun useDaemonStrategy(jvmArguments: List<String>): CompilerExecutionStrategyConfiguration {
+        selectedStrategy = CompilerExecutionStrategy.Daemon(jvmArguments)
         return this
     }
 }

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt

@@ -10,10 +10,20 @@ import org.jetbrains.kotlin.buildtools.api.SourcesChanges
 import org.jetbrains.kotlin.buildtools.api.jvm.*
 import java.io.File
 
+internal data class AggregatedIcConfiguration<P : IncrementalCompilationApproachParameters>(
+    val options: IncrementalJvmCompilationConfiguration<P>,
+    val parameters: P,
+    val sourcesChanges: SourcesChanges,
+    val workingDir: File,
+)
+
 internal class JvmCompilationConfigurationImpl(
     override var kotlinScriptFilenameExtensions: Set<String> = emptySet(),
     override var logger: KotlinLogger = DefaultKotlinLogger,
 ) : JvmCompilationConfiguration {
+    internal var aggregatedIcConfiguration: AggregatedIcConfiguration<*>? = null
+        private set
+
     override fun useLogger(logger: KotlinLogger): JvmCompilationConfiguration {
         this.logger = logger
         return this

compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.internal
+
+import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
+import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration
+import org.jetbrains.kotlin.daemon.common.*
+import java.io.Serializable
+import java.rmi.server.UnicastRemoteObject
+
+internal val JvmCompilationConfigurationImpl.asDaemonCompilationOptions: CompilationOptions
+    get() {
+        val ktsExtensionsAsArray = if (kotlinScriptFilenameExtensions.isEmpty()) null else kotlinScriptFilenameExtensions.toTypedArray()
+        val reportCategories = arrayOf(ReportCategory.COMPILER_MESSAGE.code, ReportCategory.IC_MESSAGE.code) // TODO: automagically compute the value, related to BasicCompilerServicesWithResultsFacadeServer
+        val reportSeverity = ReportSeverity.INFO.code // TODO: automagically compute the value, related to BasicCompilerServicesWithResultsFacadeServer
+        val requestedCompilationResults = emptyArray<Int>() // TODO: automagically compute the value, related to DaemonCompilationResults
+        return when (aggregatedIcConfiguration?.options) {
+            is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration -> TODO("Incremental compilation within the daemon is not yet supported")
+            else -> CompilationOptions(
+                compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER,
+                targetPlatform = CompileService.TargetPlatform.JVM,
+                reportCategories = reportCategories,
+                reportSeverity = reportSeverity,
+                requestedCompilationResults = requestedCompilationResults,
+                kotlinScriptExtensions = ktsExtensionsAsArray,
+            )
+        }
+    }
+
+internal class DaemonCompilationResults : CompilationResults,
+    UnicastRemoteObject(
+        SOCKET_ANY_FREE_PORT,
+        LoopbackNetworkInterface.clientLoopbackSocketFactory,
+        LoopbackNetworkInterface.serverLoopbackSocketFactory
+    ) {
+    /**
+     * Possible combinations:
+     * 1. [CompilationResultCategory.IC_COMPILE_ITERATION.code]       -> a [CompileIterationResult] instance
+     * 2. [CompilationResultCategory.BUILD_REPORT_LINES.code]         -> a [List] of [String]
+     * 3. [CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES.code] -> a [List] of [String]
+     * 4. [CompilationResultCategory.BUILD_METRICS.code]              -> a [BuildMetrics] instance
+     **/
+    override fun add(compilationResultCategory: Int, value: Serializable) {
+        // TODO propagate the values to the caller via callbacks, requires to make metrics a part of the API
+        println("Result category=$compilationResultCategory value=$value")
+    }
+}
+
+internal val clientIsAliveFile by lazy {
+    makeAutodeletingFlagFile()
+}
+
+internal fun createSessionIsAliveFlagFile() = makeAutodeletingFlagFile(keyword = "compilation-session")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildToolsApiJvmCompilationIT.kt

@@ -45,7 +45,6 @@ class BuildToolsApiJvmCompilationIT : KGPBaseTest() {
     }
 
     @GradleTest
-    @Disabled
     @DisplayName("Simple project non-incremental compilation within the daemon")
     fun compileJvmNonIncrementalWithinDaemon(gradleVersion: GradleVersion) = testSimpleProject(
         gradleVersion, defaultBuildOptions.copy(

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt

@@ -75,7 +75,7 @@ internal abstract class BuildToolsApiCompilationWork : WorkAction<BuildToolsApiC
             }
             val executionConfig = compilationService.makeCompilerExecutionStrategyConfiguration().apply {
                 when (executionStrategy) {
-                    KotlinCompilerExecutionStrategy.DAEMON -> TODO("The daemon strategy is not yet supported in the Build Tools API")
+                    KotlinCompilerExecutionStrategy.DAEMON -> useDaemonStrategy(workArguments.compilerExecutionSettings.daemonJvmArgs ?: emptyList())
                     KotlinCompilerExecutionStrategy.IN_PROCESS -> useInProcessStrategy()
                     else -> error("The \"$executionStrategy\" execution strategy is not supported by the Build Tools API")
                 }

repo/artifacts-tests/src/test/resources/org/jetbrains/kotlin/kotlin-build-tools-impl/kotlin-build-tools-impl.pom

@@ -44,5 +44,11 @@
       <version>ArtifactsTest.version</version>
       <scope>runtime</scope>
     </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-compiler-runner</artifactId>
+      <version>ArtifactsTest.version</version>
+      <scope>runtime</scope>
+    </dependency>
   </dependencies>
 </project>