From 61f404e493340f5ec8b8211d3c4ac5da9514226e Mon Sep 17 00:00:00 2001 From: "Alexander.Likhachev" Date: Thu, 13 Jul 2023 14:34:11 +0200 Subject: [PATCH] [BT] Implement incremental compilation within the daemon #KT-57398 In Progress --- .../api/kotlin-build-tools-api.api | 3 ++ .../api/jvm/JvmCompilationConfiguration.kt | 17 +++++++ .../internal/CompilationServiceImpl.kt | 3 +- .../JvmCompilationConfigurationImpl.kt | 15 +++++- .../buildtools/internal/daemonAdapters.kt | 48 ++++++++++++++++++- .../daemon/common/CompilationOptions.kt | 4 +- .../gradle/BuildToolsApiJvmCompilationIT.kt | 1 - .../btapi/BuildToolsApiCompilationWork.kt | 39 ++++++++++++++- 8 files changed, 121 insertions(+), 9 deletions(-) diff --git a/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api b/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api index 45ac672388d..1f884c2fa93 100644 --- a/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api +++ b/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api @@ -102,6 +102,7 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Classpat public abstract fun forceNonIncrementalMode (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun getAssuredNoClasspathSnapshotsChanges ()Z public abstract fun keepIncrementalCompilationCachesInMemory (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; + public abstract fun useOutputDirs (Ljava/util/Collection;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun usePreciseCompilationResultsBackup (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun usePreciseJavaTracking (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; @@ -115,10 +116,12 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Incremen public static synthetic fun forceNonIncrementalMode$default (Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;ZILjava/lang/Object;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun getForcedNonIncrementalMode ()Z public abstract fun getIncrementalCompilationCachesKeptInMemory ()Z + public abstract fun getOutputDirs ()Ljava/util/Set; public abstract fun getPreciseCompilationResultsBackupEnabled ()Z public abstract fun getPreciseJavaTrackingEnabled ()Z public abstract fun getProjectDir ()Ljava/io/File; public abstract fun keepIncrementalCompilationCachesInMemory (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; + public abstract fun useOutputDirs (Ljava/util/Collection;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun usePreciseCompilationResultsBackup (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun usePreciseJavaTracking (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; diff --git a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt index 903c29f5082..3a490ee3754 100644 --- a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt +++ b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt @@ -147,6 +147,21 @@ public interface IncrementalJvmCompilationConfiguration

+ + /** + * The directories that the compiler will clean in the case of fallback to non-incremental compilation. + * + * The default ones are calculated in the case of a `null` value as a set of the incremental compilation working directory + * passed to [JvmCompilationConfiguration.useIncrementalCompilation] and the classes output directory from the compiler arguments. + * + * If the value is set explicitly, it must contain the above-mentioned default directories. + */ + public val outputDirs: Set? + + /** + * @see [IncrementalJvmCompilationConfiguration.outputDirs]] + */ + public fun useOutputDirs(outputDirs: Collection): IncrementalJvmCompilationConfiguration

} /** @@ -180,6 +195,8 @@ public interface ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration : override fun keepIncrementalCompilationCachesInMemory(value: Boolean): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration override fun forceNonIncrementalMode(value: Boolean): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration + + override fun useOutputDirs(outputDirs: Collection): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration } /** diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt index 049e79802a2..690fb2d871e 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt @@ -58,8 +58,7 @@ internal object CompilationServiceImpl : CompilationService { "Initial JVM compilation configuration object must be acquired from the `makeJvmCompilationConfiguration` method." } val loggerAdapter = KotlinLoggerMessageCollectorAdapter(compilationConfig.logger) - val selectedStrategy = strategyConfig.selectedStrategy - return when (selectedStrategy) { + return when (val selectedStrategy = strategyConfig.selectedStrategy) { is CompilerExecutionStrategy.InProcess -> compileInProcess(loggerAdapter, sources, arguments) is CompilerExecutionStrategy.Daemon -> compileWithinDaemon( projectId, diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt index a0010c54847..204ea811524 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt @@ -41,7 +41,9 @@ internal class JvmCompilationConfigurationImpl( sourcesChanges: SourcesChanges, approachParameters: P, options: IncrementalJvmCompilationConfiguration

, - ) = TODO("Incremental compilation is not yet supported to run via the Build Tools API") + ) { + aggregatedIcConfiguration = AggregatedIcConfiguration(options, approachParameters, sourcesChanges, workingDirectory) + } } internal abstract class JvmIncrementalCompilationConfigurationImpl

( @@ -50,6 +52,7 @@ internal abstract class JvmIncrementalCompilationConfigurationImpl

= emptySet(), ) : IncrementalJvmCompilationConfiguration

{ override fun useProjectDir(projectDir: File): IncrementalJvmCompilationConfiguration

{ this.projectDir = projectDir @@ -75,6 +78,11 @@ internal abstract class JvmIncrementalCompilationConfigurationImpl

): IncrementalJvmCompilationConfiguration

{ + this.outputDirs = outputDirs.toSet() + return this + } } internal class ClasspathSnapshotBasedIncrementalJvmCompilationConfigurationImpl( @@ -111,4 +119,9 @@ internal class ClasspathSnapshotBasedIncrementalJvmCompilationConfigurationImpl( assuredNoClasspathSnapshotsChanges = value return this } + + override fun useOutputDirs(outputDirs: Collection): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration { + super.useOutputDirs(outputDirs) + return this + } } \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt index 722525efe3d..1f554cee089 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt @@ -6,8 +6,12 @@ package org.jetbrains.kotlin.buildtools.internal import org.jetbrains.kotlin.build.report.metrics.BuildMetrics +import org.jetbrains.kotlin.buildtools.api.SourcesChanges +import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.incremental.ClasspathChanges +import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles import java.io.Serializable import java.rmi.server.UnicastRemoteObject @@ -17,8 +21,48 @@ internal val JvmCompilationConfigurationImpl.asDaemonCompilationOptions: Compila val reportCategories = arrayOf(ReportCategory.COMPILER_MESSAGE.code, ReportCategory.IC_MESSAGE.code) // TODO: automagically compute the value, related to BasicCompilerServicesWithResultsFacadeServer val reportSeverity = ReportSeverity.INFO.code // TODO: automagically compute the value, related to BasicCompilerServicesWithResultsFacadeServer val requestedCompilationResults = emptyArray() // TODO: automagically compute the value, related to DaemonCompilationResults - return when (aggregatedIcConfiguration?.options) { - is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration -> TODO("Incremental compilation within the daemon is not yet supported") + val aggregatedIcConfiguration = aggregatedIcConfiguration + return when (val options = aggregatedIcConfiguration?.options) { + is ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration -> { + val sourcesChanges = aggregatedIcConfiguration.sourcesChanges + val params = aggregatedIcConfiguration.parameters as ClasspathSnapshotBasedIncrementalCompilationApproachParameters + val snapshotFiles = + ClasspathSnapshotFiles(params.newClasspathSnapshotFiles, params.shrunkClasspathSnapshot.parentFile) + IncrementalCompilationOptions( + areFileChangesKnown = sourcesChanges is SourcesChanges.Known, + modifiedFiles = if (sourcesChanges is SourcesChanges.Known) sourcesChanges.modifiedFiles else null, + deletedFiles = if (sourcesChanges is SourcesChanges.Known) sourcesChanges.removedFiles else null, + classpathChanges = when { + !params.shrunkClasspathSnapshot.exists() -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot( + snapshotFiles + ) + options.forcedNonIncrementalMode -> ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun( + snapshotFiles + ) + options.assuredNoClasspathSnapshotsChanges -> ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges( + snapshotFiles + ) + else -> { + ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler(snapshotFiles) + } + }, + workingDir = aggregatedIcConfiguration.workingDir, + compilerMode = CompilerMode.INCREMENTAL_COMPILER, + targetPlatform = CompileService.TargetPlatform.JVM, + reportCategories = reportCategories, + reportSeverity = reportSeverity, + requestedCompilationResults = requestedCompilationResults, + usePreciseJavaTracking = options.preciseJavaTrackingEnabled, + outputFiles = options.outputDirs, + multiModuleICSettings = null, // required only for the build history approach + modulesInfo = null, // required only for the build history approach + rootProjectDir = options.projectDir, + kotlinScriptExtensions = ktsExtensionsAsArray, + withAbiSnapshot = false, + preciseCompilationResultsBackup = options.preciseCompilationResultsBackupEnabled, + keepIncrementalCompilationCachesInMemory = options.incrementalCompilationCachesKeptInMemory, + ) + } else -> CompilationOptions( compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER, targetPlatform = CompileService.TargetPlatform.JVM, diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt index e50fb1703a1..95e1e7cdfbd 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt @@ -67,7 +67,7 @@ class IncrementalCompilationOptions( /** * Directories that should be cleared when IC decides to rebuild */ - val outputFiles: List, + val outputFiles: Collection? = null, val multiModuleICSettings: MultiModuleICSettings? = null, val modulesInfo: IncrementalModuleInfo? = null, /** @@ -87,7 +87,7 @@ class IncrementalCompilationOptions( kotlinScriptExtensions ) { companion object { - const val serialVersionUID: Long = 2 + const val serialVersionUID: Long = 3 } override fun toString(): String { diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildToolsApiJvmCompilationIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildToolsApiJvmCompilationIT.kt index 3104e5eecef..31a4a9f8f1b 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildToolsApiJvmCompilationIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildToolsApiJvmCompilationIT.kt @@ -57,7 +57,6 @@ class BuildToolsApiJvmCompilationIT : KGPBaseTest() { } @GradleTest - @Disabled @DisplayName("Simple project incremental compilation within the daemon") fun compileJvmIncrementalWithinDaemon(gradleVersion: GradleVersion) = testSimpleProject( gradleVersion, defaultBuildOptions.copy( diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt index 28df1039390..0d976081d03 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime import org.jetbrains.kotlin.buildtools.api.* +import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.compilerRunner.GradleKotlinCompilerWorkArguments import org.jetbrains.kotlin.compilerRunner.asFinishLogMessage @@ -26,9 +27,17 @@ import org.jetbrains.kotlin.gradle.plugin.internal.BuildIdService import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy import org.jetbrains.kotlin.gradle.tasks.throwExceptionIfCompilationFailed +import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.incremental.ClasspathChanges import org.slf4j.LoggerFactory import java.io.File -import java.util.* + +@OptIn(ExperimentalBuildToolsApi::class) +private val ChangedFiles.asSourcesChanges: SourcesChanges + get() = when (this) { + is ChangedFiles.Known -> SourcesChanges.Known(modified, removed) + is ChangedFiles.Unknown -> SourcesChanges.Unknown + } @OptIn(ExperimentalBuildToolsApi::class) internal abstract class BuildToolsApiCompilationWork : WorkAction { @@ -81,6 +90,34 @@ internal abstract class BuildToolsApiCompilationWork : WorkAction classpathSnapshotsConfig.assureNoClasspathSnapshotsChanges() + is ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun -> classpathSnapshotsConfig.forceNonIncrementalMode() + else -> {} + } + jvmCompilationConfig.useIncrementalCompilation( + icEnv.workingDir, + icEnv.changedFiles.asSourcesChanges, + classpathSnapshotsParameters, + classpathSnapshotsConfig, + ) + } val result = compilationService.compileJvm( buildId, executionConfig,