From c38dd1c004e799cb42aae14c9b542bcb16d5932a Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Fri, 11 Feb 2022 20:35:14 +0300 Subject: [PATCH] Fix unstable testIncrementalCompilationAfterCacheHit test --- .../storage/RelocatableCachesTest.kt | 4 +-- .../jetbrains/kotlin/cli/common/Properties.kt | 1 - .../daemon/common/CompilationOptions.kt | 9 ++--- .../kotlin/daemon/common/DaemonParams.kt | 2 -- .../kotlin/daemon/CompileServiceImpl.kt | 6 ++-- .../incremental/IncrementalCompilerRunner.kt | 26 +++++++-------- .../IncrementalJsCompilerRunner.kt | 10 +++--- .../IncrementalJvmCompilerRunner.kt | 18 +++++----- .../jetbrains/kotlin/gradle/BaseGradleIT.kt | 2 +- .../jetbrains/kotlin/gradle/BuildCacheIT.kt | 10 +++--- .../kotlin/gradle/BuildCacheRelocationIT.kt | 33 +++++++++---------- .../kotlin/gradle/testbase/BuildOptions.kt | 3 +- .../GradleKotlinCompilerWork.kt | 10 ++++-- .../IncrementalCompilationEnvironment.kt | 5 +-- .../kotlin/gradle/plugin/KotlinProperties.kt | 6 ++++ .../statistics/BuildScanStatisticsListener.kt | 5 +-- .../statistics/KotlinBuildStatListener.kt | 17 ++++++---- .../gradle/report/TaskExecutionResult.kt | 4 ++- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 6 +++- 19 files changed, 99 insertions(+), 78 deletions(-) diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt index bb52e8933ef..03e2029e72d 100644 --- a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt +++ b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt @@ -47,13 +47,13 @@ class RelocatableCachesTest : TestWithWorkingDir() { * Fills lookup storage in [projectRoot] with N fq-names, * where i_th fq-name myscope_i.MyClass_i has lookups for previous fq-names (from 0 to i-1) */ - private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean) { + private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean, storeFullFqNames: Boolean = false) { val storageRoot = projectRoot.storageRoot val fileToPathConverter = RelativeFileToPathConverter(projectRoot) val lookupStorage = LookupStorage( storageRoot, fileToPathConverter, - storeFullFqNames = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false + storeFullFqNames = storeFullFqNames ) val files = LinkedHashSet() val symbols = LinkedHashSet() diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt index 25d815bc659..c9968142efe 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt @@ -35,7 +35,6 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY("kotlin.daemon.socket.connect.interval"), KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY("kotlin.environment.keepalive"), COMPILE_DAEMON_CUSTOM_RUN_FILES_PATH_FOR_TESTS("kotlin.daemon.custom.run.files.path.for.tests"), - COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS("kotlin.incremental.classpath.snapshot.enabled"), COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM("kotlin.incremental.useClasspathSnapshot"), KOTLIN_COLORS_ENABLED_PROPERTY("kotlin.colors.enabled"), LANGUAGE_VERSION_SETTINGS("kotlin.language.settings"), diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt index 5ab9ed9c269..55cc4ed7874 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt @@ -65,12 +65,13 @@ class IncrementalCompilationOptions( requestedCompilationResults: Array, val usePreciseJavaTracking: Boolean, /** - * Directories that should be cleared when IC decides to rebuild - */ - val outputFiles: List, + * Directories that should be cleared when IC decides to rebuild + */ + val outputFiles: List, val multiModuleICSettings: MultiModuleICSettings, val modulesInfo: IncrementalModuleInfo, - kotlinScriptExtensions: Array? = null + kotlinScriptExtensions: Array? = null, + val withAbiSnapshot: Boolean = false ) : CompilationOptions( compilerMode, targetPlatform, diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt index 1058a44afc7..a2bf98aa99a 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt @@ -312,8 +312,6 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions, if (inheritAdditionalProperties) { CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.property}=\"$it\"") } CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.property}") } - //Temporary solution to test abi snapshot - CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.property}") } } if (opts.jvmParams.none { it.matches(jvmAssertArgsRegex) }) { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index fc481faee3e..aef6cb32237 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -559,7 +559,8 @@ abstract class CompileServiceImplBase( reporter = reporter, buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, scopeExpansion = if (args.isIrBackendEnabled()) CompileScopeExpansionMode.ALWAYS else CompileScopeExpansionMode.NEVER, - modulesApiHistory = modulesApiHistory + modulesApiHistory = modulesApiHistory, + withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot ) return try { compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles) @@ -617,7 +618,8 @@ abstract class CompileServiceImplBase( usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking, modulesApiHistory = modulesApiHistory, kotlinSourceFilesExtensions = allKotlinExtensions, - classpathChanges = incrementalCompilationOptions.classpathChanges + classpathChanges = incrementalCompilationOptions.classpathChanges, + withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot ) return try { compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, projectRoot) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index f729ad28d6d..61d1dd4c30f 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -49,18 +49,16 @@ abstract class IncrementalCompilerRunner< protected val buildHistoryFile: File, // there might be some additional output directories (e.g. for generated java in kapt) // to remove them correctly on rebuild, we pass them as additional argument - private val additionalOutputFiles: Collection = emptyList() + private val additionalOutputFiles: Collection = emptyList(), + protected val withAbiSnapshot: Boolean = false ) { protected val cacheDirectory = File(workingDir, cacheDirName) private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME) - protected val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) + private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) protected open val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS - //TODO(valtman) temporal measure to ensure quick disable, should be deleted after successful release - protected val withSnapshot: Boolean = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false - protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager protected abstract fun destinationDir(args: Args): File @@ -85,14 +83,14 @@ abstract class IncrementalCompilerRunner< ): ExitCode { var caches = createCacheManager(args, projectDir) - if (withSnapshot) { + if (withAbiSnapshot) { reporter.report { "Incremental compilation with ABI snapshot enabled" } } //TODO if abi-snapshot is corrupted unable to rebuild. Should roll back to withSnapshot = false? val classpathAbiSnapshot = - if (withSnapshot) { + if (withAbiSnapshot) { reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) { - setupJarDependencies(args, withSnapshot, reporter) + setupJarDependencies(args, withAbiSnapshot, reporter) } } else { emptyMap() @@ -110,8 +108,10 @@ abstract class IncrementalCompilerRunner< caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) } val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } - return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withSnapshot, - classpathAbiSnapshot = classpathAbiSnapshot) + return compileIncrementally( + args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withAbiSnapshot, + classpathAbiSnapshot = classpathAbiSnapshot + ) } // If compilation has crashed or we failed to close caches we have to clear them @@ -134,7 +134,7 @@ abstract class IncrementalCompilerRunner< val exitCode = when (compilationMode) { is CompilationMode.Incremental -> { - if (withSnapshot) { + if (withAbiSnapshot) { val abiSnapshot = AbiSnapshotImpl.read(abiSnapshotFile, reporter) if (abiSnapshot != null) { compileIncrementally( @@ -143,7 +143,7 @@ abstract class IncrementalCompilerRunner< allSourceFiles, compilationMode, messageCollector, - withSnapshot, + withAbiSnapshot, abiSnapshot, classpathAbiSnapshot ) @@ -157,7 +157,7 @@ abstract class IncrementalCompilerRunner< allSourceFiles, compilationMode, messageCollector, - withSnapshot + withAbiSnapshot ) } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index d511c317b45..0bc35a35e6d 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -84,12 +84,14 @@ class IncrementalJsCompilerRunner( reporter: BuildReporter, buildHistoryFile: File, private val modulesApiHistory: ModulesApiHistory, - private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER + private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER, + withAbiSnapshot: Boolean = false ) : IncrementalCompilerRunner( workingDir, "caches-js", reporter, - buildHistoryFile = buildHistoryFile + buildHistoryFile = buildHistoryFile, + withAbiSnapshot = withAbiSnapshot ) { override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager { @@ -99,7 +101,7 @@ class IncrementalJsCompilerRunner( projectDir, reporter, serializerProtocol, - storeFullFqNamesInLookupCache = withSnapshot + storeFullFqNamesInLookupCache = withAbiSnapshot ) } @@ -118,7 +120,7 @@ class IncrementalJsCompilerRunner( messageCollector: MessageCollector, classpathAbiSnapshots: Map //Ignore for now ): CompilationMode { - if (!withSnapshot && !buildHistoryFile.isFile) { + if (!withAbiSnapshot && !buildHistoryFile.isFile) { return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) } val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index f26020e13cd..92ed2d90c07 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -98,7 +98,6 @@ fun makeIncrementally( kotlinSourceFilesExtensions = kotlinExtensions, classpathChanges = ClasspathSnapshotDisabled ) - //TODO set properly compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null) } } @@ -135,13 +134,15 @@ class IncrementalJvmCompilerRunner( outputFiles: Collection, private val modulesApiHistory: ModulesApiHistory, override val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS, - private val classpathChanges: ClasspathChanges + private val classpathChanges: ClasspathChanges, + withAbiSnapshot: Boolean = false ) : IncrementalCompilerRunner( workingDir, "caches-jvm", reporter, additionalOutputFiles = outputFiles, - buildHistoryFile = buildHistoryFile + buildHistoryFile = buildHistoryFile, + withAbiSnapshot = withAbiSnapshot ) { override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager = IncrementalJvmCachesManager( @@ -149,7 +150,7 @@ class IncrementalJvmCompilerRunner( projectDir, File(args.destination), reporter, - storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, + storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun ) @@ -191,7 +192,7 @@ class IncrementalJvmCompilerRunner( classpathAbiSnapshots: Map ): CompilationMode { return try { - calculateSourcesToCompileImpl(caches, changedFiles, args, classpathAbiSnapshots) + calculateSourcesToCompileImpl(caches, changedFiles, args, classpathAbiSnapshots, withAbiSnapshot) } finally { psiFileProvider.messageCollector.flush(messageCollector) psiFileProvider.messageCollector.clear() @@ -228,7 +229,8 @@ class IncrementalJvmCompilerRunner( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments, - abiSnapshots: Map = HashMap() + abiSnapshots: Map = HashMap(), + withAbiSnapshot: Boolean ): CompilationMode { val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) initDirtyFiles(dirtyFiles, changedFiles) @@ -264,7 +266,7 @@ class IncrementalJvmCompilerRunner( is NotAvailableDueToMissingClasspathSnapshot -> ChangesEither.Unknown(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) is NotAvailableForNonIncrementalRun -> ChangesEither.Unknown(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { - if (!withSnapshot && !buildHistoryFile.isFile) { + if (!withAbiSnapshot && !buildHistoryFile.isFile) { // If the previous build was a Gradle cache hit, the build history file must have been deleted as it is marked as // @LocalState in the Gradle task. Therefore, this compilation will need to run non-incrementally. // (Note that buildHistoryFile is outside workingDir. We don't need to perform the same check for files inside @@ -276,7 +278,7 @@ class IncrementalJvmCompilerRunner( val scopes = caches.lookupCache.lookupSymbols.map { it.scope.ifBlank { it.name } }.distinct() getClasspathChanges( - args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot, + args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withAbiSnapshot, caches.platformCache, scopes ) } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt index 2b4ee65e308..b7e7c99d11f 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt @@ -911,7 +911,7 @@ Finished executing task ':$taskName'| add("--dry-run") } if (options.abiSnapshot) { - add("-Dkotlin.incremental.classpath.snapshot.enabled=true") + add("-Pkotlin.incremental.classpath.snapshot.enabled=true") } if (options.hierarchicalMPPStructureSupport != null) { diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheIT.kt index 40e0cae3beb..93dd1c7e6a4 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheIT.kt @@ -126,11 +126,10 @@ class BuildCacheIT : KGPBaseTest() { @DisplayName("Restore from build cache should not break incremental compilation") @GradleTest fun testIncrementalCompilationAfterCacheHit(gradleVersion: GradleVersion) { - val withSnapshotProperty = "-Dkotlin.incremental.classpath.snapshot.enabled=true" - project("incrementalMultiproject", gradleVersion) { + project("incrementalMultiproject", gradleVersion, buildOptions = defaultBuildOptions.copy(useICClasspathSnapshot = true)) { enableLocalBuildCache(localBuildCacheDir) - build("assemble", withSnapshotProperty) - build("clean", "assemble", withSnapshotProperty) { + build("assemble") + build("clean", "assemble") { assertTasksFromCache(":lib:compileKotlin") assertTasksFromCache(":app:compileKotlin") } @@ -138,8 +137,9 @@ class BuildCacheIT : KGPBaseTest() { bKtSourceFile.modify { it.replace("fun b() {}", "fun b() {}\nfun b2() {}") } - build("assemble", withSnapshotProperty, buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) { + build("assemble", buildOptions = defaultBuildOptions.copy(useICClasspathSnapshot = true, logLevel = LogLevel.DEBUG)) { assertOutputDoesNotContain("[KOTLIN] [IC] Non-incremental compilation will be performed") + assertOutputContains("Incremental compilation with ABI snapshot enabled") assertCompiledKotlinSources(setOf(bKtSourceFile).map { it.relativeTo(projectPath)}, output) } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt index 2f7c9d13339..10a4e518c91 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt @@ -18,6 +18,7 @@ package org.jetbrains.kotlin.gradle import org.gradle.api.logging.LogLevel import org.gradle.api.logging.configuration.WarningMode +import org.gradle.testkit.runner.BuildResult import org.gradle.util.GradleVersion import org.jetbrains.kotlin.gradle.testbase.* import org.junit.jupiter.api.DisplayName @@ -322,30 +323,33 @@ class BuildCacheRelocationIT : KGPBaseTest() { @DisplayName("Kotlin incremental compilation should work correctly") @GradleTest fun testKotlinIncrementalCompilation(gradleVersion: GradleVersion) { - checkKotlinIncrementalCompilation(gradleVersion) + checkKotlinIncrementalCompilationAfterCacheHit(gradleVersion) { + assertNonIncrementalCompilation() + } } @DisplayName("Kotlin incremental compilation with `kotlin.incremental.useClasspathSnapshot` feature should work correctly") @GradleTest fun testKotlinIncrementalCompilation_withGradleClasspathSnapshot(gradleVersion: GradleVersion) { - checkKotlinIncrementalCompilation(gradleVersion, useGradleClasspathSnapshot = true) + checkKotlinIncrementalCompilationAfterCacheHit(gradleVersion, defaultBuildOptions.copy(useGradleClasspathSnapshot = true)) { + assertIncrementalCompilation(listOf("src/main/kotlin/foo.kt", "src/main/kotlin/fooUsage.kt")) + } } @DisplayName("Kotlin incremental compilation with `kotlin.incremental.classpath.snapshot.enabled` feature should work correctly") @GradleTest fun testKotlinIncrementalCompilation_withICClasspathSnapshot(gradleVersion: GradleVersion) { - checkKotlinIncrementalCompilation(gradleVersion, useICClasspathSnapshot = true) + checkKotlinIncrementalCompilationAfterCacheHit(gradleVersion, defaultBuildOptions.copy(useICClasspathSnapshot = true)) { + assertIncrementalCompilation(listOf("src/main/kotlin/foo.kt", "src/main/kotlin/fooUsage.kt")) + assertOutputContains("Incremental compilation with ABI snapshot enabled") + } } - private fun checkKotlinIncrementalCompilation( + private fun checkKotlinIncrementalCompilationAfterCacheHit( gradleVersion: GradleVersion, - useGradleClasspathSnapshot: Boolean? = null, - useICClasspathSnapshot: Boolean? = null + buildOptions: BuildOptions = defaultBuildOptions, + assertions: BuildResult.() -> Unit ) { - val buildOptions = defaultBuildOptions.copy( - useGradleClasspathSnapshot = useGradleClasspathSnapshot, - useICClasspathSnapshot = useICClasspathSnapshot - ) val (firstProject, secondProject) = prepareTestProjects("buildCacheSimple", gradleVersion, buildOptions) // First build, should be stored into the build cache: @@ -361,14 +365,7 @@ class BuildCacheRelocationIT : KGPBaseTest() { // Check whether compilation after a cache hit is incremental (KT-34862) val fooKtSourceFile = secondProject.kotlinSourcesDir().resolve("foo.kt") fooKtSourceFile.modify { it.replace("Int = 1", "String = \"abc\"") } - secondProject.build("assemble", buildOptions = buildOptions.copy(logLevel = LogLevel.DEBUG)) { - if (useGradleClasspathSnapshot == true || useICClasspathSnapshot == true) { - assertIncrementalCompilation(listOf("src/main/kotlin/foo.kt", "src/main/kotlin/fooUsage.kt")) - } else { - assertNonIncrementalCompilation() - } - } - + secondProject.build("assemble", buildOptions = buildOptions.copy(logLevel = LogLevel.DEBUG), assertions = assertions) // Revert the change to the return type of foo(), and check if we get a cache hit fooKtSourceFile.modify { it.replace("String = \"abc\"", "Int = 1") } secondProject.build("clean", "assemble") { diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt index 0e7e11ac44a..2bda8705159 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt @@ -9,7 +9,6 @@ import org.gradle.api.logging.LogLevel import org.gradle.api.logging.configuration.WarningMode import org.gradle.util.GradleVersion import org.jetbrains.kotlin.cli.common.CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS import org.jetbrains.kotlin.gradle.BaseGradleIT import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType import org.jetbrains.kotlin.gradle.report.BuildReportType @@ -88,7 +87,7 @@ data class BuildOptions( } useGradleClasspathSnapshot?.let { arguments.add("-P${COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.property}=$it") } - useICClasspathSnapshot?.let { arguments.add("-D${COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.property}=$it") } + useICClasspathSnapshot?.let { arguments.add("-Pkotlin.incremental.classpath.snapshot.enabled=$it") } if (gradleVersion >= GradleVersion.version("6.5")) { if (fileSystemWatchEnabled) { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 04d2cd3cca7..3a2bdc4231e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -7,8 +7,10 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.logging.Logger import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.daemon.common.* import org.jetbrains.kotlin.gradle.logging.* @@ -20,6 +22,7 @@ import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError import org.jetbrains.kotlin.gradle.utils.stackTraceAsString import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.incremental.ClasspathChanges import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.slf4j.LoggerFactory import java.io.* @@ -129,7 +132,9 @@ internal class GradleKotlinCompilerWork @Inject constructor( } finally { val taskInfo = TaskExecutionInfo( changedFiles = incrementalCompilationEnvironment?.changedFiles, - compilerArguments = compilerArgs + compilerArguments = compilerArgs, + withAbiSnapshot = incrementalCompilationEnvironment?.withAbiSnapshot, + withArtifactTransform = incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled ) val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines, taskInfo = taskInfo) TaskExecutionResults[taskPath] = result @@ -282,7 +287,8 @@ internal class GradleKotlinCompilerWork @Inject constructor( outputFiles = outputFiles, multiModuleICSettings = icEnv.multiModuleICSettings, modulesInfo = incrementalModuleInfo!!, - kotlinScriptExtensions = kotlinScriptExtensions + kotlinScriptExtensions = kotlinScriptExtensions, + withAbiSnapshot = icEnv.withAbiSnapshot ) log.info("Options for KOTLIN DAEMON: $compilationOptions") diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt index ec09810466c..08335e1fc6d 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt @@ -17,9 +17,10 @@ internal class IncrementalCompilationEnvironment( val workingDir: File, val usePreciseJavaTracking: Boolean = false, val disableMultiModuleIC: Boolean = false, - val multiModuleICSettings: MultiModuleICSettings + val multiModuleICSettings: MultiModuleICSettings, + val withAbiSnapshot: Boolean = false ) : Serializable { companion object { - const val serialVersionUID: Long = 0 + const val serialVersionUID: Long = 1 } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt index 3b93c62809d..630d558fb56 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt @@ -13,6 +13,7 @@ import org.jetbrains.kotlin.gradle.dsl.NativeCacheKind import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessageOutputStreamHandler.Companion.IGNORE_TCSM_OVERFLOW import org.jetbrains.kotlin.gradle.plugin.Kotlin2JsPlugin.Companion.NOWARN_2JS_FLAG import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.Companion.jsCompilerProperty +import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_ABI_SNAPSHOT import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_CINTEROP_COMMONIZATION import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_GRANULAR_SOURCE_SETS_METADATA import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_OPTIMISTIC_NUMBER_COMMONIZATION @@ -49,6 +50,7 @@ internal fun PropertiesProvider.mapKotlinTaskProperties(task: AbstractKotlinComp } } task.jvmTargetValidationMode.set(jvmTargetValidationMode) + task.useKotlinAbiSnapshot.value(useKotlinAbiSnapshot).disallowChanges() } if (task is Kotlin2JsCompile) { @@ -174,6 +176,9 @@ internal class PropertiesProvider private constructor(private val project: Proje return gradleProperty || systemProperty } + val useKotlinAbiSnapshot: Boolean + get() = booleanProperty(KOTLIN_ABI_SNAPSHOT) ?: false + val useFir: Boolean? get() = booleanProperty("kotlin.useFir") @@ -503,6 +508,7 @@ internal class PropertiesProvider private constructor(private val project: Proje const val KOTLIN_MPP_ENABLE_OPTIMISTIC_NUMBER_COMMONIZATION = "kotlin.mpp.enableOptimisticNumberCommonization" const val KOTLIN_KPM_EXPERIMENTAL_MODEL_MAPPING = "kotlin.kpm.experimentalModelMapping" const val KOTLIN_MPP_ENABLE_PLATFORM_INTEGER_COMMONIZATION = "kotlin.mpp.enablePlatformIntegerCommonization" + const val KOTLIN_ABI_SNAPSHOT = "kotlin.incremental.classpath.snapshot.enabled" } companion object { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt index e59f42329fb..922b38d200e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildScanStatisticsListener.kt @@ -40,10 +40,7 @@ class BuildScanStatisticsListener( log.debug("Collect data takes $collectDataDuration: $compileStatData") compileStatData?.also { - val reportDataDuration = measureTimeMillis { - report(it) - } - log.debug("Report data takes $reportDataDuration: $compileStatData") + report(it) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt index ca0cb2b7e86..c6104b5bcc3 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatListener.kt @@ -16,6 +16,7 @@ import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData import org.jetbrains.kotlin.gradle.plugin.stat.StatTag import org.jetbrains.kotlin.gradle.report.TaskExecutionResult import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.utils.addToStdlib.ifTrue import java.lang.management.ManagementFactory import java.net.InetAddress import java.util.* @@ -100,12 +101,7 @@ class KotlinBuildStatListener { private fun parseTags(taskExecutionResult: TaskExecutionResult?): List { val tags = ArrayList() - CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient()?.let { - if (it) tags.add(StatTag.ABI_SNAPSHOT) - } - CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.value.toBooleanLenient()?.let { - if (it) tags.add(StatTag.ARTIFACT_TRANSFORM) - } + val nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap() if (nonIncrementalAttributes.isEmpty()) { @@ -114,6 +110,15 @@ class KotlinBuildStatListener { tags.add(StatTag.NON_INCREMENTAL) } + val taskInfo = taskExecutionResult?.taskInfo + + taskInfo?.withAbiSnapshot?.ifTrue { + tags.add(StatTag.ABI_SNAPSHOT) + } + taskInfo?.withArtifactTransform?.ifTrue { + tags.add(StatTag.ARTIFACT_TRANSFORM) + } + val debugConfiguration = "-agentlib:" if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) { tags.add(StatTag.GRADLE_DEBUG) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt index 0e6651d2b94..b2363d3d8af 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/TaskExecutionResult.kt @@ -16,5 +16,7 @@ internal class TaskExecutionResult( internal class TaskExecutionInfo( val changedFiles: ChangedFiles? = null, - val compilerArguments: Array = emptyArray() + val compilerArguments: Array = emptyArray(), + val withArtifactTransform: Boolean? = false, + val withAbiSnapshot: Boolean? = false ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index 0bfdeddfad2..43c5bfbfdb4 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -651,6 +651,9 @@ abstract class KotlinCompile @Inject constructor( return super.getClasspath() } + @get:Input + abstract val useKotlinAbiSnapshot: Property + @get:Nested abstract val classpathSnapshotProperties: ClasspathSnapshotProperties @@ -770,7 +773,8 @@ abstract class KotlinCompile @Inject constructor( workingDir = taskBuildCacheableOutputDirectory.get().asFile, usePreciseJavaTracking = usePreciseJavaTracking, disableMultiModuleIC = disableMultiModuleIC, - multiModuleICSettings = multiModuleICSettings + multiModuleICSettings = multiModuleICSettings, + withAbiSnapshot = useKotlinAbiSnapshot.get() ) } else null