Fix unstable testIncrementalCompilationAfterCacheHit test

This commit is contained in:
nataliya.valtman
2022-02-11 20:35:14 +03:00
parent 0463b5ec3c
commit c38dd1c004
19 changed files with 99 additions and 78 deletions
@@ -47,13 +47,13 @@ class RelocatableCachesTest : TestWithWorkingDir() {
* Fills lookup storage in [projectRoot] with N fq-names, * Fills lookup storage in [projectRoot] with N fq-names,
* where i_th fq-name myscope_i.MyClass_i has lookups for previous fq-names (from 0 to i-1) * where i_th fq-name myscope_i.MyClass_i has lookups for previous fq-names (from 0 to i-1)
*/ */
private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean) { private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean, storeFullFqNames: Boolean = false) {
val storageRoot = projectRoot.storageRoot val storageRoot = projectRoot.storageRoot
val fileToPathConverter = RelativeFileToPathConverter(projectRoot) val fileToPathConverter = RelativeFileToPathConverter(projectRoot)
val lookupStorage = LookupStorage( val lookupStorage = LookupStorage(
storageRoot, storageRoot,
fileToPathConverter, fileToPathConverter,
storeFullFqNames = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false storeFullFqNames = storeFullFqNames
) )
val files = LinkedHashSet<String>() val files = LinkedHashSet<String>()
val symbols = LinkedHashSet<LookupSymbol>() val symbols = LinkedHashSet<LookupSymbol>()
@@ -35,7 +35,6 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess
DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY("kotlin.daemon.socket.connect.interval"), DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY("kotlin.daemon.socket.connect.interval"),
KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY("kotlin.environment.keepalive"), KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY("kotlin.environment.keepalive"),
COMPILE_DAEMON_CUSTOM_RUN_FILES_PATH_FOR_TESTS("kotlin.daemon.custom.run.files.path.for.tests"), COMPILE_DAEMON_CUSTOM_RUN_FILES_PATH_FOR_TESTS("kotlin.daemon.custom.run.files.path.for.tests"),
COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS("kotlin.incremental.classpath.snapshot.enabled"),
COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM("kotlin.incremental.useClasspathSnapshot"), COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM("kotlin.incremental.useClasspathSnapshot"),
KOTLIN_COLORS_ENABLED_PROPERTY("kotlin.colors.enabled"), KOTLIN_COLORS_ENABLED_PROPERTY("kotlin.colors.enabled"),
LANGUAGE_VERSION_SETTINGS("kotlin.language.settings"), LANGUAGE_VERSION_SETTINGS("kotlin.language.settings"),
@@ -65,12 +65,13 @@ class IncrementalCompilationOptions(
requestedCompilationResults: Array<Int>, requestedCompilationResults: Array<Int>,
val usePreciseJavaTracking: Boolean, val usePreciseJavaTracking: Boolean,
/** /**
* Directories that should be cleared when IC decides to rebuild * Directories that should be cleared when IC decides to rebuild
*/ */
val outputFiles: List<File>, val outputFiles: List<File>,
val multiModuleICSettings: MultiModuleICSettings, val multiModuleICSettings: MultiModuleICSettings,
val modulesInfo: IncrementalModuleInfo, val modulesInfo: IncrementalModuleInfo,
kotlinScriptExtensions: Array<String>? = null kotlinScriptExtensions: Array<String>? = null,
val withAbiSnapshot: Boolean = false
) : CompilationOptions( ) : CompilationOptions(
compilerMode, compilerMode,
targetPlatform, targetPlatform,
@@ -312,8 +312,6 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
if (inheritAdditionalProperties) { if (inheritAdditionalProperties) {
CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.property}=\"$it\"") } CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.property}=\"$it\"") }
CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.property}") } CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.property}") }
//Temporary solution to test abi snapshot
CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.property}") }
} }
if (opts.jvmParams.none { it.matches(jvmAssertArgsRegex) }) { if (opts.jvmParams.none { it.matches(jvmAssertArgsRegex) }) {
@@ -559,7 +559,8 @@ abstract class CompileServiceImplBase(
reporter = reporter, reporter = reporter,
buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile,
scopeExpansion = if (args.isIrBackendEnabled()) CompileScopeExpansionMode.ALWAYS else CompileScopeExpansionMode.NEVER, scopeExpansion = if (args.isIrBackendEnabled()) CompileScopeExpansionMode.ALWAYS else CompileScopeExpansionMode.NEVER,
modulesApiHistory = modulesApiHistory modulesApiHistory = modulesApiHistory,
withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot
) )
return try { return try {
compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles) compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles)
@@ -617,7 +618,8 @@ abstract class CompileServiceImplBase(
usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking, usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
modulesApiHistory = modulesApiHistory, modulesApiHistory = modulesApiHistory,
kotlinSourceFilesExtensions = allKotlinExtensions, kotlinSourceFilesExtensions = allKotlinExtensions,
classpathChanges = incrementalCompilationOptions.classpathChanges classpathChanges = incrementalCompilationOptions.classpathChanges,
withAbiSnapshot = incrementalCompilationOptions.withAbiSnapshot
) )
return try { return try {
compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, projectRoot) compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, projectRoot)
@@ -49,18 +49,16 @@ abstract class IncrementalCompilerRunner<
protected val buildHistoryFile: File, protected val buildHistoryFile: File,
// there might be some additional output directories (e.g. for generated java in kapt) // there might be some additional output directories (e.g. for generated java in kapt)
// to remove them correctly on rebuild, we pass them as additional argument // to remove them correctly on rebuild, we pass them as additional argument
private val additionalOutputFiles: Collection<File> = emptyList() private val additionalOutputFiles: Collection<File> = emptyList(),
protected val withAbiSnapshot: Boolean = false
) { ) {
protected val cacheDirectory = File(workingDir, cacheDirName) protected val cacheDirectory = File(workingDir, cacheDirName)
private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME) protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
protected val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
//TODO(valtman) temporal measure to ensure quick disable, should be deleted after successful release
protected val withSnapshot: Boolean = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false
protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager
protected abstract fun destinationDir(args: Args): File protected abstract fun destinationDir(args: Args): File
@@ -85,14 +83,14 @@ abstract class IncrementalCompilerRunner<
): ExitCode { ): ExitCode {
var caches = createCacheManager(args, projectDir) var caches = createCacheManager(args, projectDir)
if (withSnapshot) { if (withAbiSnapshot) {
reporter.report { "Incremental compilation with ABI snapshot enabled" } reporter.report { "Incremental compilation with ABI snapshot enabled" }
} }
//TODO if abi-snapshot is corrupted unable to rebuild. Should roll back to withSnapshot = false? //TODO if abi-snapshot is corrupted unable to rebuild. Should roll back to withSnapshot = false?
val classpathAbiSnapshot = val classpathAbiSnapshot =
if (withSnapshot) { if (withAbiSnapshot) {
reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) { reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) {
setupJarDependencies(args, withSnapshot, reporter) setupJarDependencies(args, withAbiSnapshot, reporter)
} }
} else { } else {
emptyMap() emptyMap()
@@ -110,8 +108,10 @@ abstract class IncrementalCompilerRunner<
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
} }
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withSnapshot, return compileIncrementally(
classpathAbiSnapshot = classpathAbiSnapshot) args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withAbiSnapshot,
classpathAbiSnapshot = classpathAbiSnapshot
)
} }
// If compilation has crashed or we failed to close caches we have to clear them // If compilation has crashed or we failed to close caches we have to clear them
@@ -134,7 +134,7 @@ abstract class IncrementalCompilerRunner<
val exitCode = when (compilationMode) { val exitCode = when (compilationMode) {
is CompilationMode.Incremental -> { is CompilationMode.Incremental -> {
if (withSnapshot) { if (withAbiSnapshot) {
val abiSnapshot = AbiSnapshotImpl.read(abiSnapshotFile, reporter) val abiSnapshot = AbiSnapshotImpl.read(abiSnapshotFile, reporter)
if (abiSnapshot != null) { if (abiSnapshot != null) {
compileIncrementally( compileIncrementally(
@@ -143,7 +143,7 @@ abstract class IncrementalCompilerRunner<
allSourceFiles, allSourceFiles,
compilationMode, compilationMode,
messageCollector, messageCollector,
withSnapshot, withAbiSnapshot,
abiSnapshot, abiSnapshot,
classpathAbiSnapshot classpathAbiSnapshot
) )
@@ -157,7 +157,7 @@ abstract class IncrementalCompilerRunner<
allSourceFiles, allSourceFiles,
compilationMode, compilationMode,
messageCollector, messageCollector,
withSnapshot withAbiSnapshot
) )
} }
} }
@@ -84,12 +84,14 @@ class IncrementalJsCompilerRunner(
reporter: BuildReporter, reporter: BuildReporter,
buildHistoryFile: File, buildHistoryFile: File,
private val modulesApiHistory: ModulesApiHistory, private val modulesApiHistory: ModulesApiHistory,
private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER private val scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER,
withAbiSnapshot: Boolean = false
) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>( ) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>(
workingDir, workingDir,
"caches-js", "caches-js",
reporter, reporter,
buildHistoryFile = buildHistoryFile buildHistoryFile = buildHistoryFile,
withAbiSnapshot = withAbiSnapshot
) { ) {
override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager { override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
@@ -99,7 +101,7 @@ class IncrementalJsCompilerRunner(
projectDir, projectDir,
reporter, reporter,
serializerProtocol, serializerProtocol,
storeFullFqNamesInLookupCache = withSnapshot storeFullFqNamesInLookupCache = withAbiSnapshot
) )
} }
@@ -118,7 +120,7 @@ class IncrementalJsCompilerRunner(
messageCollector: MessageCollector, messageCollector: MessageCollector,
classpathAbiSnapshots: Map<String, AbiSnapshot> //Ignore for now classpathAbiSnapshots: Map<String, AbiSnapshot> //Ignore for now
): CompilationMode { ): CompilationMode {
if (!withSnapshot && !buildHistoryFile.isFile) { if (!withAbiSnapshot && !buildHistoryFile.isFile) {
return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
} }
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
@@ -98,7 +98,6 @@ fun makeIncrementally(
kotlinSourceFilesExtensions = kotlinExtensions, kotlinSourceFilesExtensions = kotlinExtensions,
classpathChanges = ClasspathSnapshotDisabled classpathChanges = ClasspathSnapshotDisabled
) )
//TODO set properly
compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null) compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null)
} }
} }
@@ -135,13 +134,15 @@ class IncrementalJvmCompilerRunner(
outputFiles: Collection<File>, outputFiles: Collection<File>,
private val modulesApiHistory: ModulesApiHistory, private val modulesApiHistory: ModulesApiHistory,
override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS, override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
private val classpathChanges: ClasspathChanges private val classpathChanges: ClasspathChanges,
withAbiSnapshot: Boolean = false
) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>( ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
workingDir, workingDir,
"caches-jvm", "caches-jvm",
reporter, reporter,
additionalOutputFiles = outputFiles, additionalOutputFiles = outputFiles,
buildHistoryFile = buildHistoryFile buildHistoryFile = buildHistoryFile,
withAbiSnapshot = withAbiSnapshot
) { ) {
override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager = override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
IncrementalJvmCachesManager( IncrementalJvmCachesManager(
@@ -149,7 +150,7 @@ class IncrementalJvmCompilerRunner(
projectDir, projectDir,
File(args.destination), File(args.destination),
reporter, reporter,
storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
) )
@@ -191,7 +192,7 @@ class IncrementalJvmCompilerRunner(
classpathAbiSnapshots: Map<String, AbiSnapshot> classpathAbiSnapshots: Map<String, AbiSnapshot>
): CompilationMode { ): CompilationMode {
return try { return try {
calculateSourcesToCompileImpl(caches, changedFiles, args, classpathAbiSnapshots) calculateSourcesToCompileImpl(caches, changedFiles, args, classpathAbiSnapshots, withAbiSnapshot)
} finally { } finally {
psiFileProvider.messageCollector.flush(messageCollector) psiFileProvider.messageCollector.flush(messageCollector)
psiFileProvider.messageCollector.clear() psiFileProvider.messageCollector.clear()
@@ -228,7 +229,8 @@ class IncrementalJvmCompilerRunner(
caches: IncrementalJvmCachesManager, caches: IncrementalJvmCachesManager,
changedFiles: ChangedFiles.Known, changedFiles: ChangedFiles.Known,
args: K2JVMCompilerArguments, args: K2JVMCompilerArguments,
abiSnapshots: Map<String, AbiSnapshot> = HashMap() abiSnapshots: Map<String, AbiSnapshot> = HashMap(),
withAbiSnapshot: Boolean
): CompilationMode { ): CompilationMode {
val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
initDirtyFiles(dirtyFiles, changedFiles) initDirtyFiles(dirtyFiles, changedFiles)
@@ -264,7 +266,7 @@ class IncrementalJvmCompilerRunner(
is NotAvailableDueToMissingClasspathSnapshot -> ChangesEither.Unknown(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) is NotAvailableDueToMissingClasspathSnapshot -> ChangesEither.Unknown(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND)
is NotAvailableForNonIncrementalRun -> ChangesEither.Unknown(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) is NotAvailableForNonIncrementalRun -> ChangesEither.Unknown(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
if (!withSnapshot && !buildHistoryFile.isFile) { if (!withAbiSnapshot && !buildHistoryFile.isFile) {
// If the previous build was a Gradle cache hit, the build history file must have been deleted as it is marked as // If the previous build was a Gradle cache hit, the build history file must have been deleted as it is marked as
// @LocalState in the Gradle task. Therefore, this compilation will need to run non-incrementally. // @LocalState in the Gradle task. Therefore, this compilation will need to run non-incrementally.
// (Note that buildHistoryFile is outside workingDir. We don't need to perform the same check for files inside // (Note that buildHistoryFile is outside workingDir. We don't need to perform the same check for files inside
@@ -276,7 +278,7 @@ class IncrementalJvmCompilerRunner(
val scopes = caches.lookupCache.lookupSymbols.map { it.scope.ifBlank { it.name } }.distinct() val scopes = caches.lookupCache.lookupSymbols.map { it.scope.ifBlank { it.name } }.distinct()
getClasspathChanges( getClasspathChanges(
args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot, args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withAbiSnapshot,
caches.platformCache, scopes caches.platformCache, scopes
) )
} }
@@ -911,7 +911,7 @@ Finished executing task ':$taskName'|
add("--dry-run") add("--dry-run")
} }
if (options.abiSnapshot) { if (options.abiSnapshot) {
add("-Dkotlin.incremental.classpath.snapshot.enabled=true") add("-Pkotlin.incremental.classpath.snapshot.enabled=true")
} }
if (options.hierarchicalMPPStructureSupport != null) { if (options.hierarchicalMPPStructureSupport != null) {
@@ -126,11 +126,10 @@ class BuildCacheIT : KGPBaseTest() {
@DisplayName("Restore from build cache should not break incremental compilation") @DisplayName("Restore from build cache should not break incremental compilation")
@GradleTest @GradleTest
fun testIncrementalCompilationAfterCacheHit(gradleVersion: GradleVersion) { fun testIncrementalCompilationAfterCacheHit(gradleVersion: GradleVersion) {
val withSnapshotProperty = "-Dkotlin.incremental.classpath.snapshot.enabled=true" project("incrementalMultiproject", gradleVersion, buildOptions = defaultBuildOptions.copy(useICClasspathSnapshot = true)) {
project("incrementalMultiproject", gradleVersion) {
enableLocalBuildCache(localBuildCacheDir) enableLocalBuildCache(localBuildCacheDir)
build("assemble", withSnapshotProperty) build("assemble")
build("clean", "assemble", withSnapshotProperty) { build("clean", "assemble") {
assertTasksFromCache(":lib:compileKotlin") assertTasksFromCache(":lib:compileKotlin")
assertTasksFromCache(":app:compileKotlin") assertTasksFromCache(":app:compileKotlin")
} }
@@ -138,8 +137,9 @@ class BuildCacheIT : KGPBaseTest() {
bKtSourceFile.modify { it.replace("fun b() {}", "fun b() {}\nfun b2() {}") } bKtSourceFile.modify { it.replace("fun b() {}", "fun b() {}\nfun b2() {}") }
build("assemble", withSnapshotProperty, buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) { build("assemble", buildOptions = defaultBuildOptions.copy(useICClasspathSnapshot = true, logLevel = LogLevel.DEBUG)) {
assertOutputDoesNotContain("[KOTLIN] [IC] Non-incremental compilation will be performed") assertOutputDoesNotContain("[KOTLIN] [IC] Non-incremental compilation will be performed")
assertOutputContains("Incremental compilation with ABI snapshot enabled")
assertCompiledKotlinSources(setOf(bKtSourceFile).map { it.relativeTo(projectPath)}, output) assertCompiledKotlinSources(setOf(bKtSourceFile).map { it.relativeTo(projectPath)}, output)
} }
@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.gradle
import org.gradle.api.logging.LogLevel import org.gradle.api.logging.LogLevel
import org.gradle.api.logging.configuration.WarningMode import org.gradle.api.logging.configuration.WarningMode
import org.gradle.testkit.runner.BuildResult
import org.gradle.util.GradleVersion import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.gradle.testbase.* import org.jetbrains.kotlin.gradle.testbase.*
import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.DisplayName
@@ -322,30 +323,33 @@ class BuildCacheRelocationIT : KGPBaseTest() {
@DisplayName("Kotlin incremental compilation should work correctly") @DisplayName("Kotlin incremental compilation should work correctly")
@GradleTest @GradleTest
fun testKotlinIncrementalCompilation(gradleVersion: GradleVersion) { fun testKotlinIncrementalCompilation(gradleVersion: GradleVersion) {
checkKotlinIncrementalCompilation(gradleVersion) checkKotlinIncrementalCompilationAfterCacheHit(gradleVersion) {
assertNonIncrementalCompilation()
}
} }
@DisplayName("Kotlin incremental compilation with `kotlin.incremental.useClasspathSnapshot` feature should work correctly") @DisplayName("Kotlin incremental compilation with `kotlin.incremental.useClasspathSnapshot` feature should work correctly")
@GradleTest @GradleTest
fun testKotlinIncrementalCompilation_withGradleClasspathSnapshot(gradleVersion: GradleVersion) { fun testKotlinIncrementalCompilation_withGradleClasspathSnapshot(gradleVersion: GradleVersion) {
checkKotlinIncrementalCompilation(gradleVersion, useGradleClasspathSnapshot = true) checkKotlinIncrementalCompilationAfterCacheHit(gradleVersion, defaultBuildOptions.copy(useGradleClasspathSnapshot = true)) {
assertIncrementalCompilation(listOf("src/main/kotlin/foo.kt", "src/main/kotlin/fooUsage.kt"))
}
} }
@DisplayName("Kotlin incremental compilation with `kotlin.incremental.classpath.snapshot.enabled` feature should work correctly") @DisplayName("Kotlin incremental compilation with `kotlin.incremental.classpath.snapshot.enabled` feature should work correctly")
@GradleTest @GradleTest
fun testKotlinIncrementalCompilation_withICClasspathSnapshot(gradleVersion: GradleVersion) { fun testKotlinIncrementalCompilation_withICClasspathSnapshot(gradleVersion: GradleVersion) {
checkKotlinIncrementalCompilation(gradleVersion, useICClasspathSnapshot = true) checkKotlinIncrementalCompilationAfterCacheHit(gradleVersion, defaultBuildOptions.copy(useICClasspathSnapshot = true)) {
assertIncrementalCompilation(listOf("src/main/kotlin/foo.kt", "src/main/kotlin/fooUsage.kt"))
assertOutputContains("Incremental compilation with ABI snapshot enabled")
}
} }
private fun checkKotlinIncrementalCompilation( private fun checkKotlinIncrementalCompilationAfterCacheHit(
gradleVersion: GradleVersion, gradleVersion: GradleVersion,
useGradleClasspathSnapshot: Boolean? = null, buildOptions: BuildOptions = defaultBuildOptions,
useICClasspathSnapshot: Boolean? = null assertions: BuildResult.() -> Unit
) { ) {
val buildOptions = defaultBuildOptions.copy(
useGradleClasspathSnapshot = useGradleClasspathSnapshot,
useICClasspathSnapshot = useICClasspathSnapshot
)
val (firstProject, secondProject) = prepareTestProjects("buildCacheSimple", gradleVersion, buildOptions) val (firstProject, secondProject) = prepareTestProjects("buildCacheSimple", gradleVersion, buildOptions)
// First build, should be stored into the build cache: // First build, should be stored into the build cache:
@@ -361,14 +365,7 @@ class BuildCacheRelocationIT : KGPBaseTest() {
// Check whether compilation after a cache hit is incremental (KT-34862) // Check whether compilation after a cache hit is incremental (KT-34862)
val fooKtSourceFile = secondProject.kotlinSourcesDir().resolve("foo.kt") val fooKtSourceFile = secondProject.kotlinSourcesDir().resolve("foo.kt")
fooKtSourceFile.modify { it.replace("Int = 1", "String = \"abc\"") } fooKtSourceFile.modify { it.replace("Int = 1", "String = \"abc\"") }
secondProject.build("assemble", buildOptions = buildOptions.copy(logLevel = LogLevel.DEBUG)) { secondProject.build("assemble", buildOptions = buildOptions.copy(logLevel = LogLevel.DEBUG), assertions = assertions)
if (useGradleClasspathSnapshot == true || useICClasspathSnapshot == true) {
assertIncrementalCompilation(listOf("src/main/kotlin/foo.kt", "src/main/kotlin/fooUsage.kt"))
} else {
assertNonIncrementalCompilation()
}
}
// Revert the change to the return type of foo(), and check if we get a cache hit // Revert the change to the return type of foo(), and check if we get a cache hit
fooKtSourceFile.modify { it.replace("String = \"abc\"", "Int = 1") } fooKtSourceFile.modify { it.replace("String = \"abc\"", "Int = 1") }
secondProject.build("clean", "assemble") { secondProject.build("clean", "assemble") {
@@ -9,7 +9,6 @@ import org.gradle.api.logging.LogLevel
import org.gradle.api.logging.configuration.WarningMode import org.gradle.api.logging.configuration.WarningMode
import org.gradle.util.GradleVersion import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.cli.common.CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM import org.jetbrains.kotlin.cli.common.CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM
import org.jetbrains.kotlin.cli.common.CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS
import org.jetbrains.kotlin.gradle.BaseGradleIT import org.jetbrains.kotlin.gradle.BaseGradleIT
import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType
import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.report.BuildReportType
@@ -88,7 +87,7 @@ data class BuildOptions(
} }
useGradleClasspathSnapshot?.let { arguments.add("-P${COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.property}=$it") } useGradleClasspathSnapshot?.let { arguments.add("-P${COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.property}=$it") }
useICClasspathSnapshot?.let { arguments.add("-D${COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.property}=$it") } useICClasspathSnapshot?.let { arguments.add("-Pkotlin.incremental.classpath.snapshot.enabled=$it") }
if (gradleVersion >= GradleVersion.version("6.5")) { if (gradleVersion >= GradleVersion.version("6.5")) {
if (fileSystemWatchEnabled) { if (fileSystemWatchEnabled) {
@@ -7,8 +7,10 @@ package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.logging.Logger import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.build.report.metrics.* import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.toBooleanLenient
import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.common.* import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.gradle.logging.* import org.jetbrains.kotlin.gradle.logging.*
@@ -20,6 +22,7 @@ import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState
import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError
import org.jetbrains.kotlin.gradle.utils.stackTraceAsString import org.jetbrains.kotlin.gradle.utils.stackTraceAsString
import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.incremental.ClasspathChanges
import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.io.* import java.io.*
@@ -129,7 +132,9 @@ internal class GradleKotlinCompilerWork @Inject constructor(
} finally { } finally {
val taskInfo = TaskExecutionInfo( val taskInfo = TaskExecutionInfo(
changedFiles = incrementalCompilationEnvironment?.changedFiles, changedFiles = incrementalCompilationEnvironment?.changedFiles,
compilerArguments = compilerArgs compilerArguments = compilerArgs,
withAbiSnapshot = incrementalCompilationEnvironment?.withAbiSnapshot,
withArtifactTransform = incrementalCompilationEnvironment?.classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled
) )
val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines, taskInfo = taskInfo) val result = TaskExecutionResult(buildMetrics = metrics.getMetrics(), icLogLines = icLogLines, taskInfo = taskInfo)
TaskExecutionResults[taskPath] = result TaskExecutionResults[taskPath] = result
@@ -282,7 +287,8 @@ internal class GradleKotlinCompilerWork @Inject constructor(
outputFiles = outputFiles, outputFiles = outputFiles,
multiModuleICSettings = icEnv.multiModuleICSettings, multiModuleICSettings = icEnv.multiModuleICSettings,
modulesInfo = incrementalModuleInfo!!, modulesInfo = incrementalModuleInfo!!,
kotlinScriptExtensions = kotlinScriptExtensions kotlinScriptExtensions = kotlinScriptExtensions,
withAbiSnapshot = icEnv.withAbiSnapshot
) )
log.info("Options for KOTLIN DAEMON: $compilationOptions") log.info("Options for KOTLIN DAEMON: $compilationOptions")
@@ -17,9 +17,10 @@ internal class IncrementalCompilationEnvironment(
val workingDir: File, val workingDir: File,
val usePreciseJavaTracking: Boolean = false, val usePreciseJavaTracking: Boolean = false,
val disableMultiModuleIC: Boolean = false, val disableMultiModuleIC: Boolean = false,
val multiModuleICSettings: MultiModuleICSettings val multiModuleICSettings: MultiModuleICSettings,
val withAbiSnapshot: Boolean = false
) : Serializable { ) : Serializable {
companion object { companion object {
const val serialVersionUID: Long = 0 const val serialVersionUID: Long = 1
} }
} }
@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.gradle.dsl.NativeCacheKind
import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessageOutputStreamHandler.Companion.IGNORE_TCSM_OVERFLOW import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessageOutputStreamHandler.Companion.IGNORE_TCSM_OVERFLOW
import org.jetbrains.kotlin.gradle.plugin.Kotlin2JsPlugin.Companion.NOWARN_2JS_FLAG import org.jetbrains.kotlin.gradle.plugin.Kotlin2JsPlugin.Companion.NOWARN_2JS_FLAG
import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.Companion.jsCompilerProperty import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.Companion.jsCompilerProperty
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_ABI_SNAPSHOT
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_CINTEROP_COMMONIZATION import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_CINTEROP_COMMONIZATION
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_GRANULAR_SOURCE_SETS_METADATA import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_GRANULAR_SOURCE_SETS_METADATA
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_OPTIMISTIC_NUMBER_COMMONIZATION import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_MPP_ENABLE_OPTIMISTIC_NUMBER_COMMONIZATION
@@ -49,6 +50,7 @@ internal fun PropertiesProvider.mapKotlinTaskProperties(task: AbstractKotlinComp
} }
} }
task.jvmTargetValidationMode.set(jvmTargetValidationMode) task.jvmTargetValidationMode.set(jvmTargetValidationMode)
task.useKotlinAbiSnapshot.value(useKotlinAbiSnapshot).disallowChanges()
} }
if (task is Kotlin2JsCompile) { if (task is Kotlin2JsCompile) {
@@ -174,6 +176,9 @@ internal class PropertiesProvider private constructor(private val project: Proje
return gradleProperty || systemProperty return gradleProperty || systemProperty
} }
val useKotlinAbiSnapshot: Boolean
get() = booleanProperty(KOTLIN_ABI_SNAPSHOT) ?: false
val useFir: Boolean? val useFir: Boolean?
get() = booleanProperty("kotlin.useFir") get() = booleanProperty("kotlin.useFir")
@@ -503,6 +508,7 @@ internal class PropertiesProvider private constructor(private val project: Proje
const val KOTLIN_MPP_ENABLE_OPTIMISTIC_NUMBER_COMMONIZATION = "kotlin.mpp.enableOptimisticNumberCommonization" const val KOTLIN_MPP_ENABLE_OPTIMISTIC_NUMBER_COMMONIZATION = "kotlin.mpp.enableOptimisticNumberCommonization"
const val KOTLIN_KPM_EXPERIMENTAL_MODEL_MAPPING = "kotlin.kpm.experimentalModelMapping" const val KOTLIN_KPM_EXPERIMENTAL_MODEL_MAPPING = "kotlin.kpm.experimentalModelMapping"
const val KOTLIN_MPP_ENABLE_PLATFORM_INTEGER_COMMONIZATION = "kotlin.mpp.enablePlatformIntegerCommonization" const val KOTLIN_MPP_ENABLE_PLATFORM_INTEGER_COMMONIZATION = "kotlin.mpp.enablePlatformIntegerCommonization"
const val KOTLIN_ABI_SNAPSHOT = "kotlin.incremental.classpath.snapshot.enabled"
} }
companion object { companion object {
@@ -40,10 +40,7 @@ class BuildScanStatisticsListener(
log.debug("Collect data takes $collectDataDuration: $compileStatData") log.debug("Collect data takes $collectDataDuration: $compileStatData")
compileStatData?.also { compileStatData?.also {
val reportDataDuration = measureTimeMillis { report(it)
report(it)
}
log.debug("Report data takes $reportDataDuration: $compileStatData")
} }
} }
} }
@@ -16,6 +16,7 @@ import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
import org.jetbrains.kotlin.gradle.plugin.stat.StatTag import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
import org.jetbrains.kotlin.gradle.report.TaskExecutionResult import org.jetbrains.kotlin.gradle.report.TaskExecutionResult
import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
import java.lang.management.ManagementFactory import java.lang.management.ManagementFactory
import java.net.InetAddress import java.net.InetAddress
import java.util.* import java.util.*
@@ -100,12 +101,7 @@ class KotlinBuildStatListener {
private fun parseTags(taskExecutionResult: TaskExecutionResult?): List<StatTag> { private fun parseTags(taskExecutionResult: TaskExecutionResult?): List<StatTag> {
val tags = ArrayList<StatTag>() val tags = ArrayList<StatTag>()
CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient()?.let {
if (it) tags.add(StatTag.ABI_SNAPSHOT)
}
CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_ARTIFACT_TRANSFORM.value.toBooleanLenient()?.let {
if (it) tags.add(StatTag.ARTIFACT_TRANSFORM)
}
val nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap() val nonIncrementalAttributes = taskExecutionResult?.buildMetrics?.buildAttributes?.asMap() ?: emptyMap()
if (nonIncrementalAttributes.isEmpty()) { if (nonIncrementalAttributes.isEmpty()) {
@@ -114,6 +110,15 @@ class KotlinBuildStatListener {
tags.add(StatTag.NON_INCREMENTAL) tags.add(StatTag.NON_INCREMENTAL)
} }
val taskInfo = taskExecutionResult?.taskInfo
taskInfo?.withAbiSnapshot?.ifTrue {
tags.add(StatTag.ABI_SNAPSHOT)
}
taskInfo?.withArtifactTransform?.ifTrue {
tags.add(StatTag.ARTIFACT_TRANSFORM)
}
val debugConfiguration = "-agentlib:" val debugConfiguration = "-agentlib:"
if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) { if (ManagementFactory.getRuntimeMXBean().inputArguments.firstOrNull { it.startsWith(debugConfiguration) } != null) {
tags.add(StatTag.GRADLE_DEBUG) tags.add(StatTag.GRADLE_DEBUG)
@@ -16,5 +16,7 @@ internal class TaskExecutionResult(
internal class TaskExecutionInfo( internal class TaskExecutionInfo(
val changedFiles: ChangedFiles? = null, val changedFiles: ChangedFiles? = null,
val compilerArguments: Array<String> = emptyArray() val compilerArguments: Array<String> = emptyArray(),
val withArtifactTransform: Boolean? = false,
val withAbiSnapshot: Boolean? = false
) )
@@ -651,6 +651,9 @@ abstract class KotlinCompile @Inject constructor(
return super.getClasspath() return super.getClasspath()
} }
@get:Input
abstract val useKotlinAbiSnapshot: Property<Boolean>
@get:Nested @get:Nested
abstract val classpathSnapshotProperties: ClasspathSnapshotProperties abstract val classpathSnapshotProperties: ClasspathSnapshotProperties
@@ -770,7 +773,8 @@ abstract class KotlinCompile @Inject constructor(
workingDir = taskBuildCacheableOutputDirectory.get().asFile, workingDir = taskBuildCacheableOutputDirectory.get().asFile,
usePreciseJavaTracking = usePreciseJavaTracking, usePreciseJavaTracking = usePreciseJavaTracking,
disableMultiModuleIC = disableMultiModuleIC, disableMultiModuleIC = disableMultiModuleIC,
multiModuleICSettings = multiModuleICSettings multiModuleICSettings = multiModuleICSettings,
withAbiSnapshot = useKotlinAbiSnapshot.get()
) )
} else null } else null