diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt index bdfe01a4135..3fb89625df7 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt @@ -18,9 +18,11 @@ enum class BuildAttributeKind : Serializable { enum class BuildAttribute(val kind: BuildAttributeKind, val readableString: String) : Serializable { NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON, "Build history file not found"), NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON, "ABI snapshot not found"), - INTERNAL_ERROR(BuildAttributeKind.REBUILD_REASON, "Internal error during preparation of IC round"), CLASSPATH_SNAPSHOT_NOT_FOUND(BuildAttributeKind.REBUILD_REASON, "Classpath snapshot not found"), - INCREMENTAL_COMPILATION_FAILED(BuildAttributeKind.REBUILD_REASON, "Incremental compilation failed"), + IC_FAILED_TO_GET_CHANGED_FILES(BuildAttributeKind.REBUILD_REASON, "Failed to get changed files"), + IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE(BuildAttributeKind.REBUILD_REASON, "Failed to compute files to recompile"), + IC_FAILED_TO_COMPILE_INCREMENTALLY(BuildAttributeKind.REBUILD_REASON, "Failed to compile incrementally"), + IC_FAILED_TO_CLOSE_CACHES(BuildAttributeKind.REBUILD_REASON, "Failed to close caches"), UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON, "Unknown Gradle changes"), JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON, "Untracked Java file is removed"), JAVA_CHANGE_UNEXPECTED_PSI(BuildAttributeKind.REBUILD_REASON, "Java PSI file is expected"), diff --git a/build-common/src/org/jetbrains/kotlin/incremental/fileUtils.kt b/build-common/src/org/jetbrains/kotlin/incremental/fileUtils.kt index 0e2b388f121..edbe1660827 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/fileUtils.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/fileUtils.kt @@ -29,15 +29,15 @@ fun File.isClassFile(): Boolean = extension.equals("class", ignoreCase = true) /** - * Deletes the contents of this directory (not the directory itself) if it exists, or creates the directory if it does not yet exist. + * Deletes the contents of this directory (not the directory itself). * - * If this is a regular file, this method will throw an exception. + * If the directory does not exist or if this is a regular file, this method will throw an exception. */ -fun File.cleanDirectoryContents() { +fun File.deleteDirectoryContents() { when { isDirectory -> listFiles()!!.forEach { it.deleteRecursivelyOrThrow() } - isFile -> error("File.cleanDirectoryContents does not accept a regular file: $path") - else -> mkdirsOrThrow() + isFile -> error("Expected a directory but found a regular file: $path") + else -> error("Directory does not exist: $path") } } @@ -49,11 +49,12 @@ fun File.deleteRecursivelyOrThrow() { } /** - * Creates this directory (if it does not yet exist), throwing an exception if the directiory creation failed or if a regular file already - * exists at this path. + * Creates this directory (if it does not yet exist). + * + * If a regular file already exists at this path, this method will throw an exception. */ @Suppress("SpellCheckingInspection") -fun File.mkdirsOrThrow() { +fun File.createDirectory() { when { isDirectory -> Unit isFile -> error("A regular file already exists at this path: $path") diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt index 591f1615af1..daf5a39062f 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt @@ -16,7 +16,10 @@ package org.jetbrains.kotlin.daemon.common -import org.jetbrains.kotlin.cli.common.repl.* +import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult +import org.jetbrains.kotlin.cli.common.repl.ReplEvalResult import java.io.File import java.io.Serializable import java.rmi.Remote @@ -46,20 +49,26 @@ interface CompileService : Remote { override fun equals(other: Any?): Boolean = other is Good<*> && this.result == other.result override fun hashCode(): Int = this::class.java.hashCode() + (result?.hashCode() ?: 1) } + class Ok : CallResult() { override fun get(): Nothing = throw IllegalStateException("Get is inapplicable to Ok call result") override fun equals(other: Any?): Boolean = other is Ok override fun hashCode(): Int = this::class.java.hashCode() + 1 // avoiding clash with the hash of class itself } + class Dying : CallResult() { override fun get(): Nothing = throw IllegalStateException("Service is dying") override fun equals(other: Any?): Boolean = other is Dying override fun hashCode(): Int = this::class.java.hashCode() + 1 // see comment to Ok.hashCode } - class Error(val message: String) : CallResult() { - override fun get(): Nothing = throw Exception(message) - override fun equals(other: Any?): Boolean = other is Error && this.message == other.message - override fun hashCode(): Int = this::class.java.hashCode() + message.hashCode() + + class Error(val message: String?, val cause: Throwable?) : CallResult() { + constructor(cause: Throwable) : this(message = null, cause = cause) + constructor(message: String) : this(message = message, cause = null) + + override fun get(): Nothing = throw Exception(message, cause) + override fun equals(other: Any?): Boolean = other is Error && this.message == other.message && this.cause == other.cause + override fun hashCode(): Int = this::class.java.hashCode() + (cause?.hashCode() ?: 1) + (message?.hashCode() ?: 2) // see comment to Ok.hashCode } val isGood: Boolean get() = this is Good<*> diff --git a/compiler/daemon/daemon-tests/test/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/daemon/daemon-tests/test/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index 05df17701cf..22b41198b1e 100644 --- a/compiler/daemon/daemon-tests/test/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/daemon/daemon-tests/test/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -781,7 +781,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { } catch (e: Exception) { TestCase.assertEquals( "Unable to use scripting/REPL in the daemon: no scripting plugin loaded", - e.message + e.cause?.message ) isErrorThrown = true } finally { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index 0970df74888..51e8eb30eb6 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -47,8 +47,8 @@ import org.jetbrains.kotlin.daemon.report.getBuildReporter import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.components.EnumWhenTracker import org.jetbrains.kotlin.incremental.components.ExpectActualTracker -import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.InlineConstTracker +import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryAndroid @@ -507,7 +507,7 @@ abstract class CompileServiceImplBase( body() } catch (e: Throwable) { log.log(Level.SEVERE, "Exception", e) - CompileService.CallResult.Error(e.message ?: "unknown") + CompileService.CallResult.Error(e) } } } @@ -614,7 +614,7 @@ abstract class CompileServiceImplBase( workingDir, reporter, buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, - outputFiles = incrementalCompilationOptions.outputFiles, + outputDirs = incrementalCompilationOptions.outputFiles, usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking, modulesApiHistory = modulesApiHistory, kotlinSourceFilesExtensions = allKotlinExtensions, diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt index 886e03a6a77..fbf4cd077dc 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt @@ -23,7 +23,6 @@ import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult import org.jetbrains.kotlin.cli.js.K2JSCompiler import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment -import org.jetbrains.kotlin.cli.jvm.compiler.jarfs.FastJarFileSystem import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler import org.jetbrains.kotlin.config.KotlinCompilerVersion import org.jetbrains.kotlin.config.Services @@ -749,7 +748,7 @@ class CompileServiceServerSideImpl( body() } catch (e: Throwable) { log.log(Level.SEVERE, "Exception", e) - CompileService.CallResult.Error(e.message ?: "unknown") + CompileService.CallResult.Error(e) } } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt index 027fcb06171..f657154b33a 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt @@ -5,8 +5,6 @@ package org.jetbrains.kotlin.incremental -import org.jetbrains.kotlin.build.report.BuildReporter -import org.jetbrains.kotlin.build.report.info import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmNameResolver import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil @@ -152,14 +150,9 @@ class AbiSnapshotImpl(override val protos: MutableMap) : AbiS } } - fun read(file: File, reporter: BuildReporter): AbiSnapshot? { - if (!file.exists()) { - reporter.info { "jar snapshot $file is found for jar" } - return null - } - - ObjectInputStream(FileInputStream(file)).use { - return it.readAbiSnapshot() + fun read(file: File): AbiSnapshot { + return ObjectInputStream(FileInputStream(file)).use { + it.readAbiSnapshot() } } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt index 29e5af2fefe..ea529fe5de0 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt @@ -20,10 +20,8 @@ import java.io.File import java.io.Serializable sealed class ChangedFiles : Serializable { - class Known(val modified: List, val removed: List) : ChangedFiles() + class Known(val modified: List, val removed: List, val forDependencies: Boolean = false) : ChangedFiles() class Unknown : ChangedFiles() - class Dependencies(val modified: List, val removed: List) : ChangedFiles() - companion object { const val serialVersionUID: Long = 0 } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index caaff24b041..f1caeeeac4b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -16,30 +16,29 @@ package org.jetbrains.kotlin.incremental +import com.google.common.io.Closer import org.jetbrains.kotlin.build.report.ICReporter -import org.jetbrains.kotlin.build.report.info import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol +import java.io.Closeable import java.io.File - abstract class IncrementalCachesManager>( cachesRootDir: File, rootProjectDir: File?, protected val reporter: ICReporter, storeFullFqNamesInLookupCache: Boolean = false, trackChangesInLookupCache: Boolean = false -) { +) : Closeable { val pathConverter = IncrementalFileToPathConverter(rootProjectDir) private val caches = arrayListOf() - var isClosed = false - var isSuccessfulyClosed = false + private var isClosed = false @Synchronized protected fun T.registerCache() { - assert(!isClosed) { "Attempted to add new cache into closed storage." } + check(!isClosed) { "This cache storage has already been closed" } caches.add(this) } @@ -51,33 +50,29 @@ abstract class IncrementalCachesManager = emptyList(), + + /** + * Output directories of the compilation. These include: + * 1. The classes output directory + * 2. [workingDir] + * 3. Any additional output directories (e.g., classpath snapshot directory or Kapt generated-stubs directory) + * + * We will clean these directories when compiling non-incrementally. + * + * If this property is not set, the directories to clean will include the first 2 directories above. + */ + private val outputDirs: Collection?, + protected val withAbiSnapshot: Boolean = false ) { protected val cacheDirectory = File(workingDir, cacheDirName) - protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) + private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME) private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) protected open val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS @@ -75,184 +86,204 @@ abstract class IncrementalCompilerRunner< providedChangedFiles: ChangedFiles?, projectDir: File? = null ): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION_DAEMON) { - try { - compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir) - } finally { - reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) { - reporter.addMetric( - BuildPerformanceMetric.SNAPSHOT_SIZE, - buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length() + return when (val result = tryCompileIncrementally(allSourceFiles, providedChangedFiles, args, projectDir, messageCollector)) { + is ICResult.Completed -> { + reporter.debug { "Incremental compilation completed" } + result.exitCode + } + is ICResult.RequiresRebuild -> { + reporter.info { "Non-incremental compilation will be performed: ${result.reason}" } + reporter.addAttribute(result.reason) + + compileNonIncrementally( + result.reason, allSourceFiles, args, projectDir, trackChangedFiles = providedChangedFiles == null, messageCollector ) - if (cacheDirectory.exists() && cacheDirectory.isDirectory()) { - cacheDirectory.walkTopDown().filter { it.isFile }.map { it.length() }.sum().let { - reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, it) - } + } + is ICResult.Failed -> { + reporter.warn { + // The indentation after the first line is intentional (so that this message is distinct from next message) + """ + |Incremental compilation was attempted but failed: + | ${result.reason.readableString}: ${result.cause.stackTraceToString().removeSuffixIfPresent("\n")} + | Falling back to non-incremental compilation (reason = ${result.reason}) + | To help us fix this issue, please file a bug at https://youtrack.jetbrains.com/issues/KT with the above stack trace. + | (Be sure to search for the above exception in existing issues first to avoid filing duplicated bugs.) + """.trimMargin() } + // TODO: Collect the stack trace too + reporter.addAttribute(result.reason) + + compileNonIncrementally( + result.reason, allSourceFiles, args, projectDir, trackChangedFiles = providedChangedFiles == null, messageCollector + ) } } } - fun rebuild( - reason: BuildAttribute, - allSourceFiles: List, - args: Args, - messageCollector: MessageCollector, - providedChangedFiles: ChangedFiles?, - projectDir: File? = null, - classpathAbiSnapshot: Map - ): ExitCode { - reporter.info { "Non-incremental compilation will be performed: $reason" } - reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) { - cleanOutputsAndLocalStateOnRebuild(args) - } - val caches = createCacheManager(args, projectDir) - try { - if (providedChangedFiles == null) { - caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) - } - val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } - return compileIncrementally( - args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withAbiSnapshot, - classpathAbiSnapshot = classpathAbiSnapshot - ).also { - if (it == ExitCode.OK) { - performWorkAfterSuccessfulCompilation(caches, wasIncremental = false) - } - } - } finally { - caches.close(true) - } - } + /** The result when attempting to compile incrementally ([tryCompileIncrementally]). */ + private sealed interface ICResult { - private fun compileImpl( - allSourceFiles: List, - args: Args, - messageCollector: MessageCollector, - providedChangedFiles: ChangedFiles?, - projectDir: File? = null - ): ExitCode { - var caches = createCacheManager(args, projectDir) - var rebuildReason = BuildAttribute.INTERNAL_ERROR + /** Incremental compilation completed with an [ExitCode]. */ + class Completed(val exitCode: ExitCode) : ICResult - val classpathAbiSnapshot = - if (withAbiSnapshot) { - reporter.info { "Incremental compilation with ABI snapshot enabled" } - reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) { - setupJarDependencies(args, withAbiSnapshot, reporter) - } - } else { - emptyMap() - } + /** Incremental compilation was not possible for some valid reason (e.g., for a clean build). */ + class RequiresRebuild(val reason: BuildAttribute) : ICResult - try { - val changedFiles = when (providedChangedFiles) { - is ChangedFiles.Dependencies -> { - val changedSources = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) - ChangedFiles.Known( - providedChangedFiles.modified + changedSources.modified, - providedChangedFiles.removed + changedSources.removed - ) - } - null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) - else -> providedChangedFiles - } - - var compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot) - val abiSnapshot = if (compilationMode is CompilationMode.Incremental && withAbiSnapshot) { - AbiSnapshotImpl.read(abiSnapshotFile, reporter) - } else { - if (withAbiSnapshot) { - compilationMode = CompilationMode.Rebuild(BuildAttribute.NO_ABI_SNAPSHOT) - } - null - } - - when (compilationMode) { - is CompilationMode.Incremental -> { - try { - reporter.debug { "Performing incremental compilation" } - val exitCode = if (withAbiSnapshot) { - compileIncrementally( - args, caches, allSourceFiles, compilationMode, messageCollector, - withAbiSnapshot, abiSnapshot!!, classpathAbiSnapshot - ) - } else { - compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector, withAbiSnapshot) - } - if (exitCode == ExitCode.OK) { - performWorkAfterSuccessfulCompilation(caches, wasIncremental = true) - } - return exitCode - } catch (e: Throwable) { - reporter.warn { - "Incremental compilation failed: ${e.stackTraceToString().suffixIfNot("\n")}" + - "Falling back to non-incremental compilation" - } - rebuildReason = BuildAttribute.INCREMENTAL_COMPILATION_FAILED - } - } - is CompilationMode.Rebuild -> rebuildReason = compilationMode.reason - } - } catch (e: Exception) { - reporter.warn { - "Incremental compilation analysis failed: ${e.stackTraceToString().suffixIfNot("\n")}" + - "Falling back to non-incremental compilation" - } - } finally { - if (!caches.close(flush = true)) { - reporter.info { "Unable to close IC caches. Cleaning internal state" } - cleanOutputsAndLocalStateOnRebuild(args) - } - } - return rebuild(rebuildReason, allSourceFiles, args, messageCollector, providedChangedFiles, projectDir, classpathAbiSnapshot) + /** Incremental compilation failed with an exception. */ + class Failed(val reason: BuildAttribute, val cause: Throwable) : ICResult } /** - * Deletes output files and contents of output directories on rebuild, including `@LocalState` files/directories. + * Attempts to compile incrementally and returns either [ICResult.Completed], [ICResult.RequiresRebuild], or [ICResult.Failed]. + * + * Note that parts of this function may still throw exceptions that are not caught and wrapped by [ICResult.Failed] because they are not + * meant to be caught. + */ + private fun tryCompileIncrementally( + allSourceFiles: List, + providedChangedFiles: ChangedFiles?, + args: Args, + projectDir: File?, + messageCollector: MessageCollector + ): ICResult { + if (providedChangedFiles is ChangedFiles.Unknown) { + return ICResult.RequiresRebuild(UNKNOWN_CHANGES_IN_GRADLE_INPUTS) + } + providedChangedFiles as ChangedFiles.Known? + + val caches = createCacheManager(args, projectDir) + val exitCode: ExitCode + try { + // Step 1: Get changed files + val changedFiles: ChangedFiles.Known = try { + getChangedFiles(providedChangedFiles, allSourceFiles, caches) + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e) + } + + val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null + + // Step 2: Compute files to recompile + val compilationMode = try { + reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) { + calculateSourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap()) + } + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e) + } + + if (compilationMode is CompilationMode.Rebuild) { + return ICResult.RequiresRebuild(compilationMode.reason) + } + + val abiSnapshotData = if (withAbiSnapshot) { + if (!abiSnapshotFile.exists()) { + reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" } + return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT) + } + reporter.info { "Incremental compilation with ABI snapshot enabled" } + AbiSnapshotData( + snapshot = AbiSnapshotImpl.read(abiSnapshotFile), + classpathAbiSnapshot = classpathAbiSnapshot!! + ) + } else null + + // Step 3: Compile incrementally + exitCode = try { + compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector) + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e) + } + } catch (e: Throwable) { + // Because `caches` is a Closeable resource, it is good practice to close them in the event of an exception (in addition to + // closing them after a normal use). + try { + caches.close() + } catch (e2: Throwable) { + e.addSuppressed(e2) + } + throw e + } + try { + caches.close() + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_CLOSE_CACHES, e) + } + + return ICResult.Completed(exitCode) + } + + private fun compileNonIncrementally( + rebuildReason: BuildAttribute, + allSourceFiles: List, + args: Args, + projectDir: File?, + trackChangedFiles: Boolean, // Whether we need to track changes to the source files or the build system already handles it + messageCollector: MessageCollector, + ): ExitCode { + reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) { + val mainOutputDirs = setOf(destinationDir(args), workingDir) + val outputDirsToClean = outputDirs?.also { + check(it.containsAll(mainOutputDirs)) { "outputDirs is missing classesDir and workingDir: $it" } + } ?: mainOutputDirs + + reporter.debug { "Cleaning output directories" } + cleanOrCreateDirectories(outputDirsToClean) + } + return createCacheManager(args, projectDir).use { caches -> + if (trackChangedFiles) { + caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) + } + val abiSnapshotData = if (withAbiSnapshot) { + AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args)) + } else null + + compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector) + } + } + + private class AbiSnapshotData(val snapshot: AbiSnapshot, val classpathAbiSnapshot: Map) + + private fun getClasspathAbiSnapshot(args: Args): Map { + return reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) { + setupJarDependencies(args, reporter) + } + } + + /** + * Deletes the contents of the given directories (not the directories themselves). * * If the directories do not yet exist, they will be created. */ - private fun cleanOutputsAndLocalStateOnRebuild(args: Args) { - // Use Set as additionalOutputFiles may already contain destinationDir and workingDir - val outputFiles = setOf(destinationDir(args), workingDir) + additionalOutputFiles - - reporter.debug { "Cleaning outputs on rebuild" } - outputFiles.forEach { + private fun cleanOrCreateDirectories(outputDirs: Collection) { + outputDirs.toSet().forEach { when { - it.isDirectory -> { - reporter.debug { " Deleting contents of directory '${it.path}'" } - it.cleanDirectoryContents() - } - it.isFile -> { - reporter.debug { " Deleting file '${it.path}'" } - it.deleteRecursivelyOrThrow() - } + it.isDirectory -> it.deleteDirectoryContents() + it.isFile -> "Expected a directory but found a regular file: ${it.path}" + else -> it.createDirectory() } } } - private fun sourcesToCompile( - caches: CacheManager, - changedFiles: ChangedFiles, - args: Args, - messageCollector: MessageCollector, - dependenciesAbiSnapshots: Map - ): CompilationMode = - when (changedFiles) { - is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector, dependenciesAbiSnapshots) - is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) - is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)") + private fun getChangedFiles( + providedChangedFiles: ChangedFiles.Known?, + allSourceFiles: List, + caches: CacheManager + ): ChangedFiles.Known { + return when { + providedChangedFiles == null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) + providedChangedFiles.forDependencies -> { + val moreChangedFiles = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) + ChangedFiles.Known( + modified = providedChangedFiles.modified + moreChangedFiles.modified, + removed = providedChangedFiles.removed + moreChangedFiles.removed + ) + } + else -> providedChangedFiles } + } - private fun calculateSourcesToCompile( - caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector, - abiSnapshots: Map - ): CompilationMode = - reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) { - calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, abiSnapshots) - } - - protected abstract fun calculateSourcesToCompileImpl( + protected abstract fun calculateSourcesToCompile( caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, @@ -260,7 +291,7 @@ abstract class IncrementalCompilerRunner< classpathAbiSnapshots: Map ): CompilationMode - protected open fun setupJarDependencies(args: Args, withSnapshot: Boolean, reporter: BuildReporter): Map = mapOf() + protected open fun setupJarDependencies(args: Args, reporter: BuildReporter): Map = emptyMap() protected fun initDirtyFiles(dirtyFiles: DirtyFilesContainer, changedFiles: ChangedFiles.Known) { dirtyFiles.add(changedFiles.modified, "was modified since last time") @@ -284,7 +315,6 @@ abstract class IncrementalCompilerRunner< changesCollector: ChangesCollector ) - protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {} protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List, services: Services): Iterable = emptyList() @@ -315,33 +345,53 @@ abstract class IncrementalCompilerRunner< isIncremental: Boolean ): Pair> - protected open fun compileIncrementally( + private fun compileImpl( + compilationMode: CompilationMode, + allSourceFiles: List, args: Args, caches: CacheManager, - allKotlinSources: List, - compilationMode: CompilationMode, - originalMessageCollector: MessageCollector, - withSnapshot: Boolean, - abiSnapshot: AbiSnapshot = AbiSnapshotImpl(mutableMapOf()), - classpathAbiSnapshot: Map = HashMap() + abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true + messageCollector: MessageCollector ): ExitCode { - if (compilationMode is CompilationMode.Rebuild) { - reporter.info { "Non-incremental compilation will be performed: ${compilationMode.reason}" } + performWorkBeforeCompilation(compilationMode, args) + + val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } + val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector) + + performWorkAfterCompilation(compilationMode, exitCode, caches) + return exitCode + } + + protected open fun performWorkBeforeCompilation(compilationMode: CompilationMode, args: Args) {} + + protected open fun performWorkAfterCompilation(compilationMode: CompilationMode, exitCode: ExitCode, caches: CacheManager) { + collectMetrics() + } + + private fun collectMetrics() { + reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) { + reporter.addMetric( + BuildPerformanceMetric.SNAPSHOT_SIZE, + buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length() + ) + reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, cacheDirectory.walk().sumOf { it.length() }) } + } - preBuildHook(args, compilationMode) - + private fun doCompile( + compilationMode: CompilationMode, + allKotlinSources: List, + args: Args, + caches: CacheManager, + abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true + originalMessageCollector: MessageCollector + ): ExitCode { val dirtySources = when (compilationMode) { - is CompilationMode.Incremental -> { - compilationMode.dirtyFiles.toMutableLinkedSet() - } - is CompilationMode.Rebuild -> { - reporter.addAttribute(compilationMode.reason) - LinkedHashSet(allKotlinSources) - } + is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableLinkedSet() + is CompilationMode.Rebuild -> LinkedHashSet(allKotlinSources) } - val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), classpathAbiSnapshot) + val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), abiSnapshotData?.classpathAbiSnapshot ?: emptyMap()) val buildDirtyLookupSymbols = HashSet() val buildDirtyFqNames = HashSet() val allDirtySources = HashSet() @@ -412,8 +462,8 @@ abstract class IncrementalCompilerRunner< updateCaches(services, caches, generatedFiles, changesCollector) } if (compilationMode is CompilationMode.Rebuild) { - if (withSnapshot) { - abiSnapshot.protos.putAll(changesCollector.protoDataChanges()) + if (withAbiSnapshot) { + abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges()) } break } @@ -445,10 +495,10 @@ abstract class IncrementalCompilerRunner< buildDirtyFqNames.addAll(dirtyClassFqNames) //update - if (withSnapshot) { + if (withAbiSnapshot) { //TODO(valtman) check method/ kts class remove - changesCollector.protoDataRemoved().forEach { abiSnapshot.protos.remove(it) } - abiSnapshot.protos.putAll(changesCollector.protoDataChanges()) + changesCollector.protoDataRemoved().forEach { abiSnapshotData!!.snapshot.protos.remove(it) } + abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges()) } } @@ -457,9 +507,9 @@ abstract class IncrementalCompilerRunner< BuildInfo.write(currentBuildInfo, lastBuildInfoFile) //write abi snapshot - if (withSnapshot) { + if (withAbiSnapshot) { //TODO(valtman) check method/class remove - AbiSnapshotImpl.write(abiSnapshot, abiSnapshotFile) + AbiSnapshotImpl.write(abiSnapshotData!!.snapshot, abiSnapshotFile) } } } @@ -516,15 +566,6 @@ abstract class IncrementalCompilerRunner< BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter) } - /** - * Performs some work after a compilation if the compilation completed successfully (no exceptions were thrown AND exit code == 0). - * - * This method MUST NOT be called when the compilation failed because the results produced by the work here would be invalid. - * - * @wasIncremental whether the compilation was incremental or non-incremental - */ - protected open fun performWorkAfterSuccessfulCompilation(caches: CacheManager, wasIncremental: Boolean) {} - companion object { const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt" const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin" diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt index 081158f5298..7cc7bc238b2 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt @@ -72,7 +72,7 @@ class IncrementalFirJvmCompilerRunner( workingDir: File, reporter: BuildReporter, buildHistoryFile: File, - outputFiles: Collection, + outputDirs: Collection?, modulesApiHistory: ModulesApiHistory, kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS, classpathChanges: ClasspathChanges @@ -81,7 +81,7 @@ class IncrementalFirJvmCompilerRunner( reporter, false, buildHistoryFile, - outputFiles, + outputDirs, modulesApiHistory, kotlinSourceFilesExtensions, classpathChanges diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index bcc55c17e8f..6baca2c20db 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -93,6 +93,7 @@ class IncrementalJsCompilerRunner( "caches-js", reporter, buildHistoryFile = buildHistoryFile, + outputDirs = null, withAbiSnapshot = withAbiSnapshot ) { @@ -115,7 +116,7 @@ class IncrementalJsCompilerRunner( outputFile.parentFile } - override fun calculateSourcesToCompileImpl( + override fun calculateSourcesToCompile( caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments, diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 8531832bf35..484f888f6e0 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -88,7 +88,13 @@ fun makeIncrementally( val compiler = if (args.useK2 && args.useFirIC && args.useFirLT /* TODO: move LT check into runner */ ) IncrementalFirJvmCompilerRunner( - cachesDir, buildReporter, buildHistoryFile, emptyList(), EmptyModulesApiHistory, kotlinExtensions, ClasspathSnapshotDisabled + cachesDir, + buildReporter, + buildHistoryFile, + outputDirs = null, + EmptyModulesApiHistory, + kotlinExtensions, + ClasspathSnapshotDisabled ) else IncrementalJvmCompilerRunner( @@ -96,8 +102,8 @@ fun makeIncrementally( buildReporter, // Use precise setting in case of non-Gradle build usePreciseJavaTracking = !args.useK2, // TODO: add fir-based java classes tracker when available and set this to true - outputFiles = emptyList(), buildHistoryFile = buildHistoryFile, + outputDirs = null, modulesApiHistory = EmptyModulesApiHistory, kotlinSourceFilesExtensions = kotlinExtensions, classpathChanges = ClasspathSnapshotDisabled @@ -127,7 +133,7 @@ open class IncrementalJvmCompilerRunner( reporter: BuildReporter, private val usePreciseJavaTracking: Boolean, buildHistoryFile: File, - outputFiles: Collection, + outputDirs: Collection?, private val modulesApiHistory: ModulesApiHistory, override val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS, private val classpathChanges: ClasspathChanges, @@ -136,8 +142,8 @@ open class IncrementalJvmCompilerRunner( workingDir, "caches-jvm", reporter, - additionalOutputFiles = outputFiles, buildHistoryFile = buildHistoryFile, + outputDirs = outputDirs, withAbiSnapshot = withAbiSnapshot ) { override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager = @@ -181,7 +187,7 @@ open class IncrementalJvmCompilerRunner( else null - override fun calculateSourcesToCompileImpl( + override fun calculateSourcesToCompile( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments, @@ -199,9 +205,8 @@ open class IncrementalJvmCompilerRunner( //TODO can't use the same way as for build-history files because abi-snapshot for all dependencies should be stored into last-build // and not only changed one // (but possibly we dont need to read it all and may be it is possible to update only those who was changed) - override fun setupJarDependencies(args: K2JVMCompilerArguments, withSnapshot: Boolean, reporter: BuildReporter): Map { + override fun setupJarDependencies(args: K2JVMCompilerArguments, reporter: BuildReporter): Map { //fill abiSnapshots - if (!withSnapshot) return emptyMap() val abiSnapshots = HashMap() args.classpathAsList .filter { it.extension.equals("jar", ignoreCase = true) } @@ -209,7 +214,12 @@ open class IncrementalJvmCompilerRunner( modulesApiHistory.abiSnapshot(it).let { result -> if (result is Either.Success>) { result.value.forEach { file -> - AbiSnapshotImpl.read(file, reporter)?.also { abiSnapshot -> abiSnapshots[it.absolutePath] = abiSnapshot } + if (file.exists()) { + abiSnapshots[it.absolutePath] = AbiSnapshotImpl.read(file) + } else { + // FIXME: We should throw an exception here + reporter.warn { "Snapshot file does not exist: ${file.path}. Continue anyway." } + } } } } @@ -228,7 +238,7 @@ open class IncrementalJvmCompilerRunner( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments, - abiSnapshots: Map = HashMap(), + abiSnapshots: Map, withAbiSnapshot: Boolean ): CompilationMode { val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) @@ -370,7 +380,9 @@ open class IncrementalJvmCompilerRunner( return result } - override fun preBuildHook(args: K2JVMCompilerArguments, compilationMode: CompilationMode) { + override fun performWorkBeforeCompilation(compilationMode: CompilationMode, args: K2JVMCompilerArguments) { + super.performWorkBeforeCompilation(compilationMode, args) + if (compilationMode is CompilationMode.Incremental) { val destinationDir = args.destinationAsFile destinationDir.mkdirs() @@ -478,12 +490,15 @@ open class IncrementalJvmCompilerRunner( return exitCode to sourcesToCompile } - override fun performWorkAfterSuccessfulCompilation(caches: IncrementalJvmCachesManager, wasIncremental: Boolean) { - if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) { + override fun performWorkAfterCompilation(compilationMode: CompilationMode, exitCode: ExitCode, caches: IncrementalJvmCachesManager) { + super.performWorkAfterCompilation(compilationMode, exitCode, caches) + + // No need to shrink and save classpath snapshot if exitCode != ExitCode.OK as the task will fail anyway + if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled && exitCode == ExitCode.OK) { reporter.measure(BuildTime.SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { shrinkAndSaveClasspathSnapshot( - wasIncremental, classpathChanges, caches.lookupCache, currentClasspathSnapshot, - shrunkCurrentClasspathAgainstPreviousLookups, ClasspathSnapshotBuildReporter(reporter) + compilationWasIncremental = compilationMode is CompilationMode.Incremental, classpathChanges, caches.lookupCache, + currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups, ClasspathSnapshotBuildReporter(reporter) ) } } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt index bb6125f6573..8a3c2443802 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt @@ -213,7 +213,8 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest? = null) { - assertOutputContains(PERFORMING_INCREMENTAL_COMPILATION) - // Also check that incremental compilation did not fail and fall back to non-incremental compilation + assertOutputContains(INCREMENTAL_COMPILATION_COMPLETED) + + // Also check that the other cases didn't happen assertOutputDoesNotContain(NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED) + assertOutputDoesNotContain(FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION) expectedCompiledKotlinFiles?.let { assertSameFiles(expected = it, actual = extractCompiledKotlinFiles(output), "Compiled Kotlin files differ:\n") @@ -122,9 +126,19 @@ fun BuildResult.assertIncrementalCompilation(expectedCompiledKotlinFiles: Iterab * * Note: Log level of output must be set to [LogLevel.DEBUG]. */ -fun BuildResult.assertIncrementalCompilationFellBackToNonIncremental() { - assertOutputContains("$NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED: ${BuildAttribute.INCREMENTAL_COMPILATION_FAILED.name}") +fun BuildResult.assertIncrementalCompilationFellBackToNonIncremental(reason: BuildAttribute? = null) { + if (reason != null) { + assertOutputContains("$FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION (reason = ${reason.name})") + } else { + assertOutputContains(FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION) + } + + // Also check that the other cases didn't happen + assertOutputDoesNotContain(INCREMENTAL_COMPILATION_COMPLETED) + assertOutputDoesNotContain(NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED) } -private const val PERFORMING_INCREMENTAL_COMPILATION = "Performing incremental compilation" +// Each of the following messages should uniquely correspond to a case in `IncrementalCompilerRunner.ICResult` +private const val INCREMENTAL_COMPILATION_COMPLETED = "Incremental compilation completed" const val NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED = "Non-incremental compilation will be performed" +private const val FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION = "Falling back to non-incremental compilation" diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt index 44b0fd7f779..0b20e4bf331 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.compilerRunner -import org.gradle.api.GradleException import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.FileSystemOperations import org.gradle.api.logging.Logging @@ -18,9 +17,7 @@ import org.gradle.workers.WorkerExecutor import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.build.report.metrics.measure -import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider -import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy -import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup +import org.jetbrains.kotlin.gradle.tasks.* import java.io.File import javax.inject.Inject @@ -43,7 +40,7 @@ internal class GradleCompilerRunnerWithWorkers( workQueue.submit(GradleKotlinCompilerWorkAction::class.java) { params -> params.compilerWorkArguments.set(workArgs) if (taskOutputsBackup != null) { - params.taskOutputs.set(taskOutputsBackup.outputs) + params.taskOutputsToRestore.set(taskOutputsBackup.outputsToRestore) params.buildDir.set(taskOutputsBackup.buildDirectory) params.snapshotsDir.set(taskOutputsBackup.snapshotsDir) params.metricsReporter.set(buildMetrics) @@ -64,8 +61,7 @@ internal class GradleCompilerRunnerWithWorkers( fileSystemOperations, parameters.buildDir, parameters.snapshotsDir, - parameters.taskOutputs.get(), - outputsToExclude = emptyList(), + parameters.taskOutputsToRestore.get(), logger ) } else { @@ -76,11 +72,14 @@ internal class GradleCompilerRunnerWithWorkers( GradleKotlinCompilerWork( parameters.compilerWorkArguments.get() ).run() - } catch (e: GradleException) { - // Currently, metrics are not reported as in the worker we are getting new instance of [BuildMetricsReporter] - // [BuildDataRecorder] knows nothing about this new instance. Possibly could be fixed in the future by migrating - // [BuildMetricsReporter] to be shared Gradle service. - if (taskOutputsBackup != null) { + } catch (e: FailedCompilationException) { + // Restore outputs only in cases where we expect that the user will make some changes to their project: + // - For a compilation error, the user will need to fix their source code + // - For an OOM error, the user will need to increase their memory settings + // In the other cases where there is nothing the user can fix in their project, we should not restore the outputs. + // Otherwise, the next build(s) will likely fail in exactly the same way as this build because their inputs and outputs are + // the same. + if (taskOutputsBackup != null && (e is CompilationErrorException || e is OOMErrorException)) { parameters.metricsReporter.get().measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) { logger.info("Restoring task outputs to pre-compilation state") taskOutputsBackup.restoreOutputs() @@ -96,7 +95,7 @@ internal class GradleCompilerRunnerWithWorkers( internal interface GradleKotlinCompilerWorkParameters : WorkParameters { val compilerWorkArguments: Property - val taskOutputs: ListProperty + val taskOutputsToRestore: ListProperty val snapshotsDir: DirectoryProperty val buildDir: DirectoryProperty val metricsReporter: Property diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt index 50d7f1b6360..e7d129764ff 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.compilerRunner -import org.gradle.api.GradleException import org.gradle.api.Project import org.gradle.api.invocation.Gradle import org.gradle.api.logging.Logger @@ -33,11 +32,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.plugin.variantImplementationFactory import org.jetbrains.kotlin.gradle.tasks.* -import org.jetbrains.kotlin.gradle.utils.IsolatedKotlinClasspathClassCastException -import org.jetbrains.kotlin.gradle.utils.archivePathCompatible -import org.jetbrains.kotlin.gradle.utils.findByType -import org.jetbrains.kotlin.gradle.utils.newTmpFile -import org.jetbrains.kotlin.gradle.utils.relativeOrCanonical +import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.incremental.IncrementalModuleEntry import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics @@ -223,8 +218,9 @@ internal open class GradleCompilerRunner( try { val kotlinCompilerRunnable = GradleKotlinCompilerWork(workArgs) kotlinCompilerRunnable.run() - } catch (e: GradleException) { - if (taskOutputsBackup != null) { + } catch (e: FailedCompilationException) { + // Restore outputs only for CompilationErrorException or OOMErrorException (see GradleKotlinCompilerWorkAction.execute) + if (taskOutputsBackup != null && (e is CompilationErrorException || e is OOMErrorException)) { buildMetrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) { taskOutputsBackup.restoreOutputs() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 729ff0cebc6..34559cbde3f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -17,7 +17,7 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.report.* import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState -import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError +import org.jetbrains.kotlin.gradle.tasks.throwExceptionIfCompilationFailed import org.jetbrains.kotlin.gradle.utils.stackTraceAsString import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.ClasspathChanges @@ -125,7 +125,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete() } - throwGradleExceptionIfError(exitCode, executionStrategy) + throwExceptionIfCompilationFailed(exitCode, executionStrategy) } finally { val taskInfo = TaskExecutionInfo( changedFiles = incrementalCompilationEnvironment?.changedFiles, diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt index e387e3182ea..ece7fb7d2ac 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt @@ -424,9 +424,7 @@ abstract class BuildReportsService : BuildService value != 0L } ?: emptyMap() val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) { is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } - is ChangedFiles.Dependencies -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath } else -> emptyList() - } return CompileStatisticsData( durationMs = durationMs, diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinJsDce.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinJsDce.kt index cf72828c25a..7d5c76510ad 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinJsDce.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinJsDce.kt @@ -141,7 +141,7 @@ abstract class KotlinJsDce @Inject constructor( buildDir.get().asFile, jvmArgs ) - throwGradleExceptionIfError(exitCode, KotlinCompilerExecutionStrategy.OUT_OF_PROCESS) + throwExceptionIfCompilationFailed(exitCode, KotlinCompilerExecutionStrategy.OUT_OF_PROCESS) } private fun isDceCandidate(file: File): Boolean { diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index b58d91dcf7c..1e6480743c6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -363,7 +363,7 @@ abstract class AbstractKotlinCompile @Inject constr private val systemPropertiesService = CompilerSystemPropertiesService.registerIfAbsent(project.gradle) - /** Task outputs that we don't want to include in [TaskOutputsBackup] (see [TaskOutputsBackup]'s kdoc for more info). */ + /** Task outputs that we don't want to include in [TaskOutputsBackup] (see [TaskOutputsBackup.outputsToRestore] for more info). */ @get:Internal protected open val taskOutputsBackupExcludes: List = emptyList() @@ -391,8 +391,7 @@ abstract class AbstractKotlinCompile @Inject constr fileSystemOperations, layout.buildDirectory, layout.buildDirectory.dir("snapshot/kotlin/$name"), - allOutputFiles(), - taskOutputsBackupExcludes, + outputsToRestore = allOutputFiles() - taskOutputsBackupExcludes, logger ).also { it.createSnapshot() @@ -596,7 +595,6 @@ abstract class KotlinCompile @Inject constructor( classpathSnapshotProperties.classpathSnapshot ) - // Exclude classpathSnapshotDir from TaskOutputsBackup (see TaskOutputsBackup's kdoc for more info). */ override val taskOutputsBackupExcludes: List get() = classpathSnapshotProperties.classpathSnapshotDir.orNull?.asFile?.let { listOf(it) } ?: emptyList() diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksOutputsBackup.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksOutputsBackup.kt index 477fc06300f..f374e8816ce 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksOutputsBackup.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksOutputsBackup.kt @@ -5,7 +5,9 @@ package org.jetbrains.kotlin.gradle.tasks -import org.gradle.api.file.* +import org.gradle.api.file.Directory +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.FileSystemOperations import org.gradle.api.logging.Logger import org.gradle.api.provider.Provider import java.io.File @@ -14,35 +16,31 @@ import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption -import java.util.zip.* +import java.util.zip.Deflater +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream internal class TaskOutputsBackup( - val fileSystemOperations: FileSystemOperations, + private val fileSystemOperations: FileSystemOperations, val buildDirectory: DirectoryProperty, val snapshotsDir: Provider, - allOutputs: List, - /** - * Task outputs that we don't want to back up for performance reasons (e.g., if (1) they are too big, and (2) they are usually updated - * only at the end of the task execution--in a failed task run, they are usually unchanged and therefore don't need to be restored). + * Task outputs to back up and restore. * - * NOTE: In `IncrementalCompilerRunner`, if incremental compilation fails, it will try again by cleaning all the outputs and perform - * non-incremental compilation. It is important that `IncrementalCompilerRunner` do not clean [outputsToExclude] immediately but only - * right before [outputsToExclude] are updated (which is usually at the end of the task execution). This is so that if the fallback - * compilation fails, [outputsToExclude] will remain unchanged and the other outputs will be restored, and the next task run can be - * incremental. + * Note that this could be a subset of all the outputs of a task because there could be task outputs that we don't want to back up and + * restore (e.g., if (1) they are too big and (2) they are updated only at the end of the task execution so in a failed task run, they + * are usually unchanged and therefore don't need to be restored). */ - outputsToExclude: List = emptyList(), + val outputsToRestore: List, + val logger: Logger ) { - /** The outputs to back up and restore. Note that this may be a subset of all the outputs of a task (see `outputsToExclude`). */ - val outputs: List = allOutputs - outputsToExclude.toSet() fun createSnapshot() { // Kotlin JS compilation task declares one file from 'destinationDirectory' output as task `@OutputFile' // property. To avoid snapshot sync collisions, each snapshot output directory has also 'index' as prefix. - outputs.toSortedSet().forEachIndexed { index, outputPath -> + outputsToRestore.toSortedSet().forEachIndexed { index, outputPath -> if (outputPath.isDirectory && !outputPath.isEmptyDirectory) { compressDirectoryToZip( File(snapshotsDir.get().asFile, index.asSnapshotArchiveName), @@ -59,10 +57,10 @@ internal class TaskOutputsBackup( fun restoreOutputs() { fileSystemOperations.delete { - it.delete(outputs) + it.delete(outputsToRestore) } - outputs.toSortedSet().forEachIndexed { index, outputPath -> + outputsToRestore.toSortedSet().forEachIndexed { index, outputPath -> val snapshotDir = snapshotsDir.get().file(index.asSnapshotDirectoryName).asFile if (snapshotDir.isDirectory) { fileSystemOperations.copy { spec -> diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt index 38770c2e22c..a48998bb80e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt @@ -1,6 +1,5 @@ package org.jetbrains.kotlin.gradle.tasks -import org.gradle.api.GradleException import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.build.report.metrics.measure @@ -10,18 +9,19 @@ import org.jetbrains.kotlin.gradle.internal.tasks.TaskWithLocalState import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger import org.jetbrains.kotlin.gradle.logging.kotlinDebug -import org.jetbrains.kotlin.incremental.cleanDirectoryContents +import org.jetbrains.kotlin.incremental.deleteDirectoryContents import org.jetbrains.kotlin.incremental.deleteRecursivelyOrThrow import java.io.File -fun throwGradleExceptionIfError( +/** Throws [FailedCompilationException] if compilation completed with [exitCode] != [ExitCode.OK]. */ +fun throwExceptionIfCompilationFailed( exitCode: ExitCode, executionStrategy: KotlinCompilerExecutionStrategy ) { when (exitCode) { - ExitCode.COMPILATION_ERROR -> throw GradleException("Compilation error. See log for more details") - ExitCode.INTERNAL_ERROR -> throw GradleException("Internal compiler error. See log for more details") - ExitCode.SCRIPT_EXECUTION_ERROR -> throw GradleException("Script execution error. See log for more details") + ExitCode.COMPILATION_ERROR -> throw CompilationErrorException("Compilation error. See log for more details") + ExitCode.INTERNAL_ERROR -> throw FailedCompilationException("Internal compiler error. See log for more details") + ExitCode.SCRIPT_EXECUTION_ERROR -> throw FailedCompilationException("Script execution error. See log for more details") ExitCode.OOM_ERROR -> { var exceptionMessage = "Not enough memory to run compilation." when (executionStrategy) { @@ -31,13 +31,22 @@ fun throwGradleExceptionIfError( exceptionMessage += " Try to increase it via 'gradle.properties':\norg.gradle.jvmargs=-Xmx" KotlinCompilerExecutionStrategy.OUT_OF_PROCESS -> Unit } - throw GradleException(exceptionMessage) + throw OOMErrorException(exceptionMessage) } ExitCode.OK -> Unit else -> throw IllegalStateException("Unexpected exit code: $exitCode") } } +/** Exception thrown when [ExitCode] != [ExitCode.OK]. */ +internal open class FailedCompilationException(message: String) : RuntimeException(message) + +/** Exception thrown when [ExitCode] == [ExitCode.COMPILATION_ERROR]. */ +internal class CompilationErrorException(message: String) : FailedCompilationException(message) + +/** Exception thrown when [ExitCode] == [ExitCode.OOM_ERROR]. */ +internal class OOMErrorException(message: String) : FailedCompilationException(message) + internal fun TaskWithLocalState.cleanOutputsAndLocalState(reason: String? = null) { val log = GradleKotlinLogger(logger) cleanOutputsAndLocalState(allOutputFiles(), log, metrics.get(), reason) @@ -59,7 +68,7 @@ internal fun cleanOutputsAndLocalState( when { file.isDirectory -> { log.debug("Deleting contents of output directory: $file") - file.cleanDirectoryContents() + file.deleteDirectoryContents() } file.isFile -> { log.debug("Deleting output file: $file")