azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up fall-back logic in IncrementalCompilerRunner

Browse Source 
Make it clear that there 3 distinct cases:
   1. Incremental compilation completed with an ExitCode.
   2. Incremental compilation was not possible for some valid reason
      (e.g., for a clean build), and we will perform non-incremental
      compilation.
   3. Incremental compilation failed with an exception.
      In this case, we will:
        - Print a warning with a stack trace
        - Ask the user to file a bug
        - Collect rebuild reason enum for analytics
           + TODO: Collect the stack trace too
        - Fall back to non-incremental compilation

Test: Existing BaseIncrementalCompilationMultiProjectIT.testFailureHandling_UserError,
      Updated BaseIncrementalCompilationMultiProjectIT.testFailureHandling_ToolError

^KT-53015: In progress
This commit is contained in:
Hung Nguyen
2022-06-20 13:41:40 +01:00
committed by Andrey Uskov
parent 57bbc335f4
commit def886cd31
24 changed files with 409 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt
+4 -2
View File
@@ -18,9 +18,11 @@ enum class BuildAttributeKind : Serializable {
enum class BuildAttribute(val kind: BuildAttributeKind, val readableString: String) : Serializable {
NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON, "Build history file not found"),
NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON, "ABI snapshot not found"),
INTERNAL_ERROR(BuildAttributeKind.REBUILD_REASON, "Internal error during preparation of IC round"),
CLASSPATH_SNAPSHOT_NOT_FOUND(BuildAttributeKind.REBUILD_REASON, "Classpath snapshot not found"),
INCREMENTAL_COMPILATION_FAILED(BuildAttributeKind.REBUILD_REASON, "Incremental compilation failed"),
IC_FAILED_TO_GET_CHANGED_FILES(BuildAttributeKind.REBUILD_REASON, "Failed to get changed files"),
IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE(BuildAttributeKind.REBUILD_REASON, "Failed to compute files to recompile"),
IC_FAILED_TO_COMPILE_INCREMENTALLY(BuildAttributeKind.REBUILD_REASON, "Failed to compile incrementally"),
IC_FAILED_TO_CLOSE_CACHES(BuildAttributeKind.REBUILD_REASON, "Failed to close caches"),
UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON, "Unknown Gradle changes"),
JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON, "Untracked Java file is removed"),
JAVA_CHANGE_UNEXPECTED_PSI(BuildAttributeKind.REBUILD_REASON, "Java PSI file is expected"),
build-common/src/org/jetbrains/kotlin/incremental/fileUtils.kt
+9 -8
View File
@@ -29,15 +29,15 @@ fun File.isClassFile(): Boolean =
extension.equals("class", ignoreCase = true)
/**
* Deletes the contents of this directory (not the directory itself) if it exists, or creates the directory if it does not yet exist.
* Deletes the contents of this directory (not the directory itself).
*
* If this is a regular file, this method will throw an exception.
* If the directory does not exist or if this is a regular file, this method will throw an exception.
*/
fun File.cleanDirectoryContents() {
fun File.deleteDirectoryContents() {
when {
isDirectory -> listFiles()!!.forEach { it.deleteRecursivelyOrThrow() }
isFile -> error("File.cleanDirectoryContents does not accept a regular file: $path")
else -> mkdirsOrThrow()
isFile -> error("Expected a directory but found a regular file: $path")
else -> error("Directory does not exist: $path")
}
}
@@ -49,11 +49,12 @@ fun File.deleteRecursivelyOrThrow() {
}
/**
* Creates this directory (if it does not yet exist), throwing an exception if the directiory creation failed or if a regular file already
* exists at this path.
* Creates this directory (if it does not yet exist).
*
* If a regular file already exists at this path, this method will throw an exception.
*/
@Suppress("SpellCheckingInspection")
fun File.mkdirsOrThrow() {
fun File.createDirectory() {
when {
isDirectory -> Unit
isFile -> error("A regular file already exists at this path: $path")
compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt
+14 -5
View File
@@ -16,7 +16,10 @@
package org.jetbrains.kotlin.daemon.common
import org.jetbrains.kotlin.cli.common.repl.*
import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult
import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine
import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult
import org.jetbrains.kotlin.cli.common.repl.ReplEvalResult
import java.io.File
import java.io.Serializable
import java.rmi.Remote
@@ -46,20 +49,26 @@ interface CompileService : Remote {
override fun equals(other: Any?): Boolean = other is Good<*> && this.result == other.result
override fun hashCode(): Int = this::class.java.hashCode() + (result?.hashCode() ?: 1)
}
class Ok : CallResult<Nothing>() {
override fun get(): Nothing = throw IllegalStateException("Get is inapplicable to Ok call result")
override fun equals(other: Any?): Boolean = other is Ok
override fun hashCode(): Int = this::class.java.hashCode() + 1 // avoiding clash with the hash of class itself
}
class Dying : CallResult<Nothing>() {
override fun get(): Nothing = throw IllegalStateException("Service is dying")
override fun equals(other: Any?): Boolean = other is Dying
override fun hashCode(): Int = this::class.java.hashCode() + 1 // see comment to Ok.hashCode
}
class Error(val message: String) : CallResult<Nothing>() {
override fun get(): Nothing = throw Exception(message)
override fun equals(other: Any?): Boolean = other is Error && this.message == other.message
override fun hashCode(): Int = this::class.java.hashCode() + message.hashCode()
class Error(val message: String?, val cause: Throwable?) : CallResult<Nothing>() {
constructor(cause: Throwable) : this(message = null, cause = cause)
constructor(message: String) : this(message = message, cause = null)
override fun get(): Nothing = throw Exception(message, cause)
override fun equals(other: Any?): Boolean = other is Error && this.message == other.message && this.cause == other.cause
override fun hashCode(): Int = this::class.java.hashCode() + (cause?.hashCode() ?: 1) + (message?.hashCode() ?: 2) // see comment to Ok.hashCode
}
val isGood: Boolean get() = this is Good<*>
compiler/daemon/daemon-tests/test/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt
+1 -1
View File
@@ -781,7 +781,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
} catch (e: Exception) {
TestCase.assertEquals(
"Unable to use scripting/REPL in the daemon: no scripting plugin loaded",
e.message
e.cause?.message
)
isErrorThrown = true
} finally {
compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt
+3 -3
View File
@@ -47,8 +47,8 @@ import org.jetbrains.kotlin.daemon.report.getBuildReporter
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.components.EnumWhenTracker
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.components.InlineConstTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryAndroid
@@ -507,7 +507,7 @@ abstract class CompileServiceImplBase(
body()
} catch (e: Throwable) {
log.log(Level.SEVERE, "Exception", e)
CompileService.CallResult.Error(e.message ?: "unknown")
CompileService.CallResult.Error(e)
}
}
}
@@ -614,7 +614,7 @@ abstract class CompileServiceImplBase(
workingDir,
reporter,
buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile,
outputFiles = incrementalCompilationOptions.outputFiles,
outputDirs = incrementalCompilationOptions.outputFiles,
usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
modulesApiHistory = modulesApiHistory,
kotlinSourceFilesExtensions = allKotlinExtensions,
compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt
+1 -2
View File
@@ -23,7 +23,6 @@ import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult
import org.jetbrains.kotlin.cli.js.K2JSCompiler
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.cli.jvm.compiler.jarfs.FastJarFileSystem
import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.config.Services
@@ -749,7 +748,7 @@ class CompileServiceServerSideImpl(
body()
} catch (e: Throwable) {
log.log(Level.SEVERE, "Exception", e)
CompileService.CallResult.Error(e.message ?: "unknown")
CompileService.CallResult.Error(e)
}
}
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt
+3 -10
View File
@@ -5,8 +5,6 @@
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.info
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmNameResolver
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
@@ -152,14 +150,9 @@ class AbiSnapshotImpl(override val protos: MutableMap<FqName, ProtoData>) : AbiS
}
}
fun read(file: File, reporter: BuildReporter): AbiSnapshot? {
if (!file.exists()) {
reporter.info { "jar snapshot $file is found for jar" }
return null
}
ObjectInputStream(FileInputStream(file)).use {
return it.readAbiSnapshot()
fun read(file: File): AbiSnapshot {
return ObjectInputStream(FileInputStream(file)).use {
it.readAbiSnapshot()
}
}
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt
+1 -3
View File
@@ -20,10 +20,8 @@ import java.io.File
import java.io.Serializable
sealed class ChangedFiles : Serializable {
class Known(val modified: List<File>, val removed: List<File>) : ChangedFiles()
class Known(val modified: List<File>, val removed: List<File>, val forDependencies: Boolean = false) : ChangedFiles()
class Unknown : ChangedFiles()
class Dependencies(val modified: List<File>, val removed: List<File>) : ChangedFiles()
companion object {
const val serialVersionUID: Long = 0
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt
+22 -27
View File
@@ -16,30 +16,29 @@
package org.jetbrains.kotlin.incremental
import com.google.common.io.Closer
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.info
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
import java.io.Closeable
import java.io.File
abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache<*>>(
cachesRootDir: File,
rootProjectDir: File?,
protected val reporter: ICReporter,
storeFullFqNamesInLookupCache: Boolean = false,
trackChangesInLookupCache: Boolean = false
) {
) : Closeable {
val pathConverter = IncrementalFileToPathConverter(rootProjectDir)
private val caches = arrayListOf<BasicMapsOwner>()
var isClosed = false
var isSuccessfulyClosed = false
private var isClosed = false
@Synchronized
protected fun <T : BasicMapsOwner> T.registerCache() {
assert(!isClosed) { "Attempted to add new cache into closed storage." }
check(!isClosed) { "This cache storage has already been closed" }
caches.add(this)
}
@@ -51,33 +50,29 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache, trackChangesInLookupCache).apply { registerCache() }
abstract val platformCache: PlatformCache
@Suppress("UnstableApiUsage")
@Synchronized
fun close(flush: Boolean = false): Boolean {
if (isClosed) {
return isSuccessfulyClosed
}
isSuccessfulyClosed = true
for (cache in caches) {
if (flush) {
try {
cache.flush(false)
} catch (e: Throwable) {
isSuccessfulyClosed = false
reporter.info { "Exception when flushing cache ${cache.javaClass}: $e" }
}
}
override fun close() {
check(!isClosed) { "This cache storage has already been closed" }
try {
cache.close()
} catch (e: Throwable) {
isSuccessfulyClosed = false
reporter.info { "Exception when closing cache ${cache.javaClass}: $e" }
}
val closer = Closer.create()
caches.forEach {
closer.register(CacheCloser(it))
}
closer.close()
isClosed = true
return isSuccessfulyClosed
}
private class CacheCloser(private val cache: BasicMapsOwner) : Closeable {
override fun close() {
// It's important to flush the cache when closing (see KT-53168)
cache.flush(memoryCachesOnly = false)
cache.close()
}
}
}
class IncrementalJvmCachesManager(
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt
+239 -198
View File
@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.debug
import org.jetbrains.kotlin.build.report.info
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute.*
import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
@@ -40,7 +41,7 @@ import org.jetbrains.kotlin.incremental.parsing.classesFqNames
import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
import org.jetbrains.kotlin.util.suffixIfNot
import org.jetbrains.kotlin.util.removeSuffixIfPresent
import java.io.File
abstract class IncrementalCompilerRunner<
@@ -51,14 +52,24 @@ abstract class IncrementalCompilerRunner<
cacheDirName: String,
protected val reporter: BuildReporter,
protected val buildHistoryFile: File,
// there might be some additional output directories (e.g. for generated java in kapt)
// to remove them correctly on rebuild, we pass them as additional argument
private val additionalOutputFiles: Collection<File> = emptyList(),
/**
* Output directories of the compilation. These include:
* 1. The classes output directory
* 2. [workingDir]
* 3. Any additional output directories (e.g., classpath snapshot directory or Kapt generated-stubs directory)
*
* We will clean these directories when compiling non-incrementally.
*
* If this property is not set, the directories to clean will include the first 2 directories above.
*/
private val outputDirs: Collection<File>?,
protected val withAbiSnapshot: Boolean = false
) {
protected val cacheDirectory = File(workingDir, cacheDirName)
protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
@@ -75,184 +86,204 @@ abstract class IncrementalCompilerRunner<
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION_DAEMON) {
try {
compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir)
} finally {
reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) {
reporter.addMetric(
BuildPerformanceMetric.SNAPSHOT_SIZE,
buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length()
return when (val result = tryCompileIncrementally(allSourceFiles, providedChangedFiles, args, projectDir, messageCollector)) {
is ICResult.Completed -> {
reporter.debug { "Incremental compilation completed" }
result.exitCode
}
is ICResult.RequiresRebuild -> {
reporter.info { "Non-incremental compilation will be performed: ${result.reason}" }
reporter.addAttribute(result.reason)
compileNonIncrementally(
result.reason, allSourceFiles, args, projectDir, trackChangedFiles = providedChangedFiles == null, messageCollector
)
if (cacheDirectory.exists() && cacheDirectory.isDirectory()) {
cacheDirectory.walkTopDown().filter { it.isFile }.map { it.length() }.sum().let {
reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, it)
}
}
is ICResult.Failed -> {
reporter.warn {
// The indentation after the first line is intentional (so that this message is distinct from next message)
"""
|Incremental compilation was attempted but failed:
| ${result.reason.readableString}: ${result.cause.stackTraceToString().removeSuffixIfPresent("\n")}
| Falling back to non-incremental compilation (reason = ${result.reason})
| To help us fix this issue, please file a bug at https://youtrack.jetbrains.com/issues/KT with the above stack trace.
| (Be sure to search for the above exception in existing issues first to avoid filing duplicated bugs.)
""".trimMargin()
}
// TODO: Collect the stack trace too
reporter.addAttribute(result.reason)
compileNonIncrementally(
result.reason, allSourceFiles, args, projectDir, trackChangedFiles = providedChangedFiles == null, messageCollector
)
}
}
}
fun rebuild(
reason: BuildAttribute,
allSourceFiles: List<File>,
args: Args,
messageCollector: MessageCollector,
providedChangedFiles: ChangedFiles?,
projectDir: File? = null,
classpathAbiSnapshot: Map<String, AbiSnapshot>
): ExitCode {
reporter.info { "Non-incremental compilation will be performed: $reason" }
reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
cleanOutputsAndLocalStateOnRebuild(args)
}
val caches = createCacheManager(args, projectDir)
try {
if (providedChangedFiles == null) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
}
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
return compileIncrementally(
args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withAbiSnapshot,
classpathAbiSnapshot = classpathAbiSnapshot
).also {
if (it == ExitCode.OK) {
performWorkAfterSuccessfulCompilation(caches, wasIncremental = false)
}
}
} finally {
caches.close(true)
}
}
/** The result when attempting to compile incrementally ([tryCompileIncrementally]). */
private sealed interface ICResult {
private fun compileImpl(
allSourceFiles: List<File>,
args: Args,
messageCollector: MessageCollector,
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode {
var caches = createCacheManager(args, projectDir)
var rebuildReason = BuildAttribute.INTERNAL_ERROR
/** Incremental compilation completed with an [ExitCode]. */
class Completed(val exitCode: ExitCode) : ICResult
val classpathAbiSnapshot =
if (withAbiSnapshot) {
reporter.info { "Incremental compilation with ABI snapshot enabled" }
reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) {
setupJarDependencies(args, withAbiSnapshot, reporter)
}
} else {
emptyMap()
}
/** Incremental compilation was not possible for some valid reason (e.g., for a clean build). */
class RequiresRebuild(val reason: BuildAttribute) : ICResult
try {
val changedFiles = when (providedChangedFiles) {
is ChangedFiles.Dependencies -> {
val changedSources = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
ChangedFiles.Known(
providedChangedFiles.modified + changedSources.modified,
providedChangedFiles.removed + changedSources.removed
)
}
null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
else -> providedChangedFiles
}
var compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot)
val abiSnapshot = if (compilationMode is CompilationMode.Incremental && withAbiSnapshot) {
AbiSnapshotImpl.read(abiSnapshotFile, reporter)
} else {
if (withAbiSnapshot) {
compilationMode = CompilationMode.Rebuild(BuildAttribute.NO_ABI_SNAPSHOT)
}
null
}
when (compilationMode) {
is CompilationMode.Incremental -> {
try {
reporter.debug { "Performing incremental compilation" }
val exitCode = if (withAbiSnapshot) {
compileIncrementally(
args, caches, allSourceFiles, compilationMode, messageCollector,
withAbiSnapshot, abiSnapshot!!, classpathAbiSnapshot
)
} else {
compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector, withAbiSnapshot)
}
if (exitCode == ExitCode.OK) {
performWorkAfterSuccessfulCompilation(caches, wasIncremental = true)
}
return exitCode
} catch (e: Throwable) {
reporter.warn {
"Incremental compilation failed: ${e.stackTraceToString().suffixIfNot("\n")}" +
"Falling back to non-incremental compilation"
}
rebuildReason = BuildAttribute.INCREMENTAL_COMPILATION_FAILED
}
}
is CompilationMode.Rebuild -> rebuildReason = compilationMode.reason
}
} catch (e: Exception) {
reporter.warn {
"Incremental compilation analysis failed: ${e.stackTraceToString().suffixIfNot("\n")}" +
"Falling back to non-incremental compilation"
}
} finally {
if (!caches.close(flush = true)) {
reporter.info { "Unable to close IC caches. Cleaning internal state" }
cleanOutputsAndLocalStateOnRebuild(args)
}
}
return rebuild(rebuildReason, allSourceFiles, args, messageCollector, providedChangedFiles, projectDir, classpathAbiSnapshot)
/** Incremental compilation failed with an exception. */
class Failed(val reason: BuildAttribute, val cause: Throwable) : ICResult
}
/**
* Deletes output files and contents of output directories on rebuild, including `@LocalState` files/directories.
* Attempts to compile incrementally and returns either [ICResult.Completed], [ICResult.RequiresRebuild], or [ICResult.Failed].
*
* Note that parts of this function may still throw exceptions that are not caught and wrapped by [ICResult.Failed] because they are not
* meant to be caught.
*/
private fun tryCompileIncrementally(
allSourceFiles: List<File>,
providedChangedFiles: ChangedFiles?,
args: Args,
projectDir: File?,
messageCollector: MessageCollector
): ICResult {
if (providedChangedFiles is ChangedFiles.Unknown) {
return ICResult.RequiresRebuild(UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
}
providedChangedFiles as ChangedFiles.Known?
val caches = createCacheManager(args, projectDir)
val exitCode: ExitCode
try {
// Step 1: Get changed files
val changedFiles: ChangedFiles.Known = try {
getChangedFiles(providedChangedFiles, allSourceFiles, caches)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
}
val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
// Step 2: Compute files to recompile
val compilationMode = try {
reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
calculateSourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
}
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
}
if (compilationMode is CompilationMode.Rebuild) {
return ICResult.RequiresRebuild(compilationMode.reason)
}
val abiSnapshotData = if (withAbiSnapshot) {
if (!abiSnapshotFile.exists()) {
reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
}
reporter.info { "Incremental compilation with ABI snapshot enabled" }
AbiSnapshotData(
snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
classpathAbiSnapshot = classpathAbiSnapshot!!
)
} else null
// Step 3: Compile incrementally
exitCode = try {
compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
}
} catch (e: Throwable) {
// Because `caches` is a Closeable resource, it is good practice to close them in the event of an exception (in addition to
// closing them after a normal use).
try {
caches.close()
} catch (e2: Throwable) {
e.addSuppressed(e2)
}
throw e
}
try {
caches.close()
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_CLOSE_CACHES, e)
}
return ICResult.Completed(exitCode)
}
private fun compileNonIncrementally(
rebuildReason: BuildAttribute,
allSourceFiles: List<File>,
args: Args,
projectDir: File?,
trackChangedFiles: Boolean, // Whether we need to track changes to the source files or the build system already handles it
messageCollector: MessageCollector,
): ExitCode {
reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
val mainOutputDirs = setOf(destinationDir(args), workingDir)
val outputDirsToClean = outputDirs?.also {
check(it.containsAll(mainOutputDirs)) { "outputDirs is missing classesDir and workingDir: $it" }
} ?: mainOutputDirs
reporter.debug { "Cleaning output directories" }
cleanOrCreateDirectories(outputDirsToClean)
}
return createCacheManager(args, projectDir).use { caches ->
if (trackChangedFiles) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
}
val abiSnapshotData = if (withAbiSnapshot) {
AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
} else null
compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector)
}
}
private class AbiSnapshotData(val snapshot: AbiSnapshot, val classpathAbiSnapshot: Map<String, AbiSnapshot>)
private fun getClasspathAbiSnapshot(args: Args): Map<String, AbiSnapshot> {
return reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) {
setupJarDependencies(args, reporter)
}
}
/**
* Deletes the contents of the given directories (not the directories themselves).
*
* If the directories do not yet exist, they will be created.
*/
private fun cleanOutputsAndLocalStateOnRebuild(args: Args) {
// Use Set as additionalOutputFiles may already contain destinationDir and workingDir
val outputFiles = setOf(destinationDir(args), workingDir) + additionalOutputFiles
reporter.debug { "Cleaning outputs on rebuild" }
outputFiles.forEach {
private fun cleanOrCreateDirectories(outputDirs: Collection<File>) {
outputDirs.toSet().forEach {
when {
it.isDirectory -> {
reporter.debug { " Deleting contents of directory '${it.path}'" }
it.cleanDirectoryContents()
}
it.isFile -> {
reporter.debug { " Deleting file '${it.path}'" }
it.deleteRecursivelyOrThrow()
}
it.isDirectory -> it.deleteDirectoryContents()
it.isFile -> "Expected a directory but found a regular file: ${it.path}"
else -> it.createDirectory()
}
}
}
private fun sourcesToCompile(
caches: CacheManager,
changedFiles: ChangedFiles,
args: Args,
messageCollector: MessageCollector,
dependenciesAbiSnapshots: Map<String, AbiSnapshot>
): CompilationMode =
when (changedFiles) {
is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector, dependenciesAbiSnapshots)
is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)")
private fun getChangedFiles(
providedChangedFiles: ChangedFiles.Known?,
allSourceFiles: List<File>,
caches: CacheManager
): ChangedFiles.Known {
return when {
providedChangedFiles == null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
providedChangedFiles.forDependencies -> {
val moreChangedFiles = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
ChangedFiles.Known(
modified = providedChangedFiles.modified + moreChangedFiles.modified,
removed = providedChangedFiles.removed + moreChangedFiles.removed
)
}
else -> providedChangedFiles
}
}
private fun calculateSourcesToCompile(
caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector,
abiSnapshots: Map<String, AbiSnapshot>
): CompilationMode =
reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, abiSnapshots)
}
protected abstract fun calculateSourcesToCompileImpl(
protected abstract fun calculateSourcesToCompile(
caches: CacheManager,
changedFiles: ChangedFiles.Known,
args: Args,
@@ -260,7 +291,7 @@ abstract class IncrementalCompilerRunner<
classpathAbiSnapshots: Map<String, AbiSnapshot>
): CompilationMode
protected open fun setupJarDependencies(args: Args, withSnapshot: Boolean, reporter: BuildReporter): Map<String, AbiSnapshot> = mapOf()
protected open fun setupJarDependencies(args: Args, reporter: BuildReporter): Map<String, AbiSnapshot> = emptyMap()
protected fun initDirtyFiles(dirtyFiles: DirtyFilesContainer, changedFiles: ChangedFiles.Known) {
dirtyFiles.add(changedFiles.modified, "was modified since last time")
@@ -284,7 +315,6 @@ abstract class IncrementalCompilerRunner<
changesCollector: ChangesCollector
)
protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {}
protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List<GeneratedFile>, services: Services): Iterable<File> =
emptyList()
@@ -315,33 +345,53 @@ abstract class IncrementalCompilerRunner<
isIncremental: Boolean
): Pair<ExitCode, Collection<File>>
protected open fun compileIncrementally(
private fun compileImpl(
compilationMode: CompilationMode,
allSourceFiles: List<File>,
args: Args,
caches: CacheManager,
allKotlinSources: List<File>,
compilationMode: CompilationMode,
originalMessageCollector: MessageCollector,
withSnapshot: Boolean,
abiSnapshot: AbiSnapshot = AbiSnapshotImpl(mutableMapOf()),
classpathAbiSnapshot: Map<String, AbiSnapshot> = HashMap()
abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
messageCollector: MessageCollector
): ExitCode {
if (compilationMode is CompilationMode.Rebuild) {
reporter.info { "Non-incremental compilation will be performed: ${compilationMode.reason}" }
performWorkBeforeCompilation(compilationMode, args)
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector)
performWorkAfterCompilation(compilationMode, exitCode, caches)
return exitCode
}
protected open fun performWorkBeforeCompilation(compilationMode: CompilationMode, args: Args) {}
protected open fun performWorkAfterCompilation(compilationMode: CompilationMode, exitCode: ExitCode, caches: CacheManager) {
collectMetrics()
}
private fun collectMetrics() {
reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) {
reporter.addMetric(
BuildPerformanceMetric.SNAPSHOT_SIZE,
buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length()
)
reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, cacheDirectory.walk().sumOf { it.length() })
}
}
preBuildHook(args, compilationMode)
private fun doCompile(
compilationMode: CompilationMode,
allKotlinSources: List<File>,
args: Args,
caches: CacheManager,
abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
originalMessageCollector: MessageCollector
): ExitCode {
val dirtySources = when (compilationMode) {
is CompilationMode.Incremental -> {
compilationMode.dirtyFiles.toMutableLinkedSet()
}
is CompilationMode.Rebuild -> {
reporter.addAttribute(compilationMode.reason)
LinkedHashSet(allKotlinSources)
}
is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableLinkedSet()
is CompilationMode.Rebuild -> LinkedHashSet(allKotlinSources)
}
val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), classpathAbiSnapshot)
val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), abiSnapshotData?.classpathAbiSnapshot ?: emptyMap())
val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
val buildDirtyFqNames = HashSet<FqName>()
val allDirtySources = HashSet<File>()
@@ -412,8 +462,8 @@ abstract class IncrementalCompilerRunner<
updateCaches(services, caches, generatedFiles, changesCollector)
}
if (compilationMode is CompilationMode.Rebuild) {
if (withSnapshot) {
abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
if (withAbiSnapshot) {
abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges())
}
break
}
@@ -445,10 +495,10 @@ abstract class IncrementalCompilerRunner<
buildDirtyFqNames.addAll(dirtyClassFqNames)
//update
if (withSnapshot) {
if (withAbiSnapshot) {
//TODO(valtman) check method/ kts class remove
changesCollector.protoDataRemoved().forEach { abiSnapshot.protos.remove(it) }
abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
changesCollector.protoDataRemoved().forEach { abiSnapshotData!!.snapshot.protos.remove(it) }
abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges())
}
}
@@ -457,9 +507,9 @@ abstract class IncrementalCompilerRunner<
BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
//write abi snapshot
if (withSnapshot) {
if (withAbiSnapshot) {
//TODO(valtman) check method/class remove
AbiSnapshotImpl.write(abiSnapshot, abiSnapshotFile)
AbiSnapshotImpl.write(abiSnapshotData!!.snapshot, abiSnapshotFile)
}
}
}
@@ -516,15 +566,6 @@ abstract class IncrementalCompilerRunner<
BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
}
/**
* Performs some work after a compilation if the compilation completed successfully (no exceptions were thrown AND exit code == 0).
*
* This method MUST NOT be called when the compilation failed because the results produced by the work here would be invalid.
*
* @wasIncremental whether the compilation was incremental or non-incremental
*/
protected open fun performWorkAfterSuccessfulCompilation(caches: CacheManager, wasIncremental: Boolean) {}
companion object {
const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt
+2 -2
View File
@@ -72,7 +72,7 @@ class IncrementalFirJvmCompilerRunner(
workingDir: File,
reporter: BuildReporter,
buildHistoryFile: File,
outputFiles: Collection<File>,
outputDirs: Collection<File>?,
modulesApiHistory: ModulesApiHistory,
kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
classpathChanges: ClasspathChanges
@@ -81,7 +81,7 @@ class IncrementalFirJvmCompilerRunner(
reporter,
false,
buildHistoryFile,
outputFiles,
outputDirs,
modulesApiHistory,
kotlinSourceFilesExtensions,
classpathChanges
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt
+2 -1
View File
@@ -93,6 +93,7 @@ class IncrementalJsCompilerRunner(
"caches-js",
reporter,
buildHistoryFile = buildHistoryFile,
outputDirs = null,
withAbiSnapshot = withAbiSnapshot
) {
@@ -115,7 +116,7 @@ class IncrementalJsCompilerRunner(
outputFile.parentFile
}
override fun calculateSourcesToCompileImpl(
override fun calculateSourcesToCompile(
caches: IncrementalJsCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JSCompilerArguments,
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt
+29 -14
View File
@@ -88,7 +88,13 @@ fun makeIncrementally(
val compiler =
if (args.useK2 && args.useFirIC && args.useFirLT /* TODO: move LT check into runner */ )
IncrementalFirJvmCompilerRunner(
cachesDir, buildReporter, buildHistoryFile, emptyList(), EmptyModulesApiHistory, kotlinExtensions, ClasspathSnapshotDisabled
cachesDir,
buildReporter,
buildHistoryFile,
outputDirs = null,
EmptyModulesApiHistory,
kotlinExtensions,
ClasspathSnapshotDisabled
)
else
IncrementalJvmCompilerRunner(
@@ -96,8 +102,8 @@ fun makeIncrementally(
buildReporter,
// Use precise setting in case of non-Gradle build
usePreciseJavaTracking = !args.useK2, // TODO: add fir-based java classes tracker when available and set this to true
outputFiles = emptyList(),
buildHistoryFile = buildHistoryFile,
outputDirs = null,
modulesApiHistory = EmptyModulesApiHistory,
kotlinSourceFilesExtensions = kotlinExtensions,
classpathChanges = ClasspathSnapshotDisabled
@@ -127,7 +133,7 @@ open class IncrementalJvmCompilerRunner(
reporter: BuildReporter,
private val usePreciseJavaTracking: Boolean,
buildHistoryFile: File,
outputFiles: Collection<File>,
outputDirs: Collection<File>?,
private val modulesApiHistory: ModulesApiHistory,
override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
private val classpathChanges: ClasspathChanges,
@@ -136,8 +142,8 @@ open class IncrementalJvmCompilerRunner(
workingDir,
"caches-jvm",
reporter,
additionalOutputFiles = outputFiles,
buildHistoryFile = buildHistoryFile,
outputDirs = outputDirs,
withAbiSnapshot = withAbiSnapshot
) {
override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
@@ -181,7 +187,7 @@ open class IncrementalJvmCompilerRunner(
else
null
override fun calculateSourcesToCompileImpl(
override fun calculateSourcesToCompile(
caches: IncrementalJvmCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JVMCompilerArguments,
@@ -199,9 +205,8 @@ open class IncrementalJvmCompilerRunner(
//TODO can't use the same way as for build-history files because abi-snapshot for all dependencies should be stored into last-build
// and not only changed one
// (but possibly we dont need to read it all and may be it is possible to update only those who was changed)
override fun setupJarDependencies(args: K2JVMCompilerArguments, withSnapshot: Boolean, reporter: BuildReporter): Map<String, AbiSnapshot> {
override fun setupJarDependencies(args: K2JVMCompilerArguments, reporter: BuildReporter): Map<String, AbiSnapshot> {
//fill abiSnapshots
if (!withSnapshot) return emptyMap()
val abiSnapshots = HashMap<String, AbiSnapshot>()
args.classpathAsList
.filter { it.extension.equals("jar", ignoreCase = true) }
@@ -209,7 +214,12 @@ open class IncrementalJvmCompilerRunner(
modulesApiHistory.abiSnapshot(it).let { result ->
if (result is Either.Success<Set<File>>) {
result.value.forEach { file ->
AbiSnapshotImpl.read(file, reporter)?.also { abiSnapshot -> abiSnapshots[it.absolutePath] = abiSnapshot }
if (file.exists()) {
abiSnapshots[it.absolutePath] = AbiSnapshotImpl.read(file)
} else {
// FIXME: We should throw an exception here
reporter.warn { "Snapshot file does not exist: ${file.path}. Continue anyway." }
}
}
}
}
@@ -228,7 +238,7 @@ open class IncrementalJvmCompilerRunner(
caches: IncrementalJvmCachesManager,
changedFiles: ChangedFiles.Known,
args: K2JVMCompilerArguments,
abiSnapshots: Map<String, AbiSnapshot> = HashMap(),
abiSnapshots: Map<String, AbiSnapshot>,
withAbiSnapshot: Boolean
): CompilationMode {
val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
@@ -370,7 +380,9 @@ open class IncrementalJvmCompilerRunner(
return result
}
override fun preBuildHook(args: K2JVMCompilerArguments, compilationMode: CompilationMode) {
override fun performWorkBeforeCompilation(compilationMode: CompilationMode, args: K2JVMCompilerArguments) {
super.performWorkBeforeCompilation(compilationMode, args)
if (compilationMode is CompilationMode.Incremental) {
val destinationDir = args.destinationAsFile
destinationDir.mkdirs()
@@ -478,12 +490,15 @@ open class IncrementalJvmCompilerRunner(
return exitCode to sourcesToCompile
}
override fun performWorkAfterSuccessfulCompilation(caches: IncrementalJvmCachesManager, wasIncremental: Boolean) {
if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
override fun performWorkAfterCompilation(compilationMode: CompilationMode, exitCode: ExitCode, caches: IncrementalJvmCachesManager) {
super.performWorkAfterCompilation(compilationMode, exitCode, caches)
// No need to shrink and save classpath snapshot if exitCode != ExitCode.OK as the task will fail anyway
if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled && exitCode == ExitCode.OK) {
reporter.measure(BuildTime.SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
shrinkAndSaveClasspathSnapshot(
wasIncremental, classpathChanges, caches.lookupCache, currentClasspathSnapshot,
shrunkCurrentClasspathAgainstPreviousLookups, ClasspathSnapshotBuildReporter(reporter)
compilationWasIncremental = compilationMode is CompilationMode.Incremental, classpathChanges, caches.lookupCache,
currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups, ClasspathSnapshotBuildReporter(reporter)
)
}
}
compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt
+2 -1
View File
@@ -213,7 +213,8 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest<Args : CommonCom
val moduleModifiedDependencies = modifiedLibraries.filter { it.first in moduleDependencies }.map { it.second }
val moduleDeletedDependencies = deletedLibraries.filter { it.first in moduleDependencies }.map { it.second }
val changedDepsFiles = if (isInitial) null else ChangedFiles.Dependencies(moduleModifiedDependencies, moduleDeletedDependencies)
val changedDepsFiles =
if (isInitial) null else ChangedFiles.Known(moduleModifiedDependencies, moduleDeletedDependencies, forDependencies = true)
val moduleOutDir = File(outDir, module)
val moduleCacheDir = File(cacheDir, module)
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt
+2 -1
View File
@@ -2,6 +2,7 @@ package org.jetbrains.kotlin.gradle
import org.gradle.testkit.runner.BuildResult
import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
import org.jetbrains.kotlin.gradle.testbase.*
import org.jetbrains.kotlin.gradle.util.checkedReplace
@@ -735,7 +736,7 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation
// In the next build, compilation should be incremental and fail, then fall back to non-incremental compilation and succeed
build(":lib:compileKotlin") {
assertIncrementalCompilationFellBackToNonIncremental()
assertIncrementalCompilationFellBackToNonIncremental(BuildAttribute.IC_FAILED_TO_COMPILE_INCREMENTALLY)
// Also check that the output is not deleted (regression test for KT-49780)
assertFileExists(lookupFile)
}
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/compilationAssertions.kt
+21 -7
View File
@@ -96,8 +96,10 @@ fun BuildResult.assertNonIncrementalCompilation(reason: BuildAttribute? = null)
} else {
assertOutputContains(NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED)
}
// Also check that incremental compilation was not attempted, failed, and fell back to non-incremental compilation
assertOutputDoesNotContain(PERFORMING_INCREMENTAL_COMPILATION)
// Also check that the other cases didn't happen
assertOutputDoesNotContain(INCREMENTAL_COMPILATION_COMPLETED)
assertOutputDoesNotContain(FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION)
}
/**
@@ -108,9 +110,11 @@ fun BuildResult.assertNonIncrementalCompilation(reason: BuildAttribute? = null)
* Note: Log level of output must be set to [LogLevel.DEBUG].
*/
fun BuildResult.assertIncrementalCompilation(expectedCompiledKotlinFiles: Iterable<Path>? = null) {
assertOutputContains(PERFORMING_INCREMENTAL_COMPILATION)
// Also check that incremental compilation did not fail and fall back to non-incremental compilation
assertOutputContains(INCREMENTAL_COMPILATION_COMPLETED)
// Also check that the other cases didn't happen
assertOutputDoesNotContain(NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED)
assertOutputDoesNotContain(FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION)
expectedCompiledKotlinFiles?.let {
assertSameFiles(expected = it, actual = extractCompiledKotlinFiles(output), "Compiled Kotlin files differ:\n")
@@ -122,9 +126,19 @@ fun BuildResult.assertIncrementalCompilation(expectedCompiledKotlinFiles: Iterab
*
* Note: Log level of output must be set to [LogLevel.DEBUG].
*/
fun BuildResult.assertIncrementalCompilationFellBackToNonIncremental() {
assertOutputContains("$NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED: ${BuildAttribute.INCREMENTAL_COMPILATION_FAILED.name}")
fun BuildResult.assertIncrementalCompilationFellBackToNonIncremental(reason: BuildAttribute? = null) {
if (reason != null) {
assertOutputContains("$FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION (reason = ${reason.name})")
} else {
assertOutputContains(FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION)
}
// Also check that the other cases didn't happen
assertOutputDoesNotContain(INCREMENTAL_COMPILATION_COMPLETED)
assertOutputDoesNotContain(NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED)
}
private const val PERFORMING_INCREMENTAL_COMPILATION = "Performing incremental compilation"
// Each of the following messages should uniquely correspond to a case in `IncrementalCompilerRunner.ICResult`
private const val INCREMENTAL_COMPILATION_COMPLETED = "Incremental compilation completed"
const val NON_INCREMENTAL_COMPILATION_WILL_BE_PERFORMED = "Non-incremental compilation will be performed"
private const val FALLING_BACK_TO_NON_INCREMENTAL_COMPILATION = "Falling back to non-incremental compilation"
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt
+12 -13
View File
@@ -5,7 +5,6 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.GradleException
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.logging.Logging
@@ -18,9 +17,7 @@ import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider
import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup
import org.jetbrains.kotlin.gradle.tasks.*
import java.io.File
import javax.inject.Inject
@@ -43,7 +40,7 @@ internal class GradleCompilerRunnerWithWorkers(
workQueue.submit(GradleKotlinCompilerWorkAction::class.java) { params ->
params.compilerWorkArguments.set(workArgs)
if (taskOutputsBackup != null) {
params.taskOutputs.set(taskOutputsBackup.outputs)
params.taskOutputsToRestore.set(taskOutputsBackup.outputsToRestore)
params.buildDir.set(taskOutputsBackup.buildDirectory)
params.snapshotsDir.set(taskOutputsBackup.snapshotsDir)
params.metricsReporter.set(buildMetrics)
@@ -64,8 +61,7 @@ internal class GradleCompilerRunnerWithWorkers(
fileSystemOperations,
parameters.buildDir,
parameters.snapshotsDir,
parameters.taskOutputs.get(),
outputsToExclude = emptyList(),
parameters.taskOutputsToRestore.get(),
logger
)
} else {
@@ -76,11 +72,14 @@ internal class GradleCompilerRunnerWithWorkers(
GradleKotlinCompilerWork(
parameters.compilerWorkArguments.get()
).run()
} catch (e: GradleException) {
// Currently, metrics are not reported as in the worker we are getting new instance of [BuildMetricsReporter]
// [BuildDataRecorder] knows nothing about this new instance. Possibly could be fixed in the future by migrating
// [BuildMetricsReporter] to be shared Gradle service.
if (taskOutputsBackup != null) {
} catch (e: FailedCompilationException) {
// Restore outputs only in cases where we expect that the user will make some changes to their project:
// - For a compilation error, the user will need to fix their source code
// - For an OOM error, the user will need to increase their memory settings
// In the other cases where there is nothing the user can fix in their project, we should not restore the outputs.
// Otherwise, the next build(s) will likely fail in exactly the same way as this build because their inputs and outputs are
// the same.
if (taskOutputsBackup != null && (e is CompilationErrorException || e is OOMErrorException)) {
parameters.metricsReporter.get().measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
logger.info("Restoring task outputs to pre-compilation state")
taskOutputsBackup.restoreOutputs()
@@ -96,7 +95,7 @@ internal class GradleCompilerRunnerWithWorkers(
internal interface GradleKotlinCompilerWorkParameters : WorkParameters {
val compilerWorkArguments: Property<GradleKotlinCompilerWorkArguments>
val taskOutputs: ListProperty<File>
val taskOutputsToRestore: ListProperty<File>
val snapshotsDir: DirectoryProperty
val buildDir: DirectoryProperty
val metricsReporter: Property<BuildMetricsReporter>
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt
+4 -8
View File
@@ -5,7 +5,6 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logger
@@ -33,11 +32,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.plugin.variantImplementationFactory
import org.jetbrains.kotlin.gradle.tasks.*
import org.jetbrains.kotlin.gradle.utils.IsolatedKotlinClasspathClassCastException
import org.jetbrains.kotlin.gradle.utils.archivePathCompatible
import org.jetbrains.kotlin.gradle.utils.findByType
import org.jetbrains.kotlin.gradle.utils.newTmpFile
import org.jetbrains.kotlin.gradle.utils.relativeOrCanonical
import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.incremental.IncrementalModuleEntry
import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
@@ -223,8 +218,9 @@ internal open class GradleCompilerRunner(
try {
val kotlinCompilerRunnable = GradleKotlinCompilerWork(workArgs)
kotlinCompilerRunnable.run()
} catch (e: GradleException) {
if (taskOutputsBackup != null) {
} catch (e: FailedCompilationException) {
// Restore outputs only for CompilationErrorException or OOMErrorException (see GradleKotlinCompilerWorkAction.execute)
if (taskOutputsBackup != null && (e is CompilationErrorException || e is OOMErrorException)) {
buildMetrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
taskOutputsBackup.restoreOutputs()
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt
+2 -2
View File
@@ -17,7 +17,7 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.gradle.report.*
import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState
import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError
import org.jetbrains.kotlin.gradle.tasks.throwExceptionIfCompilationFailed
import org.jetbrains.kotlin.gradle.utils.stackTraceAsString
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.incremental.ClasspathChanges
@@ -125,7 +125,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete()
}
throwGradleExceptionIfError(exitCode, executionStrategy)
throwExceptionIfCompilationFailed(exitCode, executionStrategy)
} finally {
val taskInfo = TaskExecutionInfo(
changedFiles = incrementalCompilationEnvironment?.changedFiles,
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt
-2
View File
@@ -424,9 +424,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
taskExecutionResult?.buildMetrics?.buildPerformanceMetrics?.asMap()?.filterValues { value -> value != 0L } ?: emptyMap()
val changes = when (val changedFiles = taskExecutionResult?.taskInfo?.changedFiles) {
is ChangedFiles.Known -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
is ChangedFiles.Dependencies -> changedFiles.modified.map { it.absolutePath } + changedFiles.removed.map { it.absolutePath }
else -> emptyList<String>()
}
return CompileStatisticsData(
durationMs = durationMs,
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinJsDce.kt
+1 -1
View File
@@ -141,7 +141,7 @@ abstract class KotlinJsDce @Inject constructor(
buildDir.get().asFile,
jvmArgs
)
throwGradleExceptionIfError(exitCode, KotlinCompilerExecutionStrategy.OUT_OF_PROCESS)
throwExceptionIfCompilationFailed(exitCode, KotlinCompilerExecutionStrategy.OUT_OF_PROCESS)
}
private fun isDceCandidate(file: File): Boolean {
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt
+2 -4
View File
@@ -363,7 +363,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
private val systemPropertiesService = CompilerSystemPropertiesService.registerIfAbsent(project.gradle)
/** Task outputs that we don't want to include in [TaskOutputsBackup] (see [TaskOutputsBackup]'s kdoc for more info). */
/** Task outputs that we don't want to include in [TaskOutputsBackup] (see [TaskOutputsBackup.outputsToRestore] for more info). */
@get:Internal
protected open val taskOutputsBackupExcludes: List<File> = emptyList()
@@ -391,8 +391,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
fileSystemOperations,
layout.buildDirectory,
layout.buildDirectory.dir("snapshot/kotlin/$name"),
allOutputFiles(),
taskOutputsBackupExcludes,
outputsToRestore = allOutputFiles() - taskOutputsBackupExcludes,
logger
).also {
it.createSnapshot()
@@ -596,7 +595,6 @@ abstract class KotlinCompile @Inject constructor(
classpathSnapshotProperties.classpathSnapshot
)
// Exclude classpathSnapshotDir from TaskOutputsBackup (see TaskOutputsBackup's kdoc for more info). */
override val taskOutputsBackupExcludes: List<File>
get() = classpathSnapshotProperties.classpathSnapshotDir.orNull?.asFile?.let { listOf(it) } ?: emptyList()
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksOutputsBackup.kt
+16 -18
View File
@@ -5,7 +5,9 @@
package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.file.*
import org.gradle.api.file.Directory
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.logging.Logger
import org.gradle.api.provider.Provider
import java.io.File
@@ -14,35 +16,31 @@ import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.zip.*
import java.util.zip.Deflater
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
internal class TaskOutputsBackup(
val fileSystemOperations: FileSystemOperations,
private val fileSystemOperations: FileSystemOperations,
val buildDirectory: DirectoryProperty,
val snapshotsDir: Provider<Directory>,
allOutputs: List<File>,
/**
* Task outputs that we don't want to back up for performance reasons (e.g., if (1) they are too big, and (2) they are usually updated
* only at the end of the task execution--in a failed task run, they are usually unchanged and therefore don't need to be restored).
* Task outputs to back up and restore.
*
* NOTE: In `IncrementalCompilerRunner`, if incremental compilation fails, it will try again by cleaning all the outputs and perform
* non-incremental compilation. It is important that `IncrementalCompilerRunner` do not clean [outputsToExclude] immediately but only
* right before [outputsToExclude] are updated (which is usually at the end of the task execution). This is so that if the fallback
* compilation fails, [outputsToExclude] will remain unchanged and the other outputs will be restored, and the next task run can be
* incremental.
* Note that this could be a subset of all the outputs of a task because there could be task outputs that we don't want to back up and
* restore (e.g., if (1) they are too big and (2) they are updated only at the end of the task execution so in a failed task run, they
* are usually unchanged and therefore don't need to be restored).
*/
outputsToExclude: List<File> = emptyList(),
val outputsToRestore: List<File>,
val logger: Logger
) {
/** The outputs to back up and restore. Note that this may be a subset of all the outputs of a task (see `outputsToExclude`). */
val outputs: List<File> = allOutputs - outputsToExclude.toSet()
fun createSnapshot() {
// Kotlin JS compilation task declares one file from 'destinationDirectory' output as task `@OutputFile'
// property. To avoid snapshot sync collisions, each snapshot output directory has also 'index' as prefix.
outputs.toSortedSet().forEachIndexed { index, outputPath ->
outputsToRestore.toSortedSet().forEachIndexed { index, outputPath ->
if (outputPath.isDirectory && !outputPath.isEmptyDirectory) {
compressDirectoryToZip(
File(snapshotsDir.get().asFile, index.asSnapshotArchiveName),
@@ -59,10 +57,10 @@ internal class TaskOutputsBackup(
fun restoreOutputs() {
fileSystemOperations.delete {
it.delete(outputs)
it.delete(outputsToRestore)
}
outputs.toSortedSet().forEachIndexed { index, outputPath ->
outputsToRestore.toSortedSet().forEachIndexed { index, outputPath ->
val snapshotDir = snapshotsDir.get().file(index.asSnapshotDirectoryName).asFile
if (snapshotDir.isDirectory) {
fileSystemOperations.copy { spec ->
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt
+17 -8
View File
@@ -1,6 +1,5 @@
package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.GradleException
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
@@ -10,18 +9,19 @@ import org.jetbrains.kotlin.gradle.internal.tasks.TaskWithLocalState
import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.incremental.cleanDirectoryContents
import org.jetbrains.kotlin.incremental.deleteDirectoryContents
import org.jetbrains.kotlin.incremental.deleteRecursivelyOrThrow
import java.io.File
fun throwGradleExceptionIfError(
/** Throws [FailedCompilationException] if compilation completed with [exitCode] != [ExitCode.OK]. */
fun throwExceptionIfCompilationFailed(
exitCode: ExitCode,
executionStrategy: KotlinCompilerExecutionStrategy
) {
when (exitCode) {
ExitCode.COMPILATION_ERROR -> throw GradleException("Compilation error. See log for more details")
ExitCode.INTERNAL_ERROR -> throw GradleException("Internal compiler error. See log for more details")
ExitCode.SCRIPT_EXECUTION_ERROR -> throw GradleException("Script execution error. See log for more details")
ExitCode.COMPILATION_ERROR -> throw CompilationErrorException("Compilation error. See log for more details")
ExitCode.INTERNAL_ERROR -> throw FailedCompilationException("Internal compiler error. See log for more details")
ExitCode.SCRIPT_EXECUTION_ERROR -> throw FailedCompilationException("Script execution error. See log for more details")
ExitCode.OOM_ERROR -> {
var exceptionMessage = "Not enough memory to run compilation."
when (executionStrategy) {
@@ -31,13 +31,22 @@ fun throwGradleExceptionIfError(
exceptionMessage += " Try to increase it via 'gradle.properties':\norg.gradle.jvmargs=-Xmx<size>"
KotlinCompilerExecutionStrategy.OUT_OF_PROCESS -> Unit
}
throw GradleException(exceptionMessage)
throw OOMErrorException(exceptionMessage)
}
ExitCode.OK -> Unit
else -> throw IllegalStateException("Unexpected exit code: $exitCode")
}
}
/** Exception thrown when [ExitCode] != [ExitCode.OK]. */
internal open class FailedCompilationException(message: String) : RuntimeException(message)
/** Exception thrown when [ExitCode] == [ExitCode.COMPILATION_ERROR]. */
internal class CompilationErrorException(message: String) : FailedCompilationException(message)
/** Exception thrown when [ExitCode] == [ExitCode.OOM_ERROR]. */
internal class OOMErrorException(message: String) : FailedCompilationException(message)
internal fun TaskWithLocalState.cleanOutputsAndLocalState(reason: String? = null) {
val log = GradleKotlinLogger(logger)
cleanOutputsAndLocalState(allOutputFiles(), log, metrics.get(), reason)
@@ -59,7 +68,7 @@ internal fun cleanOutputsAndLocalState(
when {
file.isDirectory -> {
log.debug("Deleting contents of output directory: $file")
file.cleanDirectoryContents()
file.deleteDirectoryContents()
}
file.isFile -> {
log.debug("Deleting output file: $file")