Restore tasks outputs to pre-execution state in worker.

Workers usually execute later than task action and does not raise
exception to the task. Such behaviour skips task outputs restoration to
the pre-execution state.

Kotlin compiler on incremental compilation error leaves 'dirty*' files
in the build output, forcing Gradle to run non-incrementally on the
next build. Fixed it by moving task outputs reset logic into worker as
well.

^KT-46406 In Progress
This commit is contained in:
Yahor Berdnikau
2021-09-16 10:52:15 +02:00
committed by Space
parent 904f9c72f2
commit 3dc69495f9
12 changed files with 166 additions and 68 deletions
@@ -5,9 +5,10 @@
package org.jetbrains.kotlin.build.report.metrics
import java.io.Serializable
import java.util.*
class BuildMetricsReporterImpl : BuildMetricsReporter {
class BuildMetricsReporterImpl : BuildMetricsReporter, Serializable {
private val myBuildTimeStartNs: EnumMap<BuildTime, Long> =
EnumMap(
BuildTime::class.java
@@ -5,13 +5,19 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.GradleException
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.provider.MapProperty
import org.gradle.api.provider.Property
import org.gradle.workers.WorkAction
import org.gradle.workers.WorkParameters
import org.gradle.workers.WorkQueue
import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider
import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup
import java.io.File
/**
@@ -21,28 +27,57 @@ internal class GradleCompilerRunnerWithWorkers(
taskProvider: GradleCompileTaskProvider,
jdkToolsJar: File?,
kotlinDaemonJvmArgs: List<String>?,
buildMetrics: BuildMetricsReporter,
private val workerExecutor: WorkerExecutor
) : GradleCompilerRunner(taskProvider, jdkToolsJar, kotlinDaemonJvmArgs) {
override fun runCompilerAsync(workArgs: GradleKotlinCompilerWorkArguments): WorkQueue {
loggerProvider.kotlinDebug { "Starting Kotlin compiler work from task '${pathProvider}'" }
) : GradleCompilerRunner(taskProvider, jdkToolsJar, kotlinDaemonJvmArgs, buildMetrics) {
override fun runCompilerAsync(
workArgs: GradleKotlinCompilerWorkArguments,
taskOutputsBackup: TaskOutputsBackup?
): WorkQueue {
val workQueue = workerExecutor.noIsolation()
workQueue.submit(GradleKotlinCompilerWorkAction::class.java) {
it.compilerWorkArguments.set(workArgs)
if (taskOutputsBackup != null) {
it.taskOutputs.from(taskOutputsBackup.outputs)
it.taskOutputsSnapshot.set(taskOutputsBackup.previousOutputs)
it.metricsReporter.set(buildMetrics)
}
}
return workQueue
}
internal abstract class GradleKotlinCompilerWorkAction
: WorkAction<GradleKotlinCompilerWorkParameters> {
override fun execute() {
GradleKotlinCompilerWork(
parameters.compilerWorkArguments.get()
).run()
try {
GradleKotlinCompilerWork(
parameters.compilerWorkArguments.get()
).run()
} catch (e: GradleException) {
if (parameters.taskOutputsSnapshot.isPresent) {
val taskOutputsBackup = TaskOutputsBackup(
parameters.taskOutputs,
parameters.taskOutputsSnapshot.get()
)
// Currently, metrics are not reported as in the worker we are getting new instance of [BuildMetricsReporter]
// [BuildDataRecorder] knows nothing about this new instance. Possibly could be fixed in the future by migrating
// [BuildMetricsReporter] to be shared Gradle service.
parameters.metricsReporter.get().measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
taskOutputsBackup.restoreOutputs()
}
}
throw e
}
}
}
internal interface GradleKotlinCompilerWorkParameters : WorkParameters {
val compilerWorkArguments: Property<GradleKotlinCompilerWorkArguments>
val taskOutputs: ConfigurableFileCollection
val taskOutputsSnapshot: MapProperty<File, Array<Byte>>
val metricsReporter: Property<BuildMetricsReporter>
}
}
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logger
@@ -12,6 +13,9 @@ import org.gradle.api.plugins.JavaPluginConvention
import org.gradle.api.tasks.bundling.AbstractArchiveTask
import org.gradle.jvm.tasks.Jar
import org.gradle.workers.WorkQueue
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.arguments.*
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.daemon.client.CompileServiceSession
@@ -26,10 +30,9 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider
import org.jetbrains.kotlin.gradle.tasks.*
import org.jetbrains.kotlin.gradle.tasks.InspectClassesForMultiModuleIC
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup
import org.jetbrains.kotlin.gradle.utils.archivePathCompatible
import org.jetbrains.kotlin.gradle.utils.newTmpFile
import org.jetbrains.kotlin.gradle.utils.relativeOrCanonical
@@ -62,7 +65,8 @@ is not assignable to 'org.gradle.api.tasks.TaskProvider'" exception
internal open class GradleCompilerRunner(
protected val taskProvider: GradleCompileTaskProvider,
protected val jdkToolsJar: File?,
protected val kotlinDaemonJvmArgs: List<String>?
protected val kotlinDaemonJvmArgs: List<String>?,
protected val buildMetrics: BuildMetricsReporter
) {
internal val pathProvider = taskProvider.path.get()
@@ -85,7 +89,8 @@ internal open class GradleCompilerRunner(
javaPackagePrefix: String?,
args: K2JVMCompilerArguments,
environment: GradleCompilerEnvironment,
jdkHome: File
jdkHome: File,
taskOutputsBackup: TaskOutputsBackup?
): WorkQueue? {
args.freeArgs += sourcesToCompile.map { it.absolutePath }
args.commonSources = commonSources.map { it.absolutePath }.toTypedArray()
@@ -93,7 +98,7 @@ internal open class GradleCompilerRunner(
args.javaPackagePrefix = javaPackagePrefix
if (args.jdkHome == null && !args.noJdk) args.jdkHome = jdkHome.absolutePath
loggerProvider.kotlinInfo("Kotlin compilation 'jdkHome' argument: ${args.jdkHome}")
return runCompilerAsync(KotlinCompilerClass.JVM, args, environment)
return runCompilerAsync(KotlinCompilerClass.JVM, args, environment, taskOutputsBackup)
}
/**
@@ -104,11 +109,12 @@ internal open class GradleCompilerRunner(
kotlinSources: List<File>,
kotlinCommonSources: List<File>,
args: K2JSCompilerArguments,
environment: GradleCompilerEnvironment
environment: GradleCompilerEnvironment,
taskOutputsBackup: TaskOutputsBackup?
): WorkQueue? {
args.freeArgs += kotlinSources.map { it.absolutePath }
args.commonSources = kotlinCommonSources.map { it.absolutePath }.toTypedArray()
return runCompilerAsync(KotlinCompilerClass.JS, args, environment)
return runCompilerAsync(KotlinCompilerClass.JS, args, environment, taskOutputsBackup)
}
/**
@@ -127,7 +133,8 @@ internal open class GradleCompilerRunner(
private fun runCompilerAsync(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: GradleCompilerEnvironment
environment: GradleCompilerEnvironment,
taskOutputsBackup: TaskOutputsBackup? = null
): WorkQueue? {
if (compilerArgs.version) {
loggerProvider.lifecycle(
@@ -192,12 +199,29 @@ internal open class GradleCompilerRunner(
daemonJvmArgs = kotlinDaemonJvmArgs
)
TaskLoggers.put(pathProvider, loggerProvider)
return runCompilerAsync(workArgs)
return runCompilerAsync(
workArgs,
taskOutputsBackup
)
}
protected open fun runCompilerAsync(workArgs: GradleKotlinCompilerWorkArguments): WorkQueue? {
val kotlinCompilerRunnable = GradleKotlinCompilerWork(workArgs)
kotlinCompilerRunnable.run()
protected open fun runCompilerAsync(
workArgs: GradleKotlinCompilerWorkArguments,
taskOutputsBackup: TaskOutputsBackup?
): WorkQueue? {
try {
val kotlinCompilerRunnable = GradleKotlinCompilerWork(workArgs)
kotlinCompilerRunnable.run()
} catch (e: GradleException) {
if (taskOutputsBackup != null) {
buildMetrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
taskOutputsBackup.restoreOutputs()
}
}
throw e
}
return null
}
@@ -176,8 +176,8 @@ abstract class KaptTask @Inject constructor(
abstract val source: ConfigurableFileCollection
@get:Internal
override val metrics: BuildMetricsReporter =
BuildMetricsReporterImpl()
override val metrics: Property<BuildMetricsReporter> = objectFactory
.property(BuildMetricsReporterImpl())
@get:Input
abstract val verbose: Property<Boolean>
@@ -11,7 +11,6 @@ import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.*
import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.gradle.work.InputChanges
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment
@@ -122,7 +121,8 @@ abstract class KaptWithKotlincTask @Inject constructor(
val compilerRunner = GradleCompilerRunner(
taskProvider.get(),
defaultKotlinJavaToolchain.get().currentJvmJdkToolsJar.orNull,
normalizedKotlinDaemonJvmArguments.orNull
normalizedKotlinDaemonJvmArguments.orNull,
metrics.get()
)
compilerRunner.runJvmCompilerAsync(
sourcesToCompile = emptyList(),
@@ -131,7 +131,8 @@ abstract class KaptWithKotlincTask @Inject constructor(
javaPackagePrefix = javaPackagePrefix.orNull,
args = args,
environment = environment,
jdkHome = defaultKotlinJavaToolchain.get().providedJvm.get().javaHome
jdkHome = defaultKotlinJavaToolchain.get().providedJvm.get().javaHome,
null
)
}
}
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.gradle.internal.tasks
import org.gradle.api.Task
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.FileCollection
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Internal
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
@@ -16,7 +17,7 @@ internal interface TaskWithLocalState : Task {
val localStateDirectories: ConfigurableFileCollection
@get:Internal
val metrics: BuildMetricsReporter
val metrics: Property<BuildMetricsReporter>
}
internal fun TaskWithLocalState.allOutputFiles(): FileCollection =
@@ -95,7 +95,7 @@ private class TaskRecord(
}
if (task is TaskWithLocalState) {
myBuildMetrics.addAll(task.metrics.getMetrics())
myBuildMetrics.addAll(task.metrics.get().getMetrics())
}
}
}
@@ -34,6 +34,7 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT
import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.PRODUCTION
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
import org.jetbrains.kotlin.gradle.tasks.SourceRoots
import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup
import org.jetbrains.kotlin.gradle.utils.getAllDependencies
import org.jetbrains.kotlin.gradle.utils.getCacheDirectory
import org.jetbrains.kotlin.gradle.utils.getDependenciesCacheDirectories
@@ -100,7 +101,12 @@ abstract class KotlinJsIrLink @Inject constructor(
return !entryModule.get().asFile.exists()
}
override fun callCompilerAsync(args: K2JSCompilerArguments, sourceRoots: SourceRoots, inputChanges: InputChanges) {
override fun callCompilerAsync(
args: K2JSCompilerArguments,
sourceRoots: SourceRoots,
inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup?
) {
KotlinBuildStatsService.applyIfInitialised {
it.report(BooleanMetrics.JS_IR_INCREMENTAL, incrementalJsIr)
}
@@ -131,7 +137,7 @@ abstract class KotlinJsIrLink @Inject constructor(
it.normalize().absolutePath
}
}
super.callCompilerAsync(args, sourceRoots, inputChanges)
super.callCompilerAsync(args, sourceRoots, inputChanges, taskOutputsBackup)
}
private fun visitCompilation(
@@ -347,7 +353,8 @@ internal class CacheBuilder(
emptyList(),
emptyList(),
compilerArgs,
environment
environment,
null
)?.await()
}
@@ -89,6 +89,7 @@ abstract class KotlinCompileCommon @Inject constructor(
it,
null,
normalizedKotlinDaemonJvmArguments.orNull,
metrics.get(),
workerExecutor
)
}
@@ -133,7 +134,12 @@ abstract class KotlinCompileCommon @Inject constructor(
@get:Internal
internal val expectActualLinker = objects.property(Boolean::class.java)
override fun callCompilerAsync(args: K2MetadataCompilerArguments, sourceRoots: SourceRoots, inputChanges: InputChanges) {
override fun callCompilerAsync(
args: K2MetadataCompilerArguments,
sourceRoots: SourceRoots,
inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup?
) {
val messageCollector = GradlePrintingMessageCollector(logger, args.allWarningsAsErrors)
val outputItemCollector = OutputItemsCollectorImpl()
val compilerRunner = compilerRunner.get()
@@ -92,8 +92,8 @@ abstract class AbstractKotlinCompileTool<T : CommonToolArguments>
}
@get:Internal
override val metrics: BuildMetricsReporter =
BuildMetricsReporterImpl()
override val metrics: Property<BuildMetricsReporter> = project.objects
.property(BuildMetricsReporterImpl())
/**
* By default, should be set by plugin from [COMPILER_CLASSPATH_CONFIGURATION_NAME] configuration.
@@ -315,7 +315,12 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
internal open val compilerRunner: Provider<GradleCompilerRunner> =
objects.propertyWithConvention(
gradleCompileTaskProvider.map {
GradleCompilerRunner(it, null, normalizedKotlinDaemonJvmArguments.orNull)
GradleCompilerRunner(
it,
null,
normalizedKotlinDaemonJvmArguments.orNull,
metrics.get()
)
}
)
@@ -323,7 +328,8 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
@TaskAction
fun execute(inputChanges: InputChanges) {
metrics.measure(BuildTime.GRADLE_TASK_ACTION) {
val buildMetrics = metrics.get()
buildMetrics.measure(BuildTime.GRADLE_TASK_ACTION) {
systemPropertiesService.get().startIntercept()
CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value = "true"
@@ -332,7 +338,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
// To prevent this, we backup outputs before incremental build and restore when exception is thrown
val outputsBackup: TaskOutputsBackup? =
if (isIncrementalCompilationEnabled() && inputChanges.isIncremental)
metrics.measure(BuildTime.BACKUP_OUTPUT) {
buildMetrics.measure(BuildTime.BACKUP_OUTPUT) {
TaskOutputsBackup(allOutputFiles())
}
else null
@@ -343,16 +349,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
clearLocalState("Task cannot run incrementally")
}
try {
executeImpl(inputChanges)
} catch (t: Throwable) {
if (outputsBackup != null) {
metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
outputsBackup.restoreOutputs()
}
}
throw t
}
executeImpl(inputChanges, outputsBackup)
}
}
@@ -369,7 +366,10 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
commonSourceSet
)
private fun executeImpl(inputChanges: InputChanges) {
private fun executeImpl(
inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup?
) {
val sourceRoots = getSourceRoots()
val allKotlinSources = sourceRoots.kotlinSourceFiles
@@ -386,7 +386,12 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
sourceRoots.log(this.name, logger)
taskBuildDirectory.get().asFile.mkdirs()
callCompilerAsync(args, sourceRoots, inputChanges)
callCompilerAsync(
args,
sourceRoots,
inputChanges,
taskOutputsBackup
)
}
protected fun getChangedFiles(
@@ -420,7 +425,12 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
* Compiler might be executed asynchronously. Do not do anything requiring end of compilation after this function is called.
* @see [GradleKotlinCompilerWork]
*/
internal abstract fun callCompilerAsync(args: T, sourceRoots: SourceRoots, inputChanges: InputChanges)
internal abstract fun callCompilerAsync(
args: T,
sourceRoots: SourceRoots,
inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup?
)
@get:Input
internal val multiPlatformEnabled: Property<Boolean> = objects.property(Boolean::class.java)
@@ -640,6 +650,7 @@ abstract class KotlinCompile @Inject constructor(
it,
toolchain.currentJvmJdkToolsJar.orNull,
normalizedKotlinDaemonJvmArguments.orNull,
metrics.get(),
workerExecutor
)
})
@@ -693,7 +704,12 @@ abstract class KotlinCompile @Inject constructor(
validateKotlinAndJavaHasSameTargetCompatibility(args)
}
override fun callCompilerAsync(args: K2JVMCompilerArguments, sourceRoots: SourceRoots, inputChanges: InputChanges) {
override fun callCompilerAsync(
args: K2JVMCompilerArguments,
sourceRoots: SourceRoots,
inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup?
) {
sourceRoots as SourceRoots.ForJvm
val messageCollector = GradlePrintingMessageCollector(logger, args.allWarningsAsErrors)
@@ -738,7 +754,8 @@ abstract class KotlinCompile @Inject constructor(
javaPackagePrefix,
args,
environment,
defaultKotlinJavaToolchain.get().providedJvm.get().javaHome
defaultKotlinJavaToolchain.get().providedJvm.get().javaHome,
taskOutputsBackup
)
}
@@ -944,6 +961,7 @@ abstract class Kotlin2JsCompile @Inject constructor(
it,
null,
normalizedKotlinDaemonJvmArguments.orNull,
metrics.get(),
workerExecutor
)
}
@@ -1052,7 +1070,12 @@ abstract class Kotlin2JsCompile @Inject constructor(
override val incrementalProps: List<FileCollection>
get() = super.incrementalProps + listOf(friendDependencies)
override fun callCompilerAsync(args: K2JSCompilerArguments, sourceRoots: SourceRoots, inputChanges: InputChanges) {
override fun callCompilerAsync(
args: K2JSCompilerArguments,
sourceRoots: SourceRoots,
inputChanges: InputChanges,
taskOutputsBackup: TaskOutputsBackup?
) {
sourceRoots as SourceRoots.KotlinOnly
logger.debug("Calling compiler")
@@ -1104,7 +1127,8 @@ abstract class Kotlin2JsCompile @Inject constructor(
sourceRoots.kotlinSourceFiles.files.toList(),
commonSourceSet.toList(),
args,
environment
environment,
taskOutputsBackup
)
}
}
@@ -10,21 +10,20 @@ import org.jetbrains.kotlin.utils.keysToMap
import java.io.File
import java.util.HashSet
internal class TaskOutputsBackup(private val outputs: FileCollection) {
private val previousOutputs: Map<File, ByteArray>
init {
val outputFiles = HashSet<File>()
outputs.forEach {
internal class TaskOutputsBackup(
val outputs: FileCollection,
val previousOutputs: Map<File, Array<Byte>> = outputs
.flatMap {
if (it.isDirectory) {
it.walk().filterTo(outputFiles, File::isFile)
} else if (it.isFile) {
outputFiles.add(it)
it.walk().filter(File::isFile)
} else {
sequenceOf(it)
}
}
previousOutputs = outputFiles.keysToMap { it.readBytes() }
}
.filter { it.exists() }
.toSet()
.keysToMap { it.readBytes().toTypedArray() }
) {
fun restoreOutputs() {
outputs.forEach {
@@ -42,7 +41,7 @@ internal class TaskOutputsBackup(private val outputs: FileCollection) {
if (dirs.add(dir)) {
dir.mkdirs()
}
file.writeBytes(bytes)
file.writeBytes(bytes.toByteArray())
}
}
}
@@ -25,7 +25,7 @@ fun throwGradleExceptionIfError(exitCode: ExitCode) {
internal fun TaskWithLocalState.clearLocalState(reason: String? = null) {
val log = GradleKotlinLogger(logger)
clearLocalState(allOutputFiles(), log, metrics, reason)
clearLocalState(allOutputFiles(), log, metrics.get(), reason)
}
internal fun clearLocalState(