From 21289722b72d816dcee1cbde6a293bebd4668ebf Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Fri, 2 Nov 2018 01:11:38 +0300 Subject: [PATCH] Move after-compilation cleanup actions to GradleKotlinCompilerWork Otherwise cleanup actions would run in-parallel or before compilation when Gradle Workers are used --- .../GradleCompilerRunnerWithWorkers.kt | 2 +- .../GradleKotlinCompilerRunner.kt | 42 +++++++++------ .../GradleKotlinCompilerWork.kt | 40 +++++++++++++- .../IncrementalCompilationEnvironment.kt | 1 + .../internal/kapt/KaptGenerateStubsTask.kt | 2 +- .../internal/kapt/KaptWithKotlincTask.kt | 3 +- .../gradle/tasks/KotlinCompileCommon.kt | 5 +- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 52 ++++++++----------- 8 files changed, 92 insertions(+), 55 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt index f47b14a3565..dae8ca0f467 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt @@ -14,7 +14,7 @@ internal class GradleCompilerRunnerWithWorkers( project: Project, private val workersExecutor: WorkerExecutor ) : GradleCompilerRunner(project) { - override fun compileWithDaemonOrFallback(workArgs: GradleKotlinCompilerWorkArguments) { + override fun runCompilerAsync(workArgs: GradleKotlinCompilerWorkArguments) { workersExecutor.submit(GradleKotlinCompilerWork::class.java) { config -> config.isolationMode = IsolationMode.NONE config.forkMode = ForkMode.NEVER diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt index 329de09e1ed..2aa4c9092c8 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt @@ -51,7 +51,11 @@ internal fun kotlinCompilerExecutionStrategy(): String = System.getProperty(KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY) ?: DAEMON_EXECUTION_STRATEGY internal open class GradleCompilerRunner(protected val project: Project) { - fun runJvmCompiler( + /** + * Compiler might be executed asynchronuosly. Do not do anything requiring end of compilation after this function is called. + * @see [GradleKotlinCompilerWork] + */ + fun runJvmCompilerAsync( sourcesToCompile: List, commonSources: List, javaSourceRoots: Iterable, @@ -75,16 +79,14 @@ internal open class GradleCompilerRunner(protected val project: Project) { args.destination = null } - try { - runCompiler(KotlinCompilerClass.JVM, args, environment) - } finally { - if (System.getProperty(DELETE_MODULE_FILE_PROPERTY) != "false") { - buildFile.delete() - } - } + runCompilerAsync(KotlinCompilerClass.JVM, args, environment, buildFile = buildFile) } - fun runJsCompiler( + /** + * Compiler might be executed asynchronuosly. Do not do anything requiring end of compilation after this function is called. + * @see [GradleKotlinCompilerWork] + */ + fun runJsCompilerAsync( kotlinSources: List, kotlinCommonSources: List, args: K2JSCompilerArguments, @@ -92,22 +94,27 @@ internal open class GradleCompilerRunner(protected val project: Project) { ) { args.freeArgs += kotlinSources.map { it.absolutePath } args.commonSources = kotlinCommonSources.map { it.absolutePath }.toTypedArray() - runCompiler(KotlinCompilerClass.JS, args, environment) + runCompilerAsync(KotlinCompilerClass.JS, args, environment) } - fun runMetadataCompiler( + /** + * Compiler might be executed asynchronuosly. Do not do anything requiring end of compilation after this function is called. + * @see [GradleKotlinCompilerWork] + */ + fun runMetadataCompilerAsync( kotlinSources: List, args: K2MetadataCompilerArguments, environment: GradleCompilerEnvironment ) { args.freeArgs += kotlinSources.map { it.absolutePath } - runCompiler(KotlinCompilerClass.METADATA, args, environment) + runCompilerAsync(KotlinCompilerClass.METADATA, args, environment) } - private fun runCompiler( + private fun runCompilerAsync( compilerClassName: String, compilerArgs: CommonCompilerArguments, - environment: GradleCompilerEnvironment + environment: GradleCompilerEnvironment, + buildFile: File? = null ) { if (compilerArgs.version) { project.logger.lifecycle( @@ -126,12 +133,13 @@ internal open class GradleCompilerRunner(protected val project: Project) { compilerArgs = argsArray, isVerbose = compilerArgs.verbose, incrementalCompilationEnvironment = incrementalCompilationEnvironment, - incrementalModuleInfo = modulesInfo + incrementalModuleInfo = modulesInfo, + buildFile = buildFile ) - compileWithDaemonOrFallback(workArgs) + runCompilerAsync(workArgs) } - protected open fun compileWithDaemonOrFallback(workArgs: GradleKotlinCompilerWorkArguments) { + protected open fun runCompilerAsync(workArgs: GradleKotlinCompilerWorkArguments) { val kotlinCompilerRunnable = GradleKotlinCompilerWork(workArgs) kotlinCompilerRunnable.run() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 97d4da80e1a..d612ad06db5 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -14,9 +14,11 @@ import org.jetbrains.kotlin.cli.common.messages.MessageRenderer import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.daemon.common.* import org.jetbrains.kotlin.gradle.plugin.kotlinDebug +import org.jetbrains.kotlin.gradle.plugin.kotlinInfo import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.incremental.DELETE_MODULE_FILE_PROPERTY import org.slf4j.LoggerFactory import java.io.ByteArrayOutputStream import java.io.File @@ -49,7 +51,8 @@ internal class GradleKotlinCompilerWorkArguments( val compilerArgs: Array, val isVerbose: Boolean, val incrementalCompilationEnvironment: IncrementalCompilationEnvironment?, - val incrementalModuleInfo: IncrementalModuleInfo? + val incrementalModuleInfo: IncrementalModuleInfo?, + val buildFile: File? ) : Serializable { companion object { const val serialVersionUID: Long = 0 @@ -75,6 +78,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( private val isVerbose = config.isVerbose private val incrementalCompilationEnvironment = config.incrementalCompilationEnvironment private val incrementalModuleInfo = config.incrementalModuleInfo + private val buildFile = config.buildFile private val log: KotlinLogger = SL4JKotlinLogger(LoggerFactory.getLogger("GradleKotlinCompilerWork")) @@ -84,10 +88,42 @@ internal class GradleKotlinCompilerWork @Inject constructor( get() = incrementalCompilationEnvironment != null override fun run() { - val exitCode = compileWithDaemonOrFallbackImpl() + val exitCode = try { + compileWithDaemonOrFallbackImpl() + } finally { + if (buildFile != null && System.getProperty(DELETE_MODULE_FILE_PROPERTY) != "false") { + buildFile.delete() + } + } + + if (incrementalCompilationEnvironment != null) { + if (incrementalCompilationEnvironment.disableMultiModuleIC) { + incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete() + } + + if (exitCode != ExitCode.OK) { + // for non-incremental compilation cleanup is always performed before compiler is called + cleanupOnError(incrementalCompilationEnvironment) + } + } + throwGradleExceptionIfError(exitCode) } + private fun cleanupOnError(incrementalCompilationEnvironment: IncrementalCompilationEnvironment) { + val localStateDirs = incrementalCompilationEnvironment.localStateDirs + log.info("Deleting output directories on error: ${localStateDirs.joinToString()}") + for (dir in localStateDirs) { + if (dir.exists()) { + if (dir.deleteRecursively()) { + log.debug("Deleted $dir") + } else { + log.debug("Could not delete $dir") + } + } + } + } + private fun compileWithDaemonOrFallbackImpl(): ExitCode { with(log) { kotlinDebug { "Kotlin compiler class: ${compilerClassName}" } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt index e39a0a5d581..b56258cfc88 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt @@ -15,6 +15,7 @@ internal class IncrementalCompilationEnvironment( val workingDir: File, val usePreciseJavaTracking: Boolean = false, val localStateDirs: List = emptyList(), + val disableMultiModuleIC: Boolean = false, val multiModuleICSettings: MultiModuleICSettings ) : Serializable { companion object { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptGenerateStubsTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptGenerateStubsTask.kt index bcc30daaca2..65cbc030b10 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptGenerateStubsTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptGenerateStubsTask.kt @@ -101,6 +101,6 @@ open class KaptGenerateStubsTask : KotlinCompile() { sourceRoots.log(this.name, logger) val args = prepareCompilerArguments() - callCompiler(args, sourceRoots, ChangedFiles(inputs)) + callCompilerAsync(args, sourceRoots, ChangedFiles(inputs)) } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt index b0b076a4d43..931799ad090 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt @@ -21,7 +21,6 @@ import org.jetbrains.kotlin.gradle.plugin.PLUGIN_CLASSPATH_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.tasks.CompilerPluginOptions import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector import org.jetbrains.kotlin.gradle.tasks.clearOutputDirectories -import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError import org.jetbrains.kotlin.gradle.utils.toSortedPathsArray open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput { @@ -70,7 +69,7 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput() : AbstractKo sourceRoots.log(this.name, logger) val args = prepareCompilerArguments() taskBuildDirectory.mkdirs() - callCompiler(args, sourceRoots, ChangedFiles(inputs)) + callCompilerAsync(args, sourceRoots, ChangedFiles(inputs)) } @Internal internal abstract fun getSourceRoots(): SourceRoots - internal abstract fun callCompiler(args: T, sourceRoots: SourceRoots, changedFiles: ChangedFiles) + /** + * Compiler might be executed asynchronuosly. Do not do anything requiring end of compilation after this function is called. + * @see [GradleKotlinCompilerWork] + */ + internal abstract fun callCompilerAsync(args: T, sourceRoots: SourceRoots, changedFiles: ChangedFiles) override fun setupCompilerArgs(args: T, defaultsOnly: Boolean) { args.coroutinesState = when (coroutines) { @@ -381,7 +384,7 @@ open class KotlinCompile : AbstractKotlinCompile(), Kotl @Internal override fun getSourceRoots() = SourceRoots.ForJvm.create(getSource(), sourceRootsContainer, sourceFilesExtensions) - override fun callCompiler(args: K2JVMCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) { + override fun callCompilerAsync(args: K2JVMCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) { sourceRoots as SourceRoots.ForJvm val messageCollector = GradleMessageCollector(logger) @@ -395,6 +398,7 @@ open class KotlinCompile : AbstractKotlinCompile(), Kotl taskBuildDirectory, usePreciseJavaTracking = usePreciseJavaTracking, localStateDirs = outputDirectories, + disableMultiModuleIC = disableMultiModuleIC(), multiModuleICSettings = multiModuleICSettings ) } else { @@ -402,26 +406,19 @@ open class KotlinCompile : AbstractKotlinCompile(), Kotl null } - try { - val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, icEnv) - compilerRunner.runJvmCompiler( - sourceRoots.kotlinSourceFiles, - commonSourceSet.toList(), - sourceRoots.javaSourceRoots, - javaPackagePrefix, - args, - environment - ) - - disableMultiModuleICIfNeeded() - } catch (e: Throwable) { - cleanupOnError() - throw e - } + val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, icEnv) + compilerRunner.runJvmCompilerAsync( + sourceRoots.kotlinSourceFiles, + commonSourceSet.toList(), + sourceRoots.javaSourceRoots, + javaPackagePrefix, + args, + environment + ) } - private fun disableMultiModuleICIfNeeded() { - if (!incremental || javaOutputDir == null) return + private fun disableMultiModuleIC(): Boolean { + if (!incremental || javaOutputDir == null) return false val illegalTask = project.tasks.matching { it is AbstractCompile && @@ -436,13 +433,10 @@ open class KotlinCompile : AbstractKotlinCompile(), Kotl "unknown task '$illegalTask' destination dir ${illegalTask.destinationDir} " + "intersects with java destination dir $javaOutputDir" ) - buildHistoryFile.delete() + return true } - } - private fun cleanupOnError() { - logger.kotlinInfo("deleting $destinationDir on error") - destinationDir.deleteRecursively() + return false } // override setSource to track source directory sets and files (for generated android folders) @@ -523,7 +517,7 @@ open class Kotlin2JsCompile() : AbstractKotlinCompile(), ?.let { if (LibraryUtils.isKotlinJavascriptLibrary(it)) it else null } ?.absolutePath - override fun callCompiler(args: K2JSCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) { + override fun callCompilerAsync(args: K2JSCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) { sourceRoots as SourceRoots.KotlinOnly logger.debug("Calling compiler") @@ -561,7 +555,7 @@ open class Kotlin2JsCompile() : AbstractKotlinCompile(), } else null val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, icEnv) - compilerRunner.runJsCompiler(sourceRoots.kotlinSourceFiles, commonSourceSet.toList(), args, environment) + compilerRunner.runJsCompilerAsync(sourceRoots.kotlinSourceFiles, commonSourceSet.toList(), args, environment) } }