diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index 2bb96efa4f5..312b5051782 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -1,7 +1,13 @@ package org.jetbrains.kotlin.gradle.tasks import com.intellij.ide.highlighter.JavaFileType +import com.intellij.lang.Language +import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.io.FileUtil +import com.intellij.psi.PsiClass +import com.intellij.psi.PsiFileFactory +import com.intellij.psi.PsiJavaFile +import com.intellij.psi.impl.PsiFileFactoryImpl import org.apache.commons.io.FileUtils import org.apache.commons.io.FilenameUtils import org.apache.commons.lang.StringUtils @@ -14,6 +20,8 @@ import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.tasks.SourceTask import org.gradle.api.tasks.TaskAction import org.gradle.api.tasks.compile.AbstractCompile +import org.gradle.api.tasks.incremental.IncrementalTaskInputs +import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.cli.common.CLICompiler import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments @@ -22,9 +30,20 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil import org.jetbrains.kotlin.cli.js.K2JSCompiler import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.compilerRunner.ArgumentUtils +import org.jetbrains.kotlin.compilerRunner.OutputItemsCollector +import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl +import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.* +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.modules.TargetId +import org.jetbrains.kotlin.progress.CompilationCanceledStatus import org.jetbrains.kotlin.utils.LibraryUtils import java.io.File import java.util.* @@ -40,16 +59,31 @@ abstract class AbstractKotlinCompile() : AbstractCo } abstract protected fun populateTargetSpecificArgs(args: T) - public var kotlinOptions: T = createBlankArgs() - public var kotlinDestinationDir: File? = destinationDir + var kotlinOptions: T = createBlankArgs() + var kotlinDestinationDir: File? = destinationDir var compilerCalled: Boolean = false private val loggerInstance = Logging.getLogger(this.javaClass) override fun getLogger() = loggerInstance - @TaskAction override fun compile() { + assert(false, { "unexpected call to compile()" }) + } + + @TaskAction + fun execute(inputs: IncrementalTaskInputs): Unit { logger.debug("Starting ${javaClass} task") + logger.kotlinDebug("all sources ${getSource().joinToString { it.path }}") + logger.kotlinDebug("is incremental == ${inputs.isIncremental}") + val modified = arrayListOf() + val removed = arrayListOf() + if (inputs.isIncremental) { + inputs.outOfDate { modified.add(it.file) } + inputs.removed { removed.add(it.file) } + } + logger.kotlinDebug("modified ${modified.joinToString { it.path }}") + logger.kotlinDebug("removed ${removed.joinToString { it.path }}") + var commonArgs = createBlankArgs() val args = createBlankArgs() val sources = getKotlinSources() if (sources.isEmpty()) { @@ -57,31 +91,45 @@ abstract class AbstractKotlinCompile() : AbstractCo return } - compilerCalled = true - populateCommonArgs(args, sources) + populateCommonArgs(args) populateTargetSpecificArgs(args) - callCompiler(args) + compilerCalled = true + val cachesDir = File(project.buildDir, "kotlin-caches") + callCompiler(args, sources, inputs.isIncremental, modified, removed, cachesDir) afterCompileHook(args) } private fun getKotlinSources(): List = (getSource() as Iterable).filter { it.isKotlinFile() } - private fun File.isKotlinFile(): Boolean { + protected fun File.isClassFile(): Boolean { + return when (FilenameUtils.getExtension(name).toLowerCase()) { + "class" -> true + else -> false + } + } + + protected fun File.isKotlinFile(): Boolean { return when (FilenameUtils.getExtension(name).toLowerCase()) { "kt", "kts" -> true else -> false } } - private fun populateCommonArgs(args: T, sources: List) { + private fun populateSources(args:T, sources: List) { args.freeArgs = sources.map { it.absolutePath } + } + + private fun populateCommonArgs(args: T) { args.suppressWarnings = kotlinOptions.suppressWarnings args.verbose = kotlinOptions.verbose args.version = kotlinOptions.version args.noInline = kotlinOptions.noInline } - private fun callCompiler(args: T) { + protected open fun callCompiler(args: T, sources: List, isIncremental: Boolean, modified: List, removed: List, cachesBaseDir: File) { + + populateSources(args, sources) + val messageCollector = GradleMessageCollector(logger) logger.debug("Calling compiler") val exitCode = compiler.exec(messageCollector, Services.EMPTY, args) @@ -95,7 +143,7 @@ abstract class AbstractKotlinCompile() : AbstractCo } -public open class KotlinCompile() : AbstractKotlinCompile() { +open class KotlinCompile() : AbstractKotlinCompile() { override val compiler = K2JVMCompiler() override fun createBlankArgs(): K2JVMCompilerArguments = K2JVMCompilerArguments() @@ -108,9 +156,12 @@ public open class KotlinCompile() : AbstractKotlinCompile("defaultModuleName") } @@ -165,16 +216,223 @@ public open class KotlinCompile() : AbstractKotlinCompile, isIncremental: Boolean, modified: List, removed: List, cachesBaseDir: File) { + + // TODO: consider other ways to pass incremental flag to compiler/builder + System.setProperty("kotlin.incremental.compilation", "true") + // TODO: experimental should be removed as soon as it becomes standard + System.setProperty("kotlin.incremental.compilation.experimental", "true") + + val targetType = "java-production" + val moduleName = args.moduleName + val targets = listOf(TargetId(moduleName, targetType)) + val outputDir = File(args.destination) + val caches = hashMapOf>() + val lookupStorage = LookupStorage(File(cachesBaseDir, "lookups")) + val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) + var currentRemoved = removed + val allGeneratedFiles = hashSetOf>() + + fun getOrCreateIncrementalCache(target: TargetId): IncrementalCacheImpl { + val cacheDir = File(cachesBaseDir, "increCache.${target.name}") + cacheDir.mkdirs() + return IncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target) + } + + fun PsiClass.findLookupSymbols(): Iterable { + val fqn = qualifiedName.orEmpty() + return listOf(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!))) + + methods.map { LookupSymbol(it.name, fqn) } + + fields.map { LookupSymbol(it.name.orEmpty(), fqn) } + + innerClasses.flatMap { it.findLookupSymbols() } + } + + fun dirtyKotlinSourcesFromGradle(): List { + val kotlinFiles = modified.filter { it.isKotlinFile() } + val javaFiles = modified.filter { it.isJavaFile() } + if (javaFiles.any()) { + val rootDisposable = Disposer.newDisposable() + val configuration = CompilerConfiguration() + val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) + val project = environment.project + val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl + val lookupSymbols = javaFiles.flatMap { + val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText()) + if (javaFile is PsiJavaFile) + javaFile.classes.flatMap { it.findLookupSymbols() } + else listOf() + } + logger.kotlinDebug("changed java symbols: ${lookupSymbols.joinToString { "${it.name}:${it.scope}" }}") + if (lookupSymbols.any()) { + return kotlinFiles + lookupSymbols.flatMap { lookupStorage.get(it) }.map { File(it) } + } + } + return kotlinFiles + } + + fun isClassPathChanged(): Boolean { + // TODO: that doesn't look to wise - join it first and then split here, consider storing it somewhere in between + val classpath = args.classpath.split(File.pathSeparator).map { File(it) }.toHashSet() + val changedClasspath = modified.filter { classpath.contains(it) } + if (changedClasspath.any()) { + logger.kotlinDebug("recompiling project because of classpath changes: $changedClasspath") + return true + } + return false + } + + fun cleanupOnError() { + val outputDirFile = File(args.destination!!) + + assert(outputDirFile.exists()) + val generatedRelPaths = allGeneratedFiles.map { it.outputFile.toRelativeString(outputDirFile) } + logger.kotlinDebug("deleting output on error: ${generatedRelPaths.joinToString()}") + + allGeneratedFiles.forEach { it.outputFile.delete() } + generatedRelPaths.forEach { File(destinationDir, it).delete() } + } + + fun outputRelativePath(f: File) = f.toRelativeString(outputDir) + fun outputRelativePath(p: String) = File(p).toRelativeString(outputDir) + + // TODO: decide what to do if no files are considered dirty - rebuild or skip the module + // TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes + var sourcesToCompile = + if (isIncremental && !isClassPathChanged()) dirtyKotlinSourcesFromGradle().distinct() + else sources + + if (isIncremental) { + args.classpath = args.classpath + File.pathSeparator + outputDir.absolutePath + } + + while (sourcesToCompile.any()) { + logger.kotlinDebug("compile iteration: ${sourcesToCompile.joinToString(", ")}") + + val (exitCode, generatedFiles) = compileChanged( + targets = targets, + sourcesToCompile = sourcesToCompile, + outputDir = outputDir, + args = args, + getIncrementalCache = { caches.getOrPut(it, { getOrCreateIncrementalCache(it) }) }, + lookupTracker = lookupTracker) + + allGeneratedFiles.addAll(generatedFiles) + // save versions? + + val changes = updateIncrementalCaches( + targets = targets, + generatedFiles = generatedFiles, + compiledWithErrors = exitCode != ExitCode.OK, + getIncrementalCache = { caches[it]!! }) + + lookupTracker.lookups.entrySet().forEach { + logger.kotlinDebug("lookups to ${it.key.name}:${it.key.scope} from ${it.value.joinToString { projectRelativePath(it) }}") + } + + lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved) + + when (exitCode) { + ExitCode.COMPILATION_ERROR -> { + cleanupOnError() + throw GradleException("Compilation error. See log for more details") + } + ExitCode.INTERNAL_ERROR -> { + cleanupOnError() + throw GradleException("Internal compiler error. See log for more details") + } + } + + logger.kotlinDebug("generated ${generatedFiles.joinToString { outputRelativePath(it.outputFile) }}") + logger.kotlinDebug("changes: ${changes.changes.joinToString { "${it.fqName}: ${it.javaClass.simpleName}" }}") + + logger.kotlinLazyDebug({ + "known lookups:\n${lookupStorage.dump(changes.changes.flatMap { + change -> + if (change is ChangeInfo.MembersChanged) + change.names.asSequence().map { LookupSymbol(it, change.fqName.asString()) } + else + sequenceOf() + }.toSet(), project.projectDir)}" }) + + if (!isIncremental) break; + + // TODO: consider using some order-preserving set for sourcesToCompile instead + val compiledSourcesSet = sourcesToCompile.toHashSet() + + val dirtyLookups = changes.dirtyLookups(caches.values.asSequence()) + + logger.kotlinDebug("dirty lookups: ${dirtyLookups.joinToString { "${it.name}:${it.scope}" }}") + + val dirty = dirtyLookups.flatMap { lookup -> + val files = lookupStorage.get(lookup).map(::File).filter { it !in compiledSourcesSet } + if (files.any()) { + logger.kotlinDebug("changes in $lookup causes recompilation of ${files.joinToString { projectRelativePath(it) }}") + } + files + } + //val dirty = changes.dirtyFiles(lookupStorage).filter { it !in compiledSourcesSet } + sourcesToCompile = dirty.filter { it in sources }.toList() + if (currentRemoved.any()) { + currentRemoved = listOf() + } + logger.kotlinDebug("dirty: ${dirty.joinToString { projectRelativePath(it) }}") + logger.kotlinDebug("to compile: ${sourcesToCompile.joinToString { projectRelativePath(it) }}") + } + lookupStorage.flush(false) + lookupStorage.close() + caches.values.forEach { it.flush(false); it.close() } + } + + private data class CompileChangedResults(val exitCode: ExitCode, val generatedFiles: List>) + + private fun compileChanged(targets: List, + sourcesToCompile: List, + outputDir: File, + args: K2JVMCompilerArguments, + getIncrementalCache: (TargetId) -> IncrementalCacheImpl, + lookupTracker: LookupTracker) + : CompileChangedResults + { + // show kotlin compiler where to look for java source files + args.freeArgs = (sourcesToCompile.map { it.absolutePath } + getJavaSourceRoots().map { it.absolutePath }).distinct() + args.destination = outputDir.absolutePath + + logger.kotlinDebug("compiling with args ${ArgumentUtils.convertArgumentsToStringList(args)}") + + val outputItemCollector = OutputItemsCollectorImpl() + + val messageCollector = GradleMessageCollector(logger, outputItemCollector) + + val incrementalCaches = makeIncrementalCachesMap(targets, { listOf() }, getIncrementalCache, { this }) + + val compilationCanceledStatus = object : CompilationCanceledStatus { + override fun checkCanceled() {} + } + + logger.kotlinDebug("compiling with args ${ArgumentUtils.convertArgumentsToStringList(args)}") + + val exitCode = compiler.exec(messageCollector, makeCompileServices(incrementalCaches, lookupTracker, compilationCanceledStatus), args) + + return CompileChangedResults( + exitCode, + outputItemCollector.generatedFiles( + targets = targets, + representativeTarget = targets.first(), + getSources = { sourcesToCompile }, + getOutputDir = { outputDir })) + } + + private fun handleKaptProperties(extraProperties: ExtraPropertiesExtension, pluginOptions: MutableList) { val kaptAnnotationsFile = extraProperties.getOrNull("kaptAnnotationsFile") if (kaptAnnotationsFile != null) { if (kaptAnnotationsFile.exists()) kaptAnnotationsFile.delete() - pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:output=$kaptAnnotationsFile") + pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:output=" + kaptAnnotationsFile) } val kaptClassFileStubsDir = extraProperties.getOrNull("kaptStubsDir") if (kaptClassFileStubsDir != null) { - pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:stubs=$kaptClassFileStubsDir") + pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:stubs=" + kaptClassFileStubsDir) } val supportInheritedAnnotations = extraProperties.getOrNull("kaptInheritedAnnotations") @@ -190,7 +448,7 @@ public open class KotlinCompile() : AbstractKotlinCompile() { +open class Kotlin2JsCompile() : AbstractKotlinCompile() { override val compiler = K2JSCompiler() override fun createBlankArgs(): K2JSCompilerArguments { @@ -257,22 +515,22 @@ public open class Kotlin2JsCompile() : AbstractKotlinCompile ExtraPropertiesExtension.getOrNull(id: String): T? { } } -class GradleMessageCollector(val logger: Logger) : MessageCollector { - public override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) { +class GradleMessageCollector(val logger: Logger, val outputCollector: OutputItemsCollector? = null) : MessageCollector { + override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) { val text = with(StringBuilder()) { append(when (severity) { in CompilerMessageSeverity.VERBOSE -> "v" @@ -355,9 +613,20 @@ class GradleMessageCollector(val logger: Logger) : MessageCollector { CompilerMessageSeverity.WARNING -> logger.warn(text) else -> throw IllegalArgumentException("Unknown CompilerMessageSeverity: $severity") } + // TODO: consider adding some other way of passing input -> output mapping from compiler, e.g. dedicated service + if (outputCollector != null && severity == CompilerMessageSeverity.OUTPUT) { + OutputMessageUtil.parseOutputMessage(message)?.let { + outputCollector.add(it.sourceFiles, it.outputFile) + } + } } } fun Logger.kotlinDebug(message: String) { this.debug("[KOTLIN] $message") } + +fun Logger.kotlinLazyDebug(makeMessage: () -> String) { + if (this.isInfoEnabled) + kotlinDebug(makeMessage()) +}