Switching gradle plugin to incremental compilation based on build lib

KT-8487

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -1,7 +1,13 @@
 package org.jetbrains.kotlin.gradle.tasks
 
 import com.intellij.ide.highlighter.JavaFileType
+import com.intellij.lang.Language
+import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.util.io.FileUtil
+import com.intellij.psi.PsiClass
+import com.intellij.psi.PsiFileFactory
+import com.intellij.psi.PsiJavaFile
+import com.intellij.psi.impl.PsiFileFactoryImpl
 import org.apache.commons.io.FileUtils
 import org.apache.commons.io.FilenameUtils
 import org.apache.commons.lang.StringUtils
@@ -14,6 +20,8 @@ import org.gradle.api.plugins.ExtraPropertiesExtension
 import org.gradle.api.tasks.SourceTask
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.compile.AbstractCompile
+import org.gradle.api.tasks.incremental.IncrementalTaskInputs
+import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.cli.common.CLICompiler
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
@@ -22,9 +30,20 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil
 import org.jetbrains.kotlin.cli.js.K2JSCompiler
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
+import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.compilerRunner.ArgumentUtils
+import org.jetbrains.kotlin.compilerRunner.OutputItemsCollector
+import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
+import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.*
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.modules.TargetId
+import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import org.jetbrains.kotlin.utils.LibraryUtils
 import java.io.File
 import java.util.*
@@ -40,16 +59,31 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
     }
     abstract protected fun populateTargetSpecificArgs(args: T)
 
-    public var kotlinOptions: T = createBlankArgs()
+    var kotlinOptions: T = createBlankArgs()
-    public var kotlinDestinationDir: File? = destinationDir
+    var kotlinDestinationDir: File? = destinationDir
     var compilerCalled: Boolean = false
 
     private val loggerInstance = Logging.getLogger(this.javaClass)
     override fun getLogger() = loggerInstance
 
-    @TaskAction
     override fun compile() {
+        assert(false, { "unexpected call to compile()" })
+    }
+
+    @TaskAction
+    fun execute(inputs: IncrementalTaskInputs): Unit {
         logger.debug("Starting ${javaClass} task")
+        logger.kotlinDebug("all sources ${getSource().joinToString { it.path }}")
+        logger.kotlinDebug("is incremental == ${inputs.isIncremental}")
+        val modified = arrayListOf<File>()
+        val removed = arrayListOf<File>()
+        if (inputs.isIncremental) {
+            inputs.outOfDate { modified.add(it.file) }
+            inputs.removed { removed.add(it.file) }
+        }
+        logger.kotlinDebug("modified ${modified.joinToString { it.path }}")
+        logger.kotlinDebug("removed ${removed.joinToString { it.path }}")
+        var commonArgs = createBlankArgs()
         val args = createBlankArgs()
         val sources = getKotlinSources()
         if (sources.isEmpty()) {
@@ -57,31 +91,45 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
             return
         }
 
-        compilerCalled = true
+        populateCommonArgs(args)
-        populateCommonArgs(args, sources)
         populateTargetSpecificArgs(args)
-        callCompiler(args)
+        compilerCalled = true
+        val cachesDir = File(project.buildDir, "kotlin-caches")
+        callCompiler(args, sources, inputs.isIncremental, modified, removed, cachesDir)
         afterCompileHook(args)
     }
 
     private fun getKotlinSources(): List<File> = (getSource() as Iterable<File>).filter { it.isKotlinFile() }
 
-    private fun File.isKotlinFile(): Boolean {
+    protected fun File.isClassFile(): Boolean {
+        return when (FilenameUtils.getExtension(name).toLowerCase()) {
+            "class" -> true
+            else -> false
+        }
+    }
+
+    protected fun File.isKotlinFile(): Boolean {
         return when (FilenameUtils.getExtension(name).toLowerCase()) {
             "kt", "kts" -> true
             else -> false
         }
     }
 
-    private fun populateCommonArgs(args: T, sources: List<File>) {
+    private fun populateSources(args:T, sources: List<File>) {
         args.freeArgs = sources.map { it.absolutePath }
+    }
+
+    private fun populateCommonArgs(args: T) {
         args.suppressWarnings = kotlinOptions.suppressWarnings
         args.verbose = kotlinOptions.verbose
         args.version = kotlinOptions.version
         args.noInline = kotlinOptions.noInline
     }
 
-    private fun callCompiler(args: T) {
+    protected open fun callCompiler(args: T, sources: List<File>, isIncremental: Boolean, modified: List<File>, removed: List<File>, cachesBaseDir: File) {
+
+        populateSources(args, sources)
+
         val messageCollector = GradleMessageCollector(logger)
         logger.debug("Calling compiler")
         val exitCode = compiler.exec(messageCollector, Services.EMPTY, args)
@@ -95,7 +143,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
 }
 
 
-public open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
+open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
     override val compiler = K2JVMCompiler()
     override fun createBlankArgs(): K2JVMCompilerArguments = K2JVMCompilerArguments()
 
@@ -108,9 +156,12 @@ public open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments
             "compileReleaseUnitTestKotlin" to  "compileReleaseKotlin"
             )
 
+    fun projectRelativePath(f: File) = f.toRelativeString(project.projectDir)
+    fun projectRelativePath(p: String) = File(p).toRelativeString(project.projectDir)
+
     override fun populateTargetSpecificArgs(args: K2JVMCompilerArguments) {
         // show kotlin compiler where to look for java source files
-        args.freeArgs = (args.freeArgs + getJavaSourceRoots().map { it.absolutePath }).toSet().toList()
+//        args.freeArgs = (args.freeArgs + getJavaSourceRoots().map { it.getAbsolutePath() }).toSet().toList()
         logger.kotlinDebug("args.freeArgs = ${args.freeArgs}")
 
         if (StringUtils.isEmpty(kotlinOptions.classpath)) {
@@ -151,7 +202,7 @@ public open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments
             if (tasks.size == 1) {
                 val task = tasks.firstOrNull() as? KotlinCompile
                 if (task != null) {
-                    logger.kotlinDebug("destinantion directory for production = ${task.destinationDir}")
+                    logger.kotlinDebug("destination directory for production = ${task.destinationDir}")
                     args.friendPaths = arrayOf(task.destinationDir.absolutePath)
                     args.moduleName = task.kotlinOptions.moduleName ?: task.extensions.extraProperties.getOrNull<String>("defaultModuleName")
                 }
@@ -165,16 +216,223 @@ public open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments
         logger.kotlinDebug("args.moduleName = ${args.moduleName}")
     }
 
+    override fun callCompiler(args: K2JVMCompilerArguments, sources: List<File>, isIncremental: Boolean, modified: List<File>, removed: List<File>, cachesBaseDir: File) {
+
+        // TODO: consider other ways to pass incremental flag to compiler/builder
+        System.setProperty("kotlin.incremental.compilation", "true")
+        // TODO: experimental should be removed as soon as it becomes standard
+        System.setProperty("kotlin.incremental.compilation.experimental", "true")
+
+        val targetType = "java-production"
+        val moduleName = args.moduleName
+        val targets = listOf(TargetId(moduleName, targetType))
+        val outputDir = File(args.destination)
+        val caches = hashMapOf<TargetId, IncrementalCacheImpl<TargetId>>()
+        val lookupStorage = LookupStorage(File(cachesBaseDir, "lookups"))
+        val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
+        var currentRemoved = removed
+        val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
+
+        fun getOrCreateIncrementalCache(target: TargetId): IncrementalCacheImpl<TargetId> {
+            val cacheDir = File(cachesBaseDir, "increCache.${target.name}")
+            cacheDir.mkdirs()
+            return IncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target)
+        }
+
+        fun PsiClass.findLookupSymbols(): Iterable<LookupSymbol> {
+            val fqn = qualifiedName.orEmpty()
+            return listOf(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!))) +
+                    methods.map { LookupSymbol(it.name, fqn) } +
+                    fields.map { LookupSymbol(it.name.orEmpty(), fqn) } +
+                    innerClasses.flatMap { it.findLookupSymbols() }
+        }
+
+        fun dirtyKotlinSourcesFromGradle(): List<File> {
+            val kotlinFiles = modified.filter { it.isKotlinFile() }
+            val javaFiles = modified.filter { it.isJavaFile() }
+            if (javaFiles.any()) {
+                val rootDisposable = Disposer.newDisposable()
+                val configuration = CompilerConfiguration()
+                val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
+                val project = environment.project
+                val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
+                val lookupSymbols = javaFiles.flatMap {
+                    val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText())
+                    if (javaFile is PsiJavaFile)
+                        javaFile.classes.flatMap { it.findLookupSymbols() }
+                    else listOf()
+                }
+                logger.kotlinDebug("changed java symbols: ${lookupSymbols.joinToString { "${it.name}:${it.scope}" }}")
+                if (lookupSymbols.any()) {
+                    return kotlinFiles + lookupSymbols.flatMap { lookupStorage.get(it) }.map { File(it) }
+                }
+            }
+            return kotlinFiles
+        }
+
+        fun isClassPathChanged(): Boolean {
+            // TODO: that doesn't look to wise - join it first and then split here, consider storing it somewhere in between
+            val classpath = args.classpath.split(File.pathSeparator).map { File(it) }.toHashSet()
+            val changedClasspath = modified.filter { classpath.contains(it) }
+            if (changedClasspath.any()) {
+                logger.kotlinDebug("recompiling project because of classpath changes: $changedClasspath")
+                return true
+            }
+            return false
+        }
+
+        fun cleanupOnError() {
+            val outputDirFile = File(args.destination!!)
+
+            assert(outputDirFile.exists())
+            val generatedRelPaths = allGeneratedFiles.map { it.outputFile.toRelativeString(outputDirFile) }
+            logger.kotlinDebug("deleting output on error: ${generatedRelPaths.joinToString()}")
+
+            allGeneratedFiles.forEach { it.outputFile.delete() }
+            generatedRelPaths.forEach { File(destinationDir, it).delete() }
+        }
+
+        fun outputRelativePath(f: File) = f.toRelativeString(outputDir)
+        fun outputRelativePath(p: String) = File(p).toRelativeString(outputDir)
+
+        // TODO: decide what to do if no files are considered dirty - rebuild or skip the module
+        // TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes
+        var sourcesToCompile =
+            if (isIncremental && !isClassPathChanged()) dirtyKotlinSourcesFromGradle().distinct()
+            else sources
+
+        if (isIncremental) {
+            args.classpath = args.classpath + File.pathSeparator + outputDir.absolutePath
+        }
+
+        while (sourcesToCompile.any()) {
+            logger.kotlinDebug("compile iteration: ${sourcesToCompile.joinToString(", ")}")
+
+            val (exitCode, generatedFiles) = compileChanged(
+                    targets = targets,
+                    sourcesToCompile = sourcesToCompile,
+                    outputDir = outputDir,
+                    args = args,
+                    getIncrementalCache = { caches.getOrPut(it, { getOrCreateIncrementalCache(it) }) },
+                    lookupTracker = lookupTracker)
+
+            allGeneratedFiles.addAll(generatedFiles)
+            // save versions?
+
+            val changes = updateIncrementalCaches(
+                    targets = targets,
+                    generatedFiles = generatedFiles,
+                    compiledWithErrors = exitCode != ExitCode.OK,
+                    getIncrementalCache = { caches[it]!! })
+
+            lookupTracker.lookups.entrySet().forEach {
+                logger.kotlinDebug("lookups to ${it.key.name}:${it.key.scope} from ${it.value.joinToString { projectRelativePath(it) }}")
+            }
+
+            lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved)
+
+            when (exitCode) {
+                ExitCode.COMPILATION_ERROR -> {
+                    cleanupOnError()
+                    throw GradleException("Compilation error. See log for more details")
+                }
+                ExitCode.INTERNAL_ERROR -> {
+                    cleanupOnError()
+                    throw GradleException("Internal compiler error. See log for more details")
+                }
+            }
+
+            logger.kotlinDebug("generated ${generatedFiles.joinToString { outputRelativePath(it.outputFile) }}")
+            logger.kotlinDebug("changes: ${changes.changes.joinToString { "${it.fqName}: ${it.javaClass.simpleName}" }}")
+
+            logger.kotlinLazyDebug({
+                "known lookups:\n${lookupStorage.dump(changes.changes.flatMap {
+                    change ->
+                        if (change is ChangeInfo.MembersChanged)
+                            change.names.asSequence().map { LookupSymbol(it, change.fqName.asString()) }
+                        else
+                            sequenceOf<LookupSymbol>()
+                }.toSet(), project.projectDir)}" })
+
+            if (!isIncremental) break;
+
+            // TODO: consider using some order-preserving set for sourcesToCompile instead
+            val compiledSourcesSet = sourcesToCompile.toHashSet()
+
+            val dirtyLookups = changes.dirtyLookups<TargetId>(caches.values.asSequence())
+
+            logger.kotlinDebug("dirty lookups: ${dirtyLookups.joinToString { "${it.name}:${it.scope}" }}")
+
+            val dirty = dirtyLookups.flatMap { lookup ->
+                val files = lookupStorage.get(lookup).map(::File).filter { it !in compiledSourcesSet }
+                if (files.any()) {
+                    logger.kotlinDebug("changes in $lookup causes recompilation of ${files.joinToString { projectRelativePath(it) }}")
+                }
+                files
+            }
+            //val dirty = changes.dirtyFiles(lookupStorage).filter { it !in compiledSourcesSet }
+            sourcesToCompile = dirty.filter { it in sources }.toList()
+            if (currentRemoved.any()) {
+                currentRemoved = listOf()
+            }
+            logger.kotlinDebug("dirty: ${dirty.joinToString { projectRelativePath(it) }}")
+            logger.kotlinDebug("to compile: ${sourcesToCompile.joinToString { projectRelativePath(it) }}")
+        }
+        lookupStorage.flush(false)
+        lookupStorage.close()
+        caches.values.forEach { it.flush(false); it.close() }
+    }
+
+    private data class CompileChangedResults(val exitCode: ExitCode, val generatedFiles: List<GeneratedFile<TargetId>>)
+
+    private fun compileChanged(targets: List<TargetId>,
+                               sourcesToCompile: List<File>,
+                               outputDir: File,
+                               args: K2JVMCompilerArguments,
+                               getIncrementalCache: (TargetId) -> IncrementalCacheImpl<TargetId>,
+                               lookupTracker: LookupTracker)
+            : CompileChangedResults
+    {
+        // show kotlin compiler where to look for java source files
+        args.freeArgs = (sourcesToCompile.map { it.absolutePath } + getJavaSourceRoots().map { it.absolutePath }).distinct()
+        args.destination = outputDir.absolutePath
+
+        logger.kotlinDebug("compiling with args ${ArgumentUtils.convertArgumentsToStringList(args)}")
+
+        val outputItemCollector = OutputItemsCollectorImpl()
+
+        val messageCollector = GradleMessageCollector(logger, outputItemCollector)
+
+        val incrementalCaches = makeIncrementalCachesMap(targets, { listOf<TargetId>() }, getIncrementalCache, { this })
+
+        val compilationCanceledStatus = object : CompilationCanceledStatus {
+            override fun checkCanceled() {}
+        }
+
+        logger.kotlinDebug("compiling with args ${ArgumentUtils.convertArgumentsToStringList(args)}")
+
+        val exitCode = compiler.exec(messageCollector, makeCompileServices(incrementalCaches, lookupTracker, compilationCanceledStatus), args)
+
+        return CompileChangedResults(
+                exitCode,
+                outputItemCollector.generatedFiles(
+                        targets = targets,
+                        representativeTarget = targets.first(),
+                        getSources = { sourcesToCompile },
+                        getOutputDir = { outputDir }))
+    }
+
+
     private fun handleKaptProperties(extraProperties: ExtraPropertiesExtension, pluginOptions: MutableList<String>) {
         val kaptAnnotationsFile = extraProperties.getOrNull<File>("kaptAnnotationsFile")
         if (kaptAnnotationsFile != null) {
             if (kaptAnnotationsFile.exists()) kaptAnnotationsFile.delete()
-            pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:output=$kaptAnnotationsFile")
+            pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:output=" + kaptAnnotationsFile)
         }
 
         val kaptClassFileStubsDir = extraProperties.getOrNull<File>("kaptStubsDir")
         if (kaptClassFileStubsDir != null) {
-            pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:stubs=$kaptClassFileStubsDir")
+            pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:stubs=" + kaptClassFileStubsDir)
         }
 
         val supportInheritedAnnotations = extraProperties.getOrNull<Boolean>("kaptInheritedAnnotations")
@@ -190,7 +448,7 @@ public open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments
             .filterNotNull()
             .toSet()
 
-    private fun File.isJavaFile() = extension.equals(JavaFileType.INSTANCE.getDefaultExtension(), ignoreCase = true)
+    private fun File.isJavaFile() = extension.equals(JavaFileType.INSTANCE.defaultExtension, ignoreCase = true)
 
     override fun afterCompileHook(args: K2JVMCompilerArguments) {
         logger.debug("Copying resulting files to classes")
@@ -246,7 +504,7 @@ public open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments
     }
 }
 
-public open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>() {
+open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>() {
     override val compiler = K2JSCompiler()
 
     override fun createBlankArgs(): K2JSCompilerArguments {
@@ -257,22 +515,22 @@ public open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArgumen
         return args
     }
 
-    public fun addLibraryFiles(vararg fs: String) {
+    fun addLibraryFiles(vararg fs: String) {
         kotlinOptions.libraryFiles += fs
     }
 
-    public fun addLibraryFiles(vararg fs: File) {
+    fun addLibraryFiles(vararg fs: File) {
         val strs = fs.map { it.path }.toTypedArray()
         addLibraryFiles(*strs)
     }
 
-    public val outputFile: String?
+    val outputFile: String?
             get() = kotlinOptions.outputFile
 
-    public val sourceMapDestinationDir: File
+    val sourceMapDestinationDir: File
             get() = File(outputFile).let { if (it.isDirectory) it else it.parentFile!! }
 
-    public val sourceMap: Boolean
+    val sourceMap: Boolean
             get() = kotlinOptions.sourceMap
 
     init {
@@ -323,8 +581,8 @@ private fun <T: Any> ExtraPropertiesExtension.getOrNull(id: String): T? {
     }
 }
 
-class GradleMessageCollector(val logger: Logger) : MessageCollector {
+class GradleMessageCollector(val logger: Logger, val outputCollector: OutputItemsCollector? = null) : MessageCollector {
-    public override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) {
+    override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) {
         val text = with(StringBuilder()) {
             append(when (severity) {
                 in CompilerMessageSeverity.VERBOSE -> "v"
@@ -355,9 +613,20 @@ class GradleMessageCollector(val logger: Logger) : MessageCollector {
             CompilerMessageSeverity.WARNING -> logger.warn(text)
             else -> throw IllegalArgumentException("Unknown CompilerMessageSeverity: $severity")
         }
+        // TODO: consider adding some other way of passing input -> output mapping from compiler, e.g. dedicated service
+        if (outputCollector != null && severity == CompilerMessageSeverity.OUTPUT) {
+            OutputMessageUtil.parseOutputMessage(message)?.let {
+                outputCollector.add(it.sourceFiles, it.outputFile)
+            }
+        }
     }
 }
 
 fun Logger.kotlinDebug(message: String) {
     this.debug("[KOTLIN] $message")
 }
+
+fun Logger.kotlinLazyDebug(makeMessage: () -> String) {
+    if (this.isInfoEnabled)
+        kotlinDebug(makeMessage())
+}