Kapt: support incremental compilation in Gradle (KT-13500)

Kapt will process sources on each step of incremental compilation.
(cherry picked from commit 4cb2127)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/Kapt2KotlinGradleSubplugin.kt

@@ -78,19 +78,24 @@ class Kapt2KotlinGradleSubplugin : KotlinGradleSubplugin {
                 kaptClasspath.addAll(kapt2Configuration.resolve())
             }
         }
-
+        
-        val generatedFilesDir = if (variantData != null) {
+        val sourceSetName = if (variantData != null) {
             for (provider in (variantData as BaseVariantData<*>).sourceProviders) {
                 handleSourceSet((provider as AndroidSourceSet).name)
             }
 
-            getKaptGeneratedDir(project, variantData.name).apply { variantData.addJavaSourceFoldersToModel(this) }
+            variantData.name
         }
         else {
             if (javaSourceSet == null) error("Java source set should not be null")
 
             handleSourceSet(javaSourceSet.name)
-            getKaptGeneratedDir(project, javaSourceSet.name)
+            javaSourceSet.name
+        }
+
+        val generatedFilesDir = getKaptGeneratedDir(project, sourceSetName)
+        if (variantData != null) {
+            (variantData as BaseVariantData<*>).addJavaSourceFoldersToModel(generatedFilesDir)
         }
 
         // Skip annotation processing in kotlinc if no kapt dependencies were provided
@@ -117,6 +122,9 @@ class Kapt2KotlinGradleSubplugin : KotlinGradleSubplugin {
             pluginOptions += SubpluginOption("verbose", "true")
         }
         
+        val incrementalCompilationDataFile = File(project.buildDir, "tmp/kapt2/$sourceSetName/incrementalData.txt")
+        pluginOptions += SubpluginOption("incrementalData", incrementalCompilationDataFile.absolutePath)
+        
         return pluginOptions
     }
 

libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt2IT.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.gradle
 
+import org.jetbrains.kotlin.gradle.util.getFileByName
 import org.junit.Test
 import java.io.File
 import java.io.FileFilter
@@ -63,6 +64,31 @@ class Kapt2IT: BaseGradleIT() {
         }
     }
 
+    @Test
+    fun testSimpleWithIC() {
+        val options = defaultBuildOptions().copy(incremental = true)
+        val project = Project("simple", GRADLE_VERSION, directoryPrefix = "kapt2")
+
+        project.build("build", options = options) {
+            assertSuccessful()
+            assertKaptSuccessful()
+            assertContains(":compileKotlin")
+            assertContains(":compileJava")
+        }
+
+        val files = listOf("InternalDummy.kt", "test.kt")
+        kotlin.repeat(2) { i ->
+            project.projectDir.getFileByName(files[i]).appendText(" ")
+
+            project.build("build", options = options) {
+                assertSuccessful()
+                assertKaptSuccessful()
+                assertContains(":compileKotlin")
+                assertContains(":compileJava")
+            }
+        }
+    }
+
     @Test
     fun testInheritedAnnotations() {
         Project("inheritedAnnotations", GRADLE_VERSION, directoryPrefix = "kapt2").build("build") {

plugins/annotation-processing/src/org/jetbrains/kotlin/annotation/processing/AnnotationProcessingExtension.kt

@@ -32,6 +32,7 @@ import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.BindingTrace
 import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisCompletedHandlerExtension
 import java.io.File
+import java.io.IOException
 import java.net.URLClassLoader
 import java.util.*
 import javax.annotation.processing.Processor
@@ -42,8 +43,10 @@ class ClasspathBasedAnnotationProcessingExtension(
         generatedSourcesOutputDir: File,
         classesOutputDir: File,
         javaSourceRoots: List<File>,
-        verboseOutput: Boolean
+        verboseOutput: Boolean,
-) : AbstractAnnotationProcessingExtension(generatedSourcesOutputDir, classesOutputDir, javaSourceRoots, verboseOutput) {
+        incrementalDataFile: File?
+) : AbstractAnnotationProcessingExtension(generatedSourcesOutputDir, 
+                                          classesOutputDir, javaSourceRoots, verboseOutput, incrementalDataFile) {
     override fun loadAnnotationProcessors(): List<Processor> {
         val classLoader = URLClassLoader(annotationProcessingClasspath.map { it.toURI().toURL() }.toTypedArray())
         return ServiceLoader.load(Processor::class.java, classLoader).toList()
@@ -54,8 +57,13 @@ abstract class AbstractAnnotationProcessingExtension(
         val generatedSourcesOutputDir: File,
         val classesOutputDir: File,
         val javaSourceRoots: List<File>,
-        val verboseOutput: Boolean
+        val verboseOutput: Boolean,
+        val incrementalDataFile: File? = null
 ) : AnalysisCompletedHandlerExtension {
+    private companion object {
+        val LINE_SEPARATOR = System.getProperty("line.separator") ?: "\n"
+    }
+    
     private var annotationProcessingComplete = false
     private val messager = KotlinMessager()
 
@@ -65,6 +73,10 @@ abstract class AbstractAnnotationProcessingExtension(
     
     private fun Int.count(noun: String) = if (this == 1) "$this $noun" else "$this ${noun}s"
 
+    private inline fun <T, R> T.runIf(condition: Boolean, block: T.() -> R) {
+        if (condition) block()
+    }
+
     override fun analysisCompleted(
             project: Project,
             module: ModuleDescriptor,
@@ -134,12 +146,13 @@ abstract class AbstractAnnotationProcessingExtension(
     }
 
     private fun KotlinProcessingEnvironment.doAnnotationProcessing(files: Collection<KtFile>): ProcessingResult {
-        run initializeProcessors@ {
+        val allSupportedAnnotationFqNames = run initializeProcessors@ {
             processors.forEach { it.init(this) }
             log { "Initialized processors: " + processors.joinToString { it.javaClass.name } }
+            processors.flatMapTo(mutableSetOf()) { it.supportedAnnotationTypes }
         }
 
-        val firstRoundAnnotations = RoundAnnotations()
+        val firstRoundAnnotations = RoundAnnotations(allSupportedAnnotationFqNames)
         
         run analyzeFilesForFirstRound@ {
             log { "Analysing Kotlin files: " + files.map { it.virtualFile.path } }
@@ -159,13 +172,54 @@ abstract class AbstractAnnotationProcessingExtension(
             }
         }
         
+        runIf(incrementalDataFile != null) processIncrementalData@ {
+            val incrementalDataFile = incrementalDataFile!!
+            
+            
+            runIf(incrementalDataFile.exists()) analyzeFilesFromPreviousIncrementalData@ {
+                val incrementalData = try {
+                    incrementalDataFile.readText()
+                } catch (e: IOException) {
+                    log { "An exception occurred while processing incremental data file: $incrementalDataFile" }
+                    null 
+                }
+
+                if (incrementalData != null) {
+                    val analyzedClasses = mutableListOf<String>()
+                    
+                    for (line in incrementalData.lines()) {
+                        if (line.length < 3 || !line.startsWith("i ")) continue
+                        val fqName = line.drop(2) 
+                        val psiClass = javaPsiFacade.findClass(fqName, projectScope) ?: continue
+                        if (firstRoundAnnotations.analyzeDeclaration(psiClass)) {
+                            analyzedClasses += fqName
+                        }
+                    }
+                    
+                    log { "Analysing files from incremental data: $analyzedClasses" }
+                }
+            }
+            
+            run saveNewIncrementalData@ {
+                val analyzedClasses = firstRoundAnnotations.analyzedClasses
+                log { "Saving incremental data: ${analyzedClasses.size} class names" }
+                try {
+                    incrementalDataFile.parentFile.mkdirs()
+                    incrementalDataFile.writeText(analyzedClasses.map { "i $it" }.joinToString(LINE_SEPARATOR))
+                }
+                catch (e: IOException) {
+                    log { "Unable to write $incrementalDataFile" }
+                }
+            }
+        }
+        
         val finalRoundNumber = run annotationProcessing@ {
             val firstRoundEnvironment = KotlinRoundEnvironment(firstRoundAnnotations, false, 1)
             process(firstRoundEnvironment)
         } + 1
         
         log { "Starting round $finalRoundNumber (final)" }
-        val finalRoundEnvironment = KotlinRoundEnvironment(RoundAnnotations(), true, finalRoundNumber)
+        val finalRoundEnvironment = KotlinRoundEnvironment(RoundAnnotations(allSupportedAnnotationFqNames), true, finalRoundNumber)
         for (processor in processors) {
             processor.process(emptySet(), finalRoundEnvironment)
         }
@@ -200,7 +254,7 @@ abstract class AbstractAnnotationProcessingExtension(
         }
         
         // Start the next round
-        val nextRoundAnnotations = RoundAnnotations().apply { analyzeFiles(psiFiles) }
+        val nextRoundAnnotations = RoundAnnotations(roundEnvironment.supportedAnnotationFqNames).apply { analyzeFiles(psiFiles) }
         val nextRoundEnvironment = KotlinRoundEnvironment(nextRoundAnnotations, false, roundEnvironment.roundNumber + 1)
         return process(nextRoundEnvironment)
     }

plugins/annotation-processing/src/org/jetbrains/kotlin/annotation/processing/AnnotationProcessingPlugin.kt

@@ -42,6 +42,9 @@ object AnnotationProcessingConfigurationKeys {
     val ANNOTATION_PROCESSOR_CLASSPATH: CompilerConfigurationKey<List<String>> =
             CompilerConfigurationKey.create<List<String>>("annotation processor classpath")
 
+    val INCREMENTAL_DATA_FILE: CompilerConfigurationKey<String> =
+            CompilerConfigurationKey.create<String>("data file for incremental compilation support")
+
     val VERBOSE_MODE: CompilerConfigurationKey<String> = 
             CompilerConfigurationKey.create<String>("verbose mode")
 }
@@ -60,6 +63,9 @@ class AnnotationProcessingCommandLineProcessor : CommandLineProcessor {
                 CliOption("apclasspath", "<classpath>", "Annotation processor classpath", 
                           required = false, allowMultipleOccurrences = true)
 
+        val INCREMENTAL_DATA_FILE_OPTION: CliOption =
+                CliOption("incrementalData", "<path>", "Location of the incremental data file", required = false)
+
         val VERBOSE_MODE_OPTION: CliOption =
                 CliOption("verbose", "true | false", "Enable verbose output", required = false)
     }
@@ -67,7 +73,8 @@ class AnnotationProcessingCommandLineProcessor : CommandLineProcessor {
     override val pluginId: String = ANNOTATION_PROCESSING_COMPILER_PLUGIN_ID
 
     override val pluginOptions: Collection<CliOption> =
-            listOf(GENERATED_OUTPUT_DIR_OPTION, ANNOTATION_PROCESSOR_CLASSPATH_OPTION, CLASS_FILES_OUTPUT_DIR_OPTION, VERBOSE_MODE_OPTION)
+            listOf(GENERATED_OUTPUT_DIR_OPTION, ANNOTATION_PROCESSOR_CLASSPATH_OPTION,
+                   CLASS_FILES_OUTPUT_DIR_OPTION, INCREMENTAL_DATA_FILE_OPTION, VERBOSE_MODE_OPTION)
 
     override fun processOption(option: CliOption, value: String, configuration: CompilerConfiguration) {
         when (option) {
@@ -78,6 +85,7 @@ class AnnotationProcessingCommandLineProcessor : CommandLineProcessor {
             }
             GENERATED_OUTPUT_DIR_OPTION -> configuration.put(AnnotationProcessingConfigurationKeys.GENERATED_OUTPUT_DIR, value)
             CLASS_FILES_OUTPUT_DIR_OPTION -> configuration.put(AnnotationProcessingConfigurationKeys.CLASS_FILES_OUTPUT_DIR, value)
+            INCREMENTAL_DATA_FILE_OPTION -> configuration.put(AnnotationProcessingConfigurationKeys.INCREMENTAL_DATA_FILE, value)
             VERBOSE_MODE_OPTION -> configuration.put(AnnotationProcessingConfigurationKeys.VERBOSE_MODE, value)
             else -> throw CliOptionProcessingException("Unknown option: ${option.name}")
         }
@@ -95,6 +103,8 @@ class AnnotationProcessingComponentRegistrar : ComponentRegistrar {
     override fun registerProjectComponents(project: MockProject, configuration: CompilerConfiguration) {
         val generatedOutputDir = configuration.get(AnnotationProcessingConfigurationKeys.GENERATED_OUTPUT_DIR) ?: return
         val apClasspath = configuration.get(AnnotationProcessingConfigurationKeys.ANNOTATION_PROCESSOR_CLASSPATH)?.map(::File) ?: return
+        
+        val incrementalDataFile = configuration.get(AnnotationProcessingConfigurationKeys.INCREMENTAL_DATA_FILE)?.let(::File)
 
         val generatedOutputDirFile = File(generatedOutputDir)
         generatedOutputDirFile.mkdirs()
@@ -117,7 +127,8 @@ class AnnotationProcessingComponentRegistrar : ComponentRegistrar {
                 .registerExtension(DefaultErrorMessagesAnnotationProcessing())
         
         val annotationProcessingExtension = ClasspathBasedAnnotationProcessingExtension(
-                classpath, generatedOutputDirFile, classesOutputDir, javaRoots, verboseOutput)
+                classpath, generatedOutputDirFile, classesOutputDir, javaRoots, verboseOutput, incrementalDataFile)
+        
         AnalysisCompletedHandlerExtension.registerExtension(project, annotationProcessingExtension)
     }
 }

plugins/annotation-processing/src/org/jetbrains/kotlin/annotation/processing/RoundAnnotations.kt

@@ -22,12 +22,19 @@ import org.jetbrains.kotlin.asJava.toLightClass
 import org.jetbrains.kotlin.java.model.internal.getAnnotationsWithInherited
 import org.jetbrains.kotlin.psi.KtClassOrObject
 import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi.psiUtil.getParentOfType
 
-internal class RoundAnnotations() {
+internal class RoundAnnotations(val supportedAnnotationFqNames: Set<String>) {
     private val mutableAnnotationsMap = mutableMapOf<String, MutableList<PsiModifierListOwner>>()
+    private val mutableAnalyzedClasses = mutableSetOf<String>()
+    
+    private val acceptsAnyAnnotation = "*" in supportedAnnotationFqNames
 
     val annotationsMap: Map<String, List<PsiModifierListOwner>>
         get() = mutableAnnotationsMap
+    
+    val analyzedClasses: Set<String>
+        get() = mutableAnalyzedClasses
 
     fun analyzeFiles(files: Collection<KtFile>) = files.forEach { analyzeFile(it) }
     
@@ -51,13 +58,28 @@ internal class RoundAnnotations() {
     fun analyzeFile(file: PsiJavaFile) {
         file.classes.forEach { analyzeDeclaration(it) }
     }
+    
+    fun PsiElement.getTopLevelClassParent(): PsiClass? = when (this) {
+        is PsiClass -> containingClass?.let { it.getTopLevelClassParent() } ?: this
+        else -> getParentOfType<PsiClass>(true)?.getTopLevelClassParent()
+    }
 
-    fun analyzeDeclaration(declaration: PsiElement) {
+    fun analyzeDeclaration(declaration: PsiElement): Boolean {
-        if (declaration !is PsiModifierListOwner) return
+        if (declaration !is PsiModifierListOwner) return false
+        
+        // Do not analyze classes twice (for incremental compilation data)
+        if (declaration is PsiClass && declaration.qualifiedName in analyzedClasses) return false
 
         for (annotation in declaration.getAnnotationsWithInherited()) {
-            val fqName = annotation.qualifiedName ?: return
+            val fqName = annotation.qualifiedName ?: continue
+            if (!acceptsAnyAnnotation && fqName !in supportedAnnotationFqNames) continue
             mutableAnnotationsMap.getOrPut(fqName, { mutableListOf() }).add(declaration)
+
+            // Add only top-level classes
+            val topLevelClassQualifiedName = declaration.getTopLevelClassParent()?.qualifiedName
+            if (topLevelClassQualifiedName != null) {
+                mutableAnalyzedClasses += topLevelClassQualifiedName
+            }
         }
 
         if (declaration is PsiClass) {
@@ -71,5 +93,7 @@ internal class RoundAnnotations() {
                 analyzeDeclaration(parameter)
             }
         }
+        
+        return true
     }
 }

plugins/annotation-processing/src/org/jetbrains/kotlin/annotation/processing/impl/KotlinRoundEnvironment.kt

@@ -33,6 +33,9 @@ internal class KotlinRoundEnvironment(
     internal val annotationsMap: Map<String, List<PsiModifierListOwner>>
         get() = roundAnnotations.annotationsMap
     
+    internal val supportedAnnotationFqNames: Set<String>
+        get() = roundAnnotations.supportedAnnotationFqNames
+    
     override fun getRootElements() = emptySet<Element>()
     
     override fun processingOver() = isProcessingOver

plugins/annotation-processing/testData/processors/IncrementalDataSimple.kt

@@ -0,0 +1,44 @@
+annotation class Anno
+
+@Anno
+class Test
+
+@Anno
+interface Intf
+
+class Test2 {
+    @Anno
+    fun test2Fun() {}
+}
+
+class Test3 {
+    @Anno
+    class Test3Nested
+    
+    @Anno
+    inner class Test3Inner
+}
+
+class Test4
+
+class Test5 {
+    fun test5Fun() {} 
+}
+
+annotation class Surprise
+
+@Surprise
+class Test6
+
+class Test7 {
+    @Surprise
+    fun test7Fun(): String = "ABC"
+}
+
+class Test8 {
+    @Anno
+    fun test8Fun1() {}
+
+    @Anno
+    fun test8Fun2() {}
+}

plugins/plugins-tests/tests/org/jetbrains/kotlin/annotation/processing/test/processor/AbstractProcessorTest.kt

@@ -21,7 +21,6 @@ import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.codegen.AbstractBytecodeTextTest
 import org.jetbrains.kotlin.codegen.CodegenTestUtil
-import org.jetbrains.kotlin.java.model.JeName
 import org.jetbrains.kotlin.java.model.elements.JeAnnotationMirror
 import org.jetbrains.kotlin.java.model.elements.JeMethodExecutableElement
 import org.jetbrains.kotlin.java.model.elements.JeTypeElement
@@ -40,13 +39,17 @@ import javax.lang.model.element.*
 
 class AnnotationProcessingExtensionForTests(
         val processors: List<Processor>
-) : AbstractAnnotationProcessingExtension(createTempDir(), createTempDir(), listOf(), true) {
+) : AbstractAnnotationProcessingExtension(createTempDir(), createTempDir(), listOf(), true, createIncrementalDataFile()) {
     override fun loadAnnotationProcessors() = processors
     
     private companion object {
         fun createTempDir(): File = Files.createTempDirectory("ap-test").toFile().apply {
             deleteOnExit()
         }
+        
+        fun createIncrementalDataFile(): File = File.createTempFile("incrementalData", "txt").apply {
+            deleteOnExit()
+        }
     }
 }
 

plugins/plugins-tests/tests/org/jetbrains/kotlin/annotation/processing/test/processor/ProcessorTests.kt

@@ -16,9 +16,11 @@
 
 package org.jetbrains.kotlin.annotation.processing.test.processor
 
+import org.intellij.lang.annotations.Language
 import org.jetbrains.kotlin.java.model.elements.*
 import org.jetbrains.kotlin.java.model.types.JeDeclaredType
 import org.jetbrains.kotlin.java.model.types.JeMethodExecutableTypeMirror
+import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisCompletedHandlerExtension
 import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
 import javax.lang.model.element.AnnotationMirror
 import javax.lang.model.type.DeclaredType
@@ -184,4 +186,18 @@ class ProcessorTests : AbstractProcessorTest() {
         check("g", "<D>(D)void")
         check("g", "(java.lang.String)void", erasure)
     }
+    
+    fun testIncrementalDataSimple() = incrementalDataTest(
+            "IncrementalDataSimple",
+            "i Intf, i Test, i Test2, i Test3, i Test8")
+    
+    private fun incrementalDataTest(fileName: String, @Language("TEXT") expectedText: String) {
+        test(fileName, "Anno", "Anno2", "Anno3") { set, roundEnv, env -> }
+        val ext = AnalysisCompletedHandlerExtension.getInstances(myEnvironment.project)
+                .firstIsInstance<AnnotationProcessingExtensionForTests>()
+        val incrementalDataFile = ext.incrementalDataFile
+        assertNotNull(incrementalDataFile)
+        val text = incrementalDataFile!!.readText().lines().sorted().joinToString(", ")
+        assertEquals(expectedText, text)
+    }
 }