Update kapt annotations file incrementally

build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt

@@ -122,6 +122,11 @@ open class IncrementalCacheImpl<Target>(
         }
     }
 
+    // used in gradle
+    @Suppress("unused")
+    fun classesBySources(sources: Iterable<File>): Iterable<JvmClassName> =
+            sources.flatMap { sourceToClassesMap[it] }
+
     fun getSubtypesOf(className: FqName): Sequence<FqName> =
             subtypesMap[className].asSequence()
 

libraries/tools/kotlin-annotation-processing/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotatedElementDescriptor.kt

@@ -2,7 +2,9 @@ package org.jetbrains.kotlin.annotation
 
 sealed class AnnotatedElementDescriptor(val classFqName: String) {
     class Class(classFqName: String) : AnnotatedElementDescriptor(classFqName) {
-        // use referential equality
+        override fun equals(other: Any?) = other is Class && classFqName == other.classFqName
+
+        override fun hashCode() = classFqName.hashCode()
     }
 
     class Method(classFqName: String, val methodName: String) : AnnotatedElementDescriptor(classFqName) {
@@ -15,7 +17,10 @@ sealed class AnnotatedElementDescriptor(val classFqName: String) {
         companion object {
             const val METHOD_NAME = "<init>"
         }
-        // use referential equality
+
+        override fun equals(other: Any?) = other is Constructor && classFqName == other.classFqName
+
+        override fun hashCode() = 31 * classFqName.hashCode() + METHOD_NAME.hashCode()
     }
 
     class Field(classFqName: String, val fieldName: String) : AnnotatedElementDescriptor(classFqName) {

libraries/tools/kotlin-annotation-processing/src/main/kotlin/org/jetbrains/kotlin/annotation/KotlinAnnotationProvider.kt

@@ -22,12 +22,12 @@ import java.io.StringReader
 import java.util.*
 import org.jetbrains.kotlin.annotation.CompactNotationType as Notation
 
-class KotlinAnnotationProvider(annotationsReader: Reader) {
+open class KotlinAnnotationProvider(annotationsReader: Reader) {
     constructor(annotationsFile: File) : this(annotationsFile.reader().buffered())
     constructor() : this(StringReader(""))
 
-    private val kotlinClassesInternal = hashSetOf<String>()
+    protected val kotlinClassesInternal = hashSetOf<String>()
-    private val annotatedKotlinElementsInternal = hashMapOf<String, MutableSet<AnnotatedElementDescriptor>>()
+    protected val annotatedKotlinElementsInternal = hashMapOf<String, MutableSet<AnnotatedElementDescriptor>>()
 
     init {
         readAnnotations(annotationsReader)
@@ -42,7 +42,7 @@ class KotlinAnnotationProvider(annotationsReader: Reader) {
     val supportInheritedAnnotations: Boolean
         get() = kotlinClassesInternal.isNotEmpty()
 
-    private fun readAnnotations(annotationsReader: Reader) {
+    protected fun readAnnotations(annotationsReader: Reader) {
         fun handleShortenedName(cache: MutableMap<String, String>, lineParts: List<String>) {
             val name = lineParts[1]
             val id = lineParts[2]

libraries/tools/kotlin-annotation-processing/src/main/kotlin/org/jetbrains/kotlin/annotation/MutableKotlinAnnotationProvider.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.annotation
+
+import java.io.File
+import java.io.StringReader
+import org.jetbrains.kotlin.annotation.CompactNotationType as Notation
+
+class MutableKotlinAnnotationProvider : KotlinAnnotationProvider(StringReader("")) {
+    fun addAnnotationsFrom(file: File) {
+        file.bufferedReader().use { readAnnotations(it) }
+    }
+
+    fun removeClasses(classesFqNames: Set<String>) {
+        kotlinClassesInternal.removeAll(classesFqNames)
+
+        for ((annotation, elements) in annotatedKotlinElementsInternal) {
+            elements.removeAll { it.classFqName in classesFqNames }
+        }
+    }
+}

libraries/tools/kotlin-annotation-processing/src/test/kotlin/org/jetbrains/kotlin/annotation/AnnotationListParseTest.kt

@@ -1,7 +1,5 @@
 package org.jetbrains.kotlin.annotation
 
-import org.junit.Assert
-import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import java.io.File
@@ -84,20 +82,4 @@ open class AnnotationListParseTest {
         val fileContents = (actualAnnotationsSorted + classDeclarationsSorted).joinToString("\n")
         assertEqualsToFile(expectedFile, fileContents)
     }
-
-    // KotlinTestUtils.assertEqualsToFile() is not reachable from here
-    public fun assertEqualsToFile(expectedFile: File, actual: String) {
-        val lineSeparator = System.getProperty("line.separator")
-        val actualText = actual.replace(lineSeparator, "\n").trim('\n', ' ', '\t')
-
-        if (!expectedFile.exists()) {
-            expectedFile.writeText(actualText.replace("\n", lineSeparator))
-            Assert.fail("Expected data file did not exist. Generating: " + expectedFile)
-        }
-
-        val expectedText = expectedFile.readText().replace(lineSeparator, "\n")
-
-        assertEquals(expectedText, actualText)
-    }
-
 }

libraries/tools/kotlin-annotation-processing/src/test/kotlin/org/jetbrains/kotlin/annotation/AnnotationSerializationTest.kt

@@ -29,14 +29,10 @@ class AnnotationSerializationTest : AnnotationListParseTest() {
         val annotationProvider = KotlinAnnotationProvider(annotationsFile)
         annotationsFile.delete()
 
-        val writer = annotationsFile.bufferedWriter()
+        annotationsFile.bufferedWriter().use { writer ->
-        try {
             val annotationWriter = CompactAnnotationWriter(writer)
             annotationProvider.writeAnnotations(annotationWriter)
         }
-        finally {
-            writer.close()
-        }
 
         super.doTest(workingDir)
     }

libraries/tools/kotlin-annotation-processing/src/test/kotlin/org/jetbrains/kotlin/annotation/MutableAnnotationProviderTest.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.annotation
+
+import org.junit.Test
+import java.io.File
+import java.io.StringWriter
+
+class MutableAnnotationProviderTest {
+    private val resourcesRootFile = File("src/test/resources/mutate")
+
+    @Test
+    fun testRemoveClass() {
+        val testDir = File(resourcesRootFile, "removeClass")
+        val annotationsFile = File(testDir, "annotations.txt")
+        val annotationsFileUpdated = File(testDir, "annotations-updated.txt")
+
+        val content = mutateAnnotationsFiles {
+            addAnnotationsFrom(annotationsFile)
+            removeClasses(setOf("foo.A"))
+        }
+
+        assertEqualsToFile(annotationsFileUpdated, content)
+    }
+
+    @Test
+    fun testMergeFiles() {
+        val testDir = File(resourcesRootFile, "mergeFiles")
+        val annotationsAFile = File(testDir, "annotationsA.txt")
+        val annotationsBFile = File(testDir, "annotationsB.txt")
+        val annotationsMergedFile = File(testDir, "annotations-merged.txt")
+
+        val content = mutateAnnotationsFiles {
+            addAnnotationsFrom(annotationsAFile)
+            addAnnotationsFrom(annotationsBFile)
+        }
+
+        assertEqualsToFile(annotationsMergedFile, content)
+    }
+
+    private fun mutateAnnotationsFiles(fn: MutableKotlinAnnotationProvider.() -> Unit): String {
+        val annotationProvider = MutableKotlinAnnotationProvider()
+        annotationProvider.fn()
+
+        val writer = StringWriter()
+        annotationProvider.writeAnnotations(CompactAnnotationWriter(writer))
+        return writer.toString()
+    }
+}

libraries/tools/kotlin-annotation-processing/src/test/kotlin/org/jetbrains/kotlin/annotation/testUtils.kt

@@ -0,0 +1,18 @@
+package org.jetbrains.kotlin.annotation
+
+import org.junit.Assert
+import java.io.File
+
+fun assertEqualsToFile(expectedFile: File, actual: String) {
+    val lineSeparator = System.getProperty("line.separator")
+    val actualText = actual.replace(lineSeparator, "\n").trim('\n', ' ', '\t')
+
+    if (!expectedFile.exists()) {
+        expectedFile.writeText(actualText.replace("\n", lineSeparator))
+        Assert.fail("Expected data file did not exist. Generating: " + expectedFile)
+    }
+
+    val expectedText = expectedFile.readText().replace(lineSeparator, "\n")
+
+    Assert.assertEquals(expectedText, actualText)
+}

libraries/tools/kotlin-annotation-processing/src/test/resources/mutate/mergeFiles/annotations-merged.txt

@@ -0,0 +1,15 @@
+a annotations.Ann 0
+p foo 0
+m 0 0/A funA
+p bar 1
+c 0 1/B
+c 0 0/A
+m 0 1/B funB
+m 0 0/A valA$annotations
+m 0 1/B valB$annotations
+a java.lang.annotation.Retention 1
+p annotations 2
+c 1 2/Ann
+d 0/A
+d 1/B
+d 2/Ann

libraries/tools/kotlin-annotation-processing/src/test/resources/mutate/mergeFiles/annotationsA.txt

@@ -0,0 +1,10 @@
+p foo 0
+d 0/A
+a annotations.Ann 0
+c 0 0/A
+m 0 0/A valA$annotations
+m 0 0/A funA
+p annotations 1
+d 1/Ann
+a java.lang.annotation.Retention 1
+c 1 1/Ann

libraries/tools/kotlin-annotation-processing/src/test/resources/mutate/mergeFiles/annotationsB.txt

@@ -0,0 +1,10 @@
+p bar 0
+d 0/B
+a annotations.Ann 0
+c 0 0/B
+m 0 0/B valB$annotations
+m 0 0/B funB
+p annotations 1
+d 1/Ann
+a java.lang.annotation.Retention 1
+c 1 1/Ann

libraries/tools/kotlin-annotation-processing/src/test/resources/mutate/removeClass/annotations-updated.txt

@@ -0,0 +1,10 @@
+a annotations.Ann 0
+p bar 0
+c 0 0/B
+m 0 0/B funB
+m 0 0/B valB$annotations
+a java.lang.annotation.Retention 1
+p annotations 1
+c 1 1/Ann
+d 0/B
+d 1/Ann

libraries/tools/kotlin-annotation-processing/src/test/resources/mutate/removeClass/annotations.txt

@@ -0,0 +1,15 @@
+p bar 0
+d 0/B
+a annotations.Ann 0
+c 0 0/B
+m 0 0/B valB$annotations
+m 0 0/B funB
+p foo 1
+d 1/A
+c 0 1/A
+m 0 1/A valA$annotations
+m 0 1/A funA
+p annotations 2
+d 2/Ann
+a java.lang.annotation.Retention 1
+c 1 2/Ann

libraries/tools/kotlin-annotation-processing/src/test/resources/parse/constructors/parsed.txt

@@ -1,2 +1 @@
-java.lang.Deprecated org.test.SomeClass <init>
 java.lang.Deprecated org.test.SomeClass <init>

libraries/tools/kotlin-gradle-plugin-core/pom.xml

@@ -47,6 +47,11 @@
             <artifactId>kotlin-compiler-embeddable</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-annotation-processing</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib</artifactId>

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt

@@ -0,0 +1,61 @@
+package org.jetbrains.kotlin.annotation
+
+import org.gradle.api.logging.Logging
+import org.jetbrains.kotlin.gradle.tasks.kotlinDebug
+import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+import java.io.File
+import java.util.*
+
+/**
+ * Annotation file is generated by collecting annotated elements of generated files.
+ * When compiling incrementally, the compiler generates only subset of all classes,
+ * so after compilation annotation file contains only a subset of annotated elements,
+ * which breaks the build.
+ *
+ * The workaround is to:
+ * 1. backup old file before incremental compilation;
+ * 2. after each iteration of IC:
+ *  2.1 remove classes corresponding to dirty source files
+ *  2.2 add annotations from newly generated annotations file
+ */
+class AnnotationFileUpdater(private val generatedAnnotationFile: File) {
+    private val logger = Logging.getLogger(this.javaClass)
+    private val lastSuccessfullyUpdatedFile = File.createTempFile("kapt-annotations-copy", "tmp")
+
+    init {
+        if (generatedAnnotationFile.exists()) {
+            generatedAnnotationFile.copyTo(lastSuccessfullyUpdatedFile, overwrite = true)
+        }
+        else {
+            lastSuccessfullyUpdatedFile.writeText("")
+        }
+
+        lastSuccessfullyUpdatedFile.deleteOnExit()
+    }
+
+    fun updateAnnotations(outdatedClasses: Iterable<JvmClassName>) {
+        val outdatedClassesFqNames = outdatedClasses.mapTo(HashSet<String>()) { it.fqNameForClassNameWithoutDollars.asString() }
+
+        val annotationsProvider = MutableKotlinAnnotationProvider().apply {
+            addAnnotationsFrom(lastSuccessfullyUpdatedFile)
+            removeClasses(outdatedClassesFqNames)
+            logger.kotlinDebug { "Removed annotation entries for fq-names [${outdatedClassesFqNames.joinToString()}]" }
+
+            if (generatedAnnotationFile.exists()) {
+                addAnnotationsFrom(generatedAnnotationFile)
+                logger.kotlinDebug { "Added annotation entries from $generatedAnnotationFile" }
+            }
+        }
+
+        generatedAnnotationFile.delete()
+        generatedAnnotationFile.bufferedWriter().use { writer ->
+            annotationsProvider.writeAnnotations(CompactAnnotationWriter(writer))
+            logger.kotlinDebug { "Written updated annotations to $generatedAnnotationFile" }
+        }
+        generatedAnnotationFile.copyTo(lastSuccessfullyUpdatedFile, overwrite = true)
+    }
+
+    fun revert() {
+        lastSuccessfullyUpdatedFile.copyTo(generatedAnnotationFile, overwrite = true)
+    }
+}

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -21,6 +21,7 @@ import org.gradle.api.tasks.SourceTask
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.compile.AbstractCompile
 import org.gradle.api.tasks.incremental.IncrementalTaskInputs
+import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.cli.common.CLICompiler
 import org.jetbrains.kotlin.cli.common.ExitCode
@@ -173,6 +174,8 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
                gradleCacheVersion(taskBuildDirectory))
     }
 
+    private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null
+
     override fun populateTargetSpecificArgs(args: K2JVMCompilerArguments) {
         // show kotlin compiler where to look for java source files
 //        args.freeArgs = (args.freeArgs + getJavaSourceRoots().map { it.getAbsolutePath() }).toSet().toList()
@@ -370,9 +373,15 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             // TODO: process as list here, merge into string later
             args.classpath = args.classpath + File.pathSeparator + outputDir.absolutePath
         }
+        else {
+            // there is no point in updating annotation file since all files will be compiled anyway
+            kaptAnnotationsFileUpdater = null
+        }
 
         while (sourcesToCompile.any() || currentRemoved.any()) {
             val removedAndModified = (sourcesToCompile + currentRemoved).toList()
+            val outdatedClasses = targets.flatMap { getIncrementalCache(it).classesBySources(removedAndModified) }
+
             targets.forEach { getIncrementalCache(it).let {
                 it.markOutputClassesDirty(removedAndModified)
                 it.removeClassfilesBySources(removedAndModified)
@@ -393,6 +402,10 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
 
             if (exitCode == ExitCode.OK) {
                 dirtySourcesSinceLastTimeFile.delete()
+                kaptAnnotationsFileUpdater?.updateAnnotations(outdatedClasses)
+            }
+            else {
+                kaptAnnotationsFileUpdater?.revert()
             }
 
             allGeneratedFiles.addAll(generatedFiles)
@@ -485,6 +498,10 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
     private fun handleKaptProperties(extraProperties: ExtraPropertiesExtension, pluginOptions: MutableList<String>) {
         val kaptAnnotationsFile = extraProperties.getOrNull<File>("kaptAnnotationsFile")
         if (kaptAnnotationsFile != null) {
+            if (incremental) {
+                kaptAnnotationsFileUpdater = AnnotationFileUpdater(kaptAnnotationsFile)
+            }
+
             if (kaptAnnotationsFile.exists()) kaptAnnotationsFile.delete()
             pluginOptions.add("plugin:$ANNOTATIONS_PLUGIN_NAME:output=" + kaptAnnotationsFile)
         }