Incremental KAPT - track constants usage

This commit adds support for tracking of used constants in
source files. For every constant used in a source file, class
that defines the constant and the constant name are tracked.

Value of the constant can be obtained using
annotation processing APIs, so if the constant value changes, a source file
using it has to be reprocessed.

 #KT-23880

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/annotationProcessing.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.kapt3.base
 
+import com.sun.source.util.Trees
 import com.sun.tools.javac.comp.CompileStates.CompileState
 import com.sun.tools.javac.main.JavaCompiler
 import com.sun.tools.javac.processing.AnnotationProcessingError
@@ -55,7 +56,7 @@ fun KaptContext.doAnnotationProcessing(
         val sourcesStructureListener = cacheManager?.let {
             if (processors.any { it.kind == DeclaredProcType.NON_INCREMENTAL }) return@let null
 
-            val recordTypesListener = MentionedTypesTaskListener(cacheManager.javaCache, processingEnvironment.elementUtils)
+            val recordTypesListener = MentionedTypesTaskListener(cacheManager.javaCache, processingEnvironment.elementUtils, Trees.instance(processingEnvironment))
             compiler.getTaskListeners().add(recordTypesListener)
             recordTypesListener
         }

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/incremental/classStructureCache.kt

@@ -51,6 +51,12 @@ class JavaClassCache() : Serializable {
                 dependants.add(sourceInfo.sourceFile)
                 dependencyCache[mentionedType] = dependants
             }
+            // Treat referred constants as ABI dependencies until we start supporting per-constant classpath updates.
+            for (mentionedConstants in sourceInfo.getMentionedConstants().keys) {
+                val dependants = dependencyCache[mentionedConstants] ?: mutableSetOf()
+                dependants.add(sourceInfo.sourceFile)
+                dependencyCache[mentionedConstants] = dependants
+            }
         }
         nonTransitiveCache = HashMap(sourceCache.size * 2)
         for (sourceInfo in sourceCache.values) {
@@ -67,7 +73,7 @@ class JavaClassCache() : Serializable {
         output.writeObject(generatedTypes)
     }
 
-    fun isAlreadyProcessed(sourceFile: URI) = sourceCache.containsKey(sourceFile)
+    fun isAlreadyProcessed(sourceFile: URI) = sourceCache.containsKey(sourceFile) || generatedTypes.containsKey(File(sourceFile))
 
     /** Used for testing only. */
     internal fun getStructure(sourceFile: File) = sourceCache[sourceFile.toURI()]
@@ -78,7 +84,18 @@ class JavaClassCache() : Serializable {
      * */
     fun invalidateEntriesForChangedFiles(changes: Changes): SourcesToReprocess {
         val allDirtyFiles = mutableSetOf<URI>()
-        var currentDirtyFiles = changes.sourceChanges.map { it.toURI() }
+        var currentDirtyFiles = changes.sourceChanges.map { it.toURI() }.toMutableSet()
+
+        for (classpathFqName in changes.dirtyFqNamesFromClasspath) {
+            nonTransitiveCache[classpathFqName]?.let {
+                allDirtyFiles.addAll(it)
+            }
+
+            dependencyCache[classpathFqName]?.let {
+                currentDirtyFiles.addAll(it)
+            }
+        }
+
         val allDirtyTypes = mutableSetOf<String>()
 
         // check for constants first because they cause full rebuilt
@@ -112,7 +129,7 @@ class JavaClassCache() : Serializable {
                 }
             }
 
-            currentDirtyFiles = nextRound.filter { !allDirtyFiles.contains(it) }
+            currentDirtyFiles = nextRound.filter { !allDirtyFiles.contains(it) }.toMutableSet()
         }
 
         return SourcesToReprocess.Incremental(allDirtyFiles.map { File(it) }, allDirtyTypes)
@@ -161,12 +178,14 @@ class SourceFileStructure(
 
     private val definedConstants: MutableMap<String, Any> = mutableMapOf()
     private val mentionedAnnotations: MutableSet<String> = mutableSetOf()
+    private val mentionedConstants: MutableMap<String, MutableSet<String>> = mutableMapOf()
 
     fun getDeclaredTypes(): Set<String> = declaredTypes
     fun getMentionedTypes(): Set<String> = mentionedTypes
     fun getPrivateTypes(): Set<String> = privateTypes
     fun getDefinedConstants(): Map<String, Any> = definedConstants
     fun getMentionedAnnotations(): Set<String> = mentionedAnnotations
+    fun getMentionedConstants(): Map<String, Set<String>> = mentionedConstants
 
     fun addDeclaredType(declaredType: String) {
         declaredTypes.add(declaredType)
@@ -189,6 +208,14 @@ class SourceFileStructure(
     fun addPrivateType(name: String) {
         privateTypes.add(name)
     }
+
+    fun addMentionedConstant(containingClass: String, name: String) {
+        if (!declaredTypes.contains(containingClass)) {
+            val names = mentionedConstants[containingClass] ?: HashSet()
+            names.add(name)
+            mentionedConstants[containingClass] = names
+        }
+    }
 }
 
 

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/incremental/javacVisitors.kt

@@ -6,9 +6,7 @@
 package org.jetbrains.kotlin.kapt3.base.incremental
 
 import com.sun.source.tree.*
-import com.sun.source.util.SimpleTreeVisitor
-import com.sun.source.util.TaskEvent
-import com.sun.source.util.TaskListener
+import com.sun.source.util.*
 import com.sun.tools.javac.code.Symbol
 import com.sun.tools.javac.tree.JCTree
 import java.io.File
@@ -19,7 +17,8 @@ import javax.lang.model.util.Elements
 
 class MentionedTypesTaskListener(
     private val cache: JavaClassCache,
-    private val elementUtils: Elements
+    private val elementUtils: Elements,
+    private val trees: Trees
 ) : TaskListener {
 
     var time = 0L
@@ -35,7 +34,7 @@ class MentionedTypesTaskListener(
 
         val structure = SourceFileStructure(e.sourceFile.toUri())
 
-        val treeVisitor = TypeTreeVisitor(elementUtils, structure)
+        val treeVisitor = TypeTreeVisitor(elementUtils, trees, compilationUnit, structure)
         compilationUnit.typeDecls.forEach {
             it.accept(treeVisitor, Visibility.ABI)
         }
@@ -48,9 +47,11 @@ private enum class Visibility {
     ABI, NON_ABI
 }
 
-private class TypeTreeVisitor(val elementUtils: Elements, val sourceStructure: SourceFileStructure) :
+private class TypeTreeVisitor(val elementUtils: Elements, val trees: Trees, val compilationUnit: CompilationUnitTree,  val sourceStructure: SourceFileStructure) :
     SimpleTreeVisitor<Void, Visibility>() {
 
+    val constantTreeVisitor = ConstantTreeVisitor(sourceStructure)
+
     /** Handle annotations on this class, including the @Inherited ones as those are not visible using Tree APIs. */
     private fun handleClassAnnotations(classSymbol: Symbol.ClassSymbol) {
         elementUtils.getAllAnnotationMirrors(classSymbol).forEach {
@@ -90,6 +91,9 @@ private class TypeTreeVisitor(val elementUtils: Elements, val sourceStructure: S
         visit(node.returnType, methodVisibility)
         node.parameters.forEach { visit(it, methodVisibility) }
         visit(node.defaultValue, methodVisibility)
+        node.defaultValue?.let {
+            constantTreeVisitor.scan(trees.getPath(compilationUnit, it), null)
+        }
         node.throws.forEach { visit(it, methodVisibility) }
         node.typeParameters.forEach { visit(it, methodVisibility) }
 
@@ -109,6 +113,8 @@ private class TypeTreeVisitor(val elementUtils: Elements, val sourceStructure: S
             val flags = node.modifiers.getFlags()
 
             node.sym.constValue?.let { constValue ->
+                constantTreeVisitor.scan(trees.getPath(compilationUnit, node.init), null)
+
                 if (flags.contains(Modifier.FINAL)
                     && flags.contains(Modifier.STATIC)
                     && !flags.contains(Modifier.PRIVATE)
@@ -149,7 +155,10 @@ private class TypeTreeVisitor(val elementUtils: Elements, val sourceStructure: S
 
     override fun visitAnnotation(node: AnnotationTree, visibility: Visibility): Void? {
         visit(node.annotationType, visibility)
-        node.arguments.forEach { visit(it, visibility) }
+        node.arguments.forEach {
+            visit(it, visibility)
+            constantTreeVisitor.scan(TreePath.getPath(compilationUnit, it), null)
+        }
 
         return null
     }
@@ -194,6 +203,37 @@ private class TypeTreeVisitor(val elementUtils: Elements, val sourceStructure: S
     }
 }
 
+/**
+ * Visits a constant initializer expression, and extracts all references to constants, either through field select (A.MY_FIELD) or
+ * identifier (MY_FIELD, which happens if A.MY_FIELD is statically imported).
+ */
+private class ConstantTreeVisitor(val sourceStructure: SourceFileStructure) : TreePathScanner<Void, Void>() {
+
+
+    override fun visitAssignment(node: AssignmentTree, p: Void?): Void? {
+        // Annotation element values are in "element = expression" form, and we only want to analyze "expression" part. So ignore variable.
+        scan(node.expression, p)
+        return null
+    }
+
+    override fun visitMemberSelect(node: MemberSelectTree, p: Void?): Void? {
+        addConstantSymbol((node as JCTree.JCFieldAccess).sym)
+        return null
+    }
+
+    override fun visitIdentifier(node: IdentifierTree, p: Void?): Void? {
+        addConstantSymbol((node as JCTree.JCIdent).sym)
+        return null
+    }
+
+    private fun addConstantSymbol(sym: Symbol) {
+        val name = sym.name
+        val containingClass = sym.owner
+
+        sourceStructure.addMentionedConstant(containingClass.qualifiedName.toString(), name.toString())
+    }
+}
+
 class GeneratedTypesTaskListener(private val cache: JavaClassCache) : TaskListener {
 
     override fun started(e: TaskEvent) {

plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/InheritedAnnotationTest.kt

@@ -44,7 +44,7 @@ class TestInheritedAnnotation {
                 srcFiles,
                 listOf(processor),
                 generatedSources
-            ) { trees -> MentionedTypesTaskListener(cache.javaCache, trees) }
+            ) { elementUtils, trees -> MentionedTypesTaskListener(cache.javaCache, elementUtils, trees) }
             cache.updateCache(listOf(processor))
         }
     }

plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/ReferencedContantsTest.kt

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.kapt.base.test.org.jetbrains.kotlin.kapt3.base.incremental
+
+import org.jetbrains.kotlin.kapt3.base.incremental.JavaClassCacheManager
+import org.jetbrains.kotlin.kapt3.base.incremental.MentionedTypesTaskListener
+import org.junit.Assert.assertEquals
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import java.io.File
+
+private val MY_TEST_DIR = File("plugins/kapt3/kapt3-base/testData/runner/incremental/constants")
+
+class ReferencedConstantsTest {
+
+    companion object {
+        @ClassRule
+        @JvmField
+        var tmp = TemporaryFolder()
+
+        private lateinit var cache: JavaClassCacheManager
+        private lateinit var generatedSources: File
+
+        @JvmStatic
+        @BeforeClass
+        fun setUp() {
+            val compiledClasses = tmp.newFolder()
+            compileSources(listOf(MY_TEST_DIR.resolve("CKlass.java")), compiledClasses)
+
+            val classpathHistory = tmp.newFolder()
+            cache = JavaClassCacheManager(tmp.newFolder(), classpathHistory)
+            generatedSources = tmp.newFolder()
+            cache.close()
+            classpathHistory.resolve("0").createNewFile()
+            val processor = SimpleProcessor().toAggregating()
+            val srcFiles = listOf(
+                "A.java",
+                "B.java",
+                "AnnotationA.java",
+                "AnnotatedType.java"
+            ).map { File(MY_TEST_DIR, it) }
+            runAnnotationProcessing(
+                srcFiles,
+                listOf(processor),
+                generatedSources,
+                listOf(compiledClasses)
+            ) { elements, trees -> MentionedTypesTaskListener(cache.javaCache, elements, trees) }
+            cache.updateCache(listOf(processor))
+        }
+    }
+
+    @Test
+    fun testConstantInField() {
+        val klassA = cache.javaCache.getStructure(MY_TEST_DIR.resolve("A.java"))!!
+
+        assertEquals(setOf("test.A"), klassA.getDeclaredTypes())
+        assertEquals(emptySet<String>(), klassA.getMentionedAnnotations())
+        assertEquals(emptySet<String>(), klassA.getPrivateTypes())
+        assertEquals(setOf("test.A"), klassA.getMentionedTypes())
+        assertEquals(
+            mapOf(
+                "test.B" to setOf("INT_VALUE"),
+                "test.CKlass" to setOf("INT_VALUE")
+            ), klassA.getMentionedConstants()
+        )
+    }
+
+    @Test
+    fun testConstantInDefaultValue() {
+        val annotationA = cache.javaCache.getStructure(MY_TEST_DIR.resolve("AnnotationA.java"))!!
+
+        assertEquals(setOf("test.AnnotationA"), annotationA.getDeclaredTypes())
+        assertEquals(emptySet<String>(), annotationA.getMentionedAnnotations())
+        assertEquals(emptySet<String>(), annotationA.getPrivateTypes())
+        assertEquals(setOf("test.AnnotationA"), annotationA.getMentionedTypes())
+        assertEquals(mapOf("test.B" to setOf("INT_VALUE")), annotationA.getMentionedConstants()
+        )
+    }
+
+    @Test
+    fun testConstantInAnnotationElementValue() {
+        val annotated = cache.javaCache.getStructure(MY_TEST_DIR.resolve("AnnotatedType.java"))!!
+
+        assertEquals(setOf("test.AnnotatedType"), annotated.getDeclaredTypes())
+        assertEquals(setOf("test.AnnotationA"), annotated.getMentionedAnnotations())
+        assertEquals(emptySet<String>(), annotated.getPrivateTypes())
+        assertEquals(setOf("test.AnnotatedType", "test.AnnotationA"), annotated.getMentionedTypes())
+        assertEquals(
+            mapOf(
+                "test.B" to setOf("INT_VALUE"),
+                "test.CKlass" to setOf("INT_VALUE")
+            ), annotated.getMentionedConstants()
+        )
+    }
+}

plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/TestComplexIncrementalAptCache.kt

@@ -48,7 +48,7 @@ class TestComplexIncrementalAptCache {
                 srcFiles,
                 listOf(processor),
                 generatedSources
-            ) { trees -> MentionedTypesTaskListener(cache.javaCache, trees) }
+            ) { elements, trees -> MentionedTypesTaskListener(cache.javaCache, elements, trees) }
             cache.updateCache(listOf(processor))
         }
     }

plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/TestSimpleIncrementalAptCache.kt

@@ -73,7 +73,7 @@ class TestSimpleIncrementalAptCache {
             srcFiles,
             listOf(processor),
             generatedSources
-        ) { elementUtils -> MentionedTypesTaskListener(cache.javaCache, elementUtils) }
+        ) { elementUtils, trees -> MentionedTypesTaskListener(cache.javaCache, elementUtils, trees) }
         cache.updateCache(listOf(processor))
     }
 }

plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/fixtures.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.kapt.base.test.org.jetbrains.kotlin.kapt3.base.incremental
 
 import com.sun.source.util.TaskListener
+import com.sun.source.util.Trees
 import com.sun.tools.javac.api.JavacTaskImpl
 import org.jetbrains.kotlin.kapt3.base.incremental.DeclaredProcType
 import org.jetbrains.kotlin.kapt3.base.incremental.IncrementalProcessor
@@ -27,7 +28,8 @@ fun runAnnotationProcessing(
     srcFiles: List<File>,
     processor: List<IncrementalProcessor>,
     generatedSources: File,
-    listener: (Elements) -> TaskListener? = { null }
+    classpath: List<File> = emptyList(),
+    listener: (Elements, Trees) -> TaskListener? = { _, _ -> null }
 ) {
     val compiler = ToolProvider.getSystemJavaCompiler()
     compiler.getStandardFileManager(null, null, null).use { fileManager ->
@@ -37,12 +39,12 @@ fun runAnnotationProcessing(
                 null,
                 fileManager,
                 null,
-                listOf("-proc:only", "-s", generatedSources.absolutePath, "-d", generatedSources.absolutePath),
+                listOf("-proc:only", "-s", generatedSources.absolutePath, "-d", generatedSources.absolutePath, "-cp", classpath.joinToString(separator = File.pathSeparator)),
                 null,
                 javaSrcs
             ) as JavacTaskImpl
 
-        val taskListener = listener(compilationTask.elements)
+        val taskListener = listener(compilationTask.elements, Trees.instance(compilationTask))
         taskListener?.let { compilationTask.addTaskListener(it) }
 
         compilationTask.setProcessors(processor)

plugins/kapt3/kapt3-base/testData/runner/incremental/constants/A.java

@@ -0,0 +1,5 @@
+package test;
+
+public class A {
+    final int myField = B.INT_VALUE + (1 < 5 ? CKlass.INT_VALUE : 13);
+}

plugins/kapt3/kapt3-base/testData/runner/incremental/constants/AnnotatedType.java

@@ -0,0 +1,5 @@
+package test;
+
+@AnnotationA(CKlass.INT_VALUE + B.INT_VALUE)
+public class AnnotatedType {
+}

plugins/kapt3/kapt3-base/testData/runner/incremental/constants/AnnotationA.java

@@ -0,0 +1,5 @@
+package test;
+
+public @interface AnnotationA {
+    int value() default (B.INT_VALUE + B.INT_VALUE) ;
+}

plugins/kapt3/kapt3-base/testData/runner/incremental/constants/B.java

@@ -0,0 +1,5 @@
+package test;
+
+public class B {
+    static final int INT_VALUE = 12;
+}

plugins/kapt3/kapt3-base/testData/runner/incremental/constants/CKlass.java

@@ -0,0 +1,5 @@
+package test;
+
+public class CKlass {
+    static final int INT_VALUE = 123;
+}