Support fileAccessHistoryReportFile in KAPT

This change expanded KAPT to support a new param
'fileAccessHistoryReportFile', which reports all the classes used during
 annotation processing into a file, in the form of a list of URIs.

This is useful for build speed improvements described in https://engineering.fb.com/2017/11/09/android/rethinking-android-app-compilation-with-buck/.
Essentially, using the list of used classes, we can compile only
the dependencies that are really affected by the developer's
code changes and improve Kotlin build speed.

^KT-52853

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt

@@ -366,7 +366,8 @@ private class KaptExecution @Inject constructor(
 
             processingClassLoader,
             disableClassloaderCacheForProcessors,
-            /*processorsPerfReportFile=*/null
+            /*processorsPerfReportFile=*/null,
+            /*fileReadHistoryReportFile*/null,
         )
     }
 

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptContext.kt

@@ -47,7 +47,7 @@ open class KaptContext(val options: KaptOptions, val withJdk: Boolean, val logge
 
     init {
         preregisterLog(context)
-        KaptJavaFileManager.preRegister(context)
+        KaptJavaFileManager.preRegister(context, options.fileReadHistoryReportFile != null)
 
         @Suppress("LeakingThis")
         preregisterTreeMaker(context)

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt

@@ -43,7 +43,8 @@ class KaptOptions(
     val processingClassLoader: ClassLoader?,
     //construct new classloader for these processors instead of using one defined in processingClassLoader
     val separateClassloaderForProcessors: Set<String>,
-    val processorsStatsReportFile: File?
+    val processorsStatsReportFile: File?,
+    val fileReadHistoryReportFile: File?
 ) : KaptFlags {
     override fun get(flag: KaptFlag) = flags[flag]
 
@@ -74,6 +75,7 @@ class KaptOptions(
         var mode: AptMode = AptMode.WITH_COMPILATION
         var detectMemoryLeaks: DetectMemoryLeaksMode = DetectMemoryLeaksMode.DEFAULT
         var processorsStatsReportFile: File? = null
+        var fileReadHistoryReportFile: File? = null
 
         fun build(): KaptOptions {
             val sourcesOutputDir = this.sourcesOutputDir ?: error("'sourcesOutputDir' must be set")
@@ -88,7 +90,8 @@ class KaptOptions(
                 mode, detectMemoryLeaks,
                 processingClassLoader = null,
                 separateClassloaderForProcessors = emptySet(),
-                processorsStatsReportFile = processorsStatsReportFile
+                processorsStatsReportFile = processorsStatsReportFile,
+                fileReadHistoryReportFile = fileReadHistoryReportFile
             )
         }
     }

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/annotationProcessing.kt

@@ -14,6 +14,7 @@ import com.sun.tools.javac.processing.JavacProcessingEnvironment
 import com.sun.tools.javac.tree.JCTree
 import org.jetbrains.kotlin.base.kapt3.KaptFlag
 import org.jetbrains.kotlin.kapt3.base.incremental.*
+import org.jetbrains.kotlin.kapt3.base.javac.KaptJavaFileManager
 import org.jetbrains.kotlin.kapt3.base.util.KaptBaseError
 import org.jetbrains.kotlin.kapt3.base.util.KaptLogger
 import org.jetbrains.kotlin.kapt3.base.util.isJava9OrLater
@@ -25,6 +26,7 @@ import javax.annotation.processing.RoundEnvironment
 import javax.lang.model.SourceVersion
 import javax.lang.model.element.TypeElement
 import javax.tools.JavaFileObject
+import kotlin.collections.List
 import kotlin.system.measureTimeMillis
 import com.sun.tools.javac.util.List as JavacList
 
@@ -124,6 +126,10 @@ fun KaptContext.doAnnotationProcessing(
 
         options.processorsStatsReportFile?.let { dumpProcessorStats(wrappedProcessors, it, logger::info) }
 
+        options.fileReadHistoryReportFile?.let {
+            dumpFileReadHistory(fileManager, it, logger::info)
+        }
+
         if (logger.isVerbose) {
             filer.displayState()
         }
@@ -163,6 +169,12 @@ private fun dumpProcessorStats(wrappedProcessors: List<ProcessorWrapper>, apRepo
     })
 }
 
+private fun dumpFileReadHistory(fileManager: KaptJavaFileManager, reportFile: File, logger: (String) -> Unit) {
+    logger("Dumping KAPT file read history to ${reportFile.absolutePath}")
+
+    reportFile.writeText(fileManager.renderFileReadHistory())
+}
+
 private fun reportIfRunningNonIncrementally(
     listener: MentionedTypesTaskListener?,
     cacheManager: JavaClassCacheManager?,
@@ -296,4 +308,4 @@ private fun KaptContext.initModulesIfNeeded(files: JavacList<JCTree.JCCompilatio
     }
 
     return files
-}
+}

plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/javac/KaptJavaFileManager.kt

@@ -9,12 +9,15 @@ import com.sun.tools.javac.file.JavacFileManager
 import com.sun.tools.javac.main.Option
 import com.sun.tools.javac.util.Context
 import java.io.File
+import java.io.InputStream
+import java.io.Reader
+import java.net.URI
 import java.util.*
-import javax.tools.JavaFileManager
+import javax.tools.*
-import javax.tools.JavaFileObject
+
-import javax.tools.StandardLocation
+class KaptJavaFileManager(context: Context, private val shouldRecordFileRead: Boolean) : JavacFileManager(context, true, null) {
+    private val fileReadHistory = mutableSetOf<URI>()
 
-class KaptJavaFileManager(context: Context) : JavacFileManager(context, true, null) {
     fun handleOptionJavac9(option: Option, value: String) {
         val handleOptionMethod = JavacFileManager::class.java
             .getMethod("handleOption", Option::class.java, String::class.java)
@@ -34,24 +37,70 @@ class KaptJavaFileManager(context: Context) : JavacFileManager(context, true, nu
     ): MutableIterable<JavaFileObject> {
         val originalList = super.list(location, packageName, kinds, recurse)
 
-        if (location == null
+        val shouldPackageBeFiltered =
-            || location != StandardLocation.CLASS_PATH
+            location != null &&
-            || rootsToFilter.isEmpty()
+                    location == StandardLocation.CLASS_PATH &&
-            || typeToIgnore.isEmpty()
+                    rootsToFilter.isNotEmpty() &&
-            || !filterThisPath(packageName)
+                    typeToIgnore.isNotEmpty() &&
-        ) {
+                    filterThisPath(packageName)
-            return originalList
-        }
 
         val filteredList = LinkedList<JavaFileObject>()
         for (file in originalList)
-            if (!shouldBeFiltered(packageName, file)) {
+            if (!shouldPackageBeFiltered || !shouldBeFiltered(packageName, file)) {
-                filteredList.add(file)
+                filteredList.add(wrapWithReadMonitoringIfNeeded(location, file) as JavaFileObject)
             }
 
         return filteredList
     }
 
+    override fun getJavaFileForInput(location: JavaFileManager.Location?, className: String?, kind: JavaFileObject.Kind?): JavaFileObject? {
+        val file = super.getJavaFileForInput(location, className, kind)
+        return wrapWithReadMonitoringIfNeeded(location, file) as JavaFileObject?
+    }
+
+    override fun getJavaFileForOutput(
+        location: JavaFileManager.Location?,
+        className: String?,
+        kind: JavaFileObject.Kind?,
+        sibling: FileObject?
+    ): JavaFileObject? {
+        val file = super.getJavaFileForOutput(location, className, kind, sibling)
+        return wrapWithReadMonitoringIfNeeded(location, file) as JavaFileObject?
+    }
+
+    override fun getFileForInput(location: JavaFileManager.Location?, packageName: String?, relativeName: String?): FileObject? {
+        val file = super.getFileForInput(location, packageName, relativeName)
+        return wrapWithReadMonitoringIfNeeded(location, file)
+    }
+
+    override fun getFileForOutput(
+        location: JavaFileManager.Location?,
+        packageName: String?,
+        relativeName: String?,
+        sibling: FileObject?
+    ): FileObject? {
+        val file = super.getFileForOutput(location, packageName, relativeName, sibling)
+        return wrapWithReadMonitoringIfNeeded(location, file)
+    }
+
+    /** javac does not play nice with wrapped file objects in this method; so we unwrap */
+    override fun inferBinaryName(location: JavaFileManager.Location?, file: JavaFileObject): String? =
+        super.inferBinaryName(location, unwrapObject(file) as JavaFileObject)
+
+    /** javac does not play nice with wrapped file objects in this method; so we unwrap */
+    override fun isSameFile(a: FileObject, b: FileObject): Boolean {
+        return super.isSameFile(unwrapObject(a), unwrapObject(b))
+    }
+
+    private fun wrapWithReadMonitoringIfNeeded(location: JavaFileManager.Location?, file: FileObject?) =
+        if (shouldRecordFileRead && location != StandardLocation.ANNOTATION_PROCESSOR_PATH && file != null && file is JavaFileObject)
+            ReadMonitoredJavaFileObject(file)
+        else file
+
+    private fun unwrapObject(file: FileObject): FileObject =
+        if (file is ReadMonitoredJavaFileObject) file.getJavaFileObject() else file
+
+    fun renderFileReadHistory() = fileReadHistory.sorted().joinToString("\n")
 
     private fun filterThisPath(packageName: String?): Boolean {
         packageName ?: return false
@@ -72,9 +121,32 @@ class KaptJavaFileManager(context: Context) : JavacFileManager(context, true, nu
         }
     }
 
+    private inner class ReadMonitoredJavaFileObject(innerFile: JavaFileObject) : ForwardingJavaFileObject<JavaFileObject>(innerFile) {
+        fun getJavaFileObject(): JavaFileObject = fileObject
+        override fun toString(): String = fileObject.name
+        override fun openInputStream(): InputStream {
+            recordFileRead()
+            return super.openInputStream()
+        }
+
+        override fun openReader(ignoreEncodingErrors: Boolean): Reader {
+            recordFileRead()
+            return super.openReader(ignoreEncodingErrors)
+        }
+
+        override fun getCharContent(ignoreEncodingErrors: Boolean): CharSequence {
+            recordFileRead()
+            return super.getCharContent(ignoreEncodingErrors)
+        }
+
+        private fun recordFileRead() {
+            fileReadHistory.add(fileObject.toUri())
+        }
+    }
+
     companion object {
-        internal fun preRegister(context: Context) {
+        internal fun preRegister(context: Context, shouldRecordFileRead: Boolean) {
-            context.put(JavaFileManager::class.java, Context.Factory<JavaFileManager> { KaptJavaFileManager(it) })
+            context.put(JavaFileManager::class.java, Context.Factory { KaptJavaFileManager(it, shouldRecordFileRead) })
         }
     }
 }

plugins/kapt3/kapt3-base/test/JavaKaptContextTest.kt

@@ -58,7 +58,12 @@ class JavaKaptContextTest {
         )
     }
 
-    private fun doAnnotationProcessing(javaSourceFile: File, processor: IncrementalProcessor, outputDir: File) {
+    private fun doAnnotationProcessing(
+        javaSourceFile: File,
+        processor: IncrementalProcessor,
+        outputDir: File,
+        fileReadOutput: File? = null
+    ) {
         val options = KaptOptions.Builder().apply {
             projectBaseDir = javaSourceFile.parentFile
 
@@ -67,6 +72,8 @@ class JavaKaptContextTest {
             stubsOutputDir = outputDir
             incrementalDataOutputDir = outputDir
 
+            fileReadHistoryReportFile = fileReadOutput
+
             flags.add(KaptFlag.MAP_DIAGNOSTIC_LOCATIONS)
             detectMemoryLeaks = DetectMemoryLeaksMode.NONE
         }.build()
@@ -86,6 +93,26 @@ class JavaKaptContextTest {
         }
     }
 
+    @Test
+    fun testDumpFileReadHistory() {
+        val sourceOutputDir = Files.createTempDirectory("kaptRunner").toFile()
+        val fileReadOutputFile = File.createTempFile("kapt_read_history", ".txt")
+        try {
+            doAnnotationProcessing(
+                File(TEST_DATA_DIR, "Simple.java"),
+                simpleProcessor(),
+                sourceOutputDir,
+                fileReadOutput = fileReadOutputFile
+            )
+            assertTrue(fileReadOutputFile.exists())
+            assertTrue(fileReadOutputFile.readText().contains("generated/MyMethodMyAnnotation.java"))
+            assertTrue(fileReadOutputFile.readText().contains("java/lang/Enum.class"))
+        } finally {
+            sourceOutputDir.deleteRecursively()
+            fileReadOutputFile.delete()
+        }
+    }
+
     @Test
     fun testException() {
         val exceptionMessage = "Here we are!"

plugins/kapt3/kapt3-cli/src/KaptCliOption.kt

@@ -187,6 +187,13 @@ enum class KaptCliOption(
         cliToolOption = CliToolOption("-Kapt-dump-processor-stats", VALUE)
     ),
 
+    DUMP_FILE_READ_HISTORY(
+        "dumpFileReadHistory",
+        "<path>",
+        "Dump list of files read by processors to the specified file",
+        cliToolOption = CliToolOption("-Kapt-dump-file-read-history", VALUE)
+    ),
+
     STRICT_MODE_OPTION(
         "strict",
         "true | false",

plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt

@@ -128,6 +128,7 @@ class Kapt3CommandLineProcessor : CommandLineProcessor {
 
             SHOW_PROCESSOR_STATS -> setFlag(KaptFlag.SHOW_PROCESSOR_STATS, value)
             DUMP_PROCESSOR_STATS -> processorsStatsReportFile = File(value)
+            DUMP_FILE_READ_HISTORY -> fileReadHistoryReportFile = File(value)
             INCLUDE_COMPILE_CLASSPATH -> setFlag(KaptFlag.INCLUDE_COMPILE_CLASSPATH, value)
 
             DETECT_MEMORY_LEAKS_OPTION -> setSelector(enumValues<DetectMemoryLeaksMode>(), value) { detectMemoryLeaks = it }