From 7a13173e6a5faab9d491c79f776250f9bad7b62e Mon Sep 17 00:00:00 2001 From: Jingbo Yang Date: Fri, 17 Jun 2022 16:27:16 -0400 Subject: [PATCH] Support fileAccessHistoryReportFile in KAPT This change expanded KAPT to support a new param 'fileAccessHistoryReportFile', which reports all the classes used during annotation processing into a file, in the form of a list of URIs. This is useful for build speed improvements described in https://engineering.fb.com/2017/11/09/android/rethinking-android-app-compilation-with-buck/. Essentially, using the list of used classes, we can compile only the dependencies that are really affected by the developer's code changes and improve Kotlin build speed. ^KT-52853 --- .../internal/kapt/KaptWithoutKotlincTask.kt | 3 +- .../kotlin/kapt3/base/KaptContext.kt | 2 +- .../kotlin/kapt3/base/KaptOptions.kt | 7 +- .../kotlin/kapt3/base/annotationProcessing.kt | 14 ++- .../kapt3/base/javac/KaptJavaFileManager.kt | 104 +++++++++++++++--- .../kapt3-base/test/JavaKaptContextTest.kt | 29 ++++- plugins/kapt3/kapt3-cli/src/KaptCliOption.kt | 7 ++ .../org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt | 1 + 8 files changed, 145 insertions(+), 22 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt index e627fe4d9df..e0d4f2aeb8b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt @@ -366,7 +366,8 @@ private class KaptExecution @Inject constructor( processingClassLoader, disableClassloaderCacheForProcessors, - /*processorsPerfReportFile=*/null + /*processorsPerfReportFile=*/null, + /*fileReadHistoryReportFile*/null, ) } diff --git a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptContext.kt b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptContext.kt index ff30bd16338..eebabbc849d 100644 --- a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptContext.kt +++ b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptContext.kt @@ -47,7 +47,7 @@ open class KaptContext(val options: KaptOptions, val withJdk: Boolean, val logge init { preregisterLog(context) - KaptJavaFileManager.preRegister(context) + KaptJavaFileManager.preRegister(context, options.fileReadHistoryReportFile != null) @Suppress("LeakingThis") preregisterTreeMaker(context) diff --git a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt index 74224d5e94a..aee74300072 100644 --- a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt +++ b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt @@ -43,7 +43,8 @@ class KaptOptions( val processingClassLoader: ClassLoader?, //construct new classloader for these processors instead of using one defined in processingClassLoader val separateClassloaderForProcessors: Set, - val processorsStatsReportFile: File? + val processorsStatsReportFile: File?, + val fileReadHistoryReportFile: File? ) : KaptFlags { override fun get(flag: KaptFlag) = flags[flag] @@ -74,6 +75,7 @@ class KaptOptions( var mode: AptMode = AptMode.WITH_COMPILATION var detectMemoryLeaks: DetectMemoryLeaksMode = DetectMemoryLeaksMode.DEFAULT var processorsStatsReportFile: File? = null + var fileReadHistoryReportFile: File? = null fun build(): KaptOptions { val sourcesOutputDir = this.sourcesOutputDir ?: error("'sourcesOutputDir' must be set") @@ -88,7 +90,8 @@ class KaptOptions( mode, detectMemoryLeaks, processingClassLoader = null, separateClassloaderForProcessors = emptySet(), - processorsStatsReportFile = processorsStatsReportFile + processorsStatsReportFile = processorsStatsReportFile, + fileReadHistoryReportFile = fileReadHistoryReportFile ) } } diff --git a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/annotationProcessing.kt b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/annotationProcessing.kt index 5070fcc0e17..a569b31d9ab 100644 --- a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/annotationProcessing.kt +++ b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/annotationProcessing.kt @@ -14,6 +14,7 @@ import com.sun.tools.javac.processing.JavacProcessingEnvironment import com.sun.tools.javac.tree.JCTree import org.jetbrains.kotlin.base.kapt3.KaptFlag import org.jetbrains.kotlin.kapt3.base.incremental.* +import org.jetbrains.kotlin.kapt3.base.javac.KaptJavaFileManager import org.jetbrains.kotlin.kapt3.base.util.KaptBaseError import org.jetbrains.kotlin.kapt3.base.util.KaptLogger import org.jetbrains.kotlin.kapt3.base.util.isJava9OrLater @@ -25,6 +26,7 @@ import javax.annotation.processing.RoundEnvironment import javax.lang.model.SourceVersion import javax.lang.model.element.TypeElement import javax.tools.JavaFileObject +import kotlin.collections.List import kotlin.system.measureTimeMillis import com.sun.tools.javac.util.List as JavacList @@ -124,6 +126,10 @@ fun KaptContext.doAnnotationProcessing( options.processorsStatsReportFile?.let { dumpProcessorStats(wrappedProcessors, it, logger::info) } + options.fileReadHistoryReportFile?.let { + dumpFileReadHistory(fileManager, it, logger::info) + } + if (logger.isVerbose) { filer.displayState() } @@ -163,6 +169,12 @@ private fun dumpProcessorStats(wrappedProcessors: List, apRepo }) } +private fun dumpFileReadHistory(fileManager: KaptJavaFileManager, reportFile: File, logger: (String) -> Unit) { + logger("Dumping KAPT file read history to ${reportFile.absolutePath}") + + reportFile.writeText(fileManager.renderFileReadHistory()) +} + private fun reportIfRunningNonIncrementally( listener: MentionedTypesTaskListener?, cacheManager: JavaClassCacheManager?, @@ -296,4 +308,4 @@ private fun KaptContext.initModulesIfNeeded(files: JavacList() -class KaptJavaFileManager(context: Context) : JavacFileManager(context, true, null) { fun handleOptionJavac9(option: Option, value: String) { val handleOptionMethod = JavacFileManager::class.java .getMethod("handleOption", Option::class.java, String::class.java) @@ -34,24 +37,70 @@ class KaptJavaFileManager(context: Context) : JavacFileManager(context, true, nu ): MutableIterable { val originalList = super.list(location, packageName, kinds, recurse) - if (location == null - || location != StandardLocation.CLASS_PATH - || rootsToFilter.isEmpty() - || typeToIgnore.isEmpty() - || !filterThisPath(packageName) - ) { - return originalList - } + val shouldPackageBeFiltered = + location != null && + location == StandardLocation.CLASS_PATH && + rootsToFilter.isNotEmpty() && + typeToIgnore.isNotEmpty() && + filterThisPath(packageName) val filteredList = LinkedList() for (file in originalList) - if (!shouldBeFiltered(packageName, file)) { - filteredList.add(file) + if (!shouldPackageBeFiltered || !shouldBeFiltered(packageName, file)) { + filteredList.add(wrapWithReadMonitoringIfNeeded(location, file) as JavaFileObject) } return filteredList } + override fun getJavaFileForInput(location: JavaFileManager.Location?, className: String?, kind: JavaFileObject.Kind?): JavaFileObject? { + val file = super.getJavaFileForInput(location, className, kind) + return wrapWithReadMonitoringIfNeeded(location, file) as JavaFileObject? + } + + override fun getJavaFileForOutput( + location: JavaFileManager.Location?, + className: String?, + kind: JavaFileObject.Kind?, + sibling: FileObject? + ): JavaFileObject? { + val file = super.getJavaFileForOutput(location, className, kind, sibling) + return wrapWithReadMonitoringIfNeeded(location, file) as JavaFileObject? + } + + override fun getFileForInput(location: JavaFileManager.Location?, packageName: String?, relativeName: String?): FileObject? { + val file = super.getFileForInput(location, packageName, relativeName) + return wrapWithReadMonitoringIfNeeded(location, file) + } + + override fun getFileForOutput( + location: JavaFileManager.Location?, + packageName: String?, + relativeName: String?, + sibling: FileObject? + ): FileObject? { + val file = super.getFileForOutput(location, packageName, relativeName, sibling) + return wrapWithReadMonitoringIfNeeded(location, file) + } + + /** javac does not play nice with wrapped file objects in this method; so we unwrap */ + override fun inferBinaryName(location: JavaFileManager.Location?, file: JavaFileObject): String? = + super.inferBinaryName(location, unwrapObject(file) as JavaFileObject) + + /** javac does not play nice with wrapped file objects in this method; so we unwrap */ + override fun isSameFile(a: FileObject, b: FileObject): Boolean { + return super.isSameFile(unwrapObject(a), unwrapObject(b)) + } + + private fun wrapWithReadMonitoringIfNeeded(location: JavaFileManager.Location?, file: FileObject?) = + if (shouldRecordFileRead && location != StandardLocation.ANNOTATION_PROCESSOR_PATH && file != null && file is JavaFileObject) + ReadMonitoredJavaFileObject(file) + else file + + private fun unwrapObject(file: FileObject): FileObject = + if (file is ReadMonitoredJavaFileObject) file.getJavaFileObject() else file + + fun renderFileReadHistory() = fileReadHistory.sorted().joinToString("\n") private fun filterThisPath(packageName: String?): Boolean { packageName ?: return false @@ -72,9 +121,32 @@ class KaptJavaFileManager(context: Context) : JavacFileManager(context, true, nu } } + private inner class ReadMonitoredJavaFileObject(innerFile: JavaFileObject) : ForwardingJavaFileObject(innerFile) { + fun getJavaFileObject(): JavaFileObject = fileObject + override fun toString(): String = fileObject.name + override fun openInputStream(): InputStream { + recordFileRead() + return super.openInputStream() + } + + override fun openReader(ignoreEncodingErrors: Boolean): Reader { + recordFileRead() + return super.openReader(ignoreEncodingErrors) + } + + override fun getCharContent(ignoreEncodingErrors: Boolean): CharSequence { + recordFileRead() + return super.getCharContent(ignoreEncodingErrors) + } + + private fun recordFileRead() { + fileReadHistory.add(fileObject.toUri()) + } + } + companion object { - internal fun preRegister(context: Context) { - context.put(JavaFileManager::class.java, Context.Factory { KaptJavaFileManager(it) }) + internal fun preRegister(context: Context, shouldRecordFileRead: Boolean) { + context.put(JavaFileManager::class.java, Context.Factory { KaptJavaFileManager(it, shouldRecordFileRead) }) } } } \ No newline at end of file diff --git a/plugins/kapt3/kapt3-base/test/JavaKaptContextTest.kt b/plugins/kapt3/kapt3-base/test/JavaKaptContextTest.kt index 8feae8d9dea..dae130347dd 100644 --- a/plugins/kapt3/kapt3-base/test/JavaKaptContextTest.kt +++ b/plugins/kapt3/kapt3-base/test/JavaKaptContextTest.kt @@ -58,7 +58,12 @@ class JavaKaptContextTest { ) } - private fun doAnnotationProcessing(javaSourceFile: File, processor: IncrementalProcessor, outputDir: File) { + private fun doAnnotationProcessing( + javaSourceFile: File, + processor: IncrementalProcessor, + outputDir: File, + fileReadOutput: File? = null + ) { val options = KaptOptions.Builder().apply { projectBaseDir = javaSourceFile.parentFile @@ -67,6 +72,8 @@ class JavaKaptContextTest { stubsOutputDir = outputDir incrementalDataOutputDir = outputDir + fileReadHistoryReportFile = fileReadOutput + flags.add(KaptFlag.MAP_DIAGNOSTIC_LOCATIONS) detectMemoryLeaks = DetectMemoryLeaksMode.NONE }.build() @@ -86,6 +93,26 @@ class JavaKaptContextTest { } } + @Test + fun testDumpFileReadHistory() { + val sourceOutputDir = Files.createTempDirectory("kaptRunner").toFile() + val fileReadOutputFile = File.createTempFile("kapt_read_history", ".txt") + try { + doAnnotationProcessing( + File(TEST_DATA_DIR, "Simple.java"), + simpleProcessor(), + sourceOutputDir, + fileReadOutput = fileReadOutputFile + ) + assertTrue(fileReadOutputFile.exists()) + assertTrue(fileReadOutputFile.readText().contains("generated/MyMethodMyAnnotation.java")) + assertTrue(fileReadOutputFile.readText().contains("java/lang/Enum.class")) + } finally { + sourceOutputDir.deleteRecursively() + fileReadOutputFile.delete() + } + } + @Test fun testException() { val exceptionMessage = "Here we are!" diff --git a/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt b/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt index 140ef755f94..0ae443685fe 100644 --- a/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt +++ b/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt @@ -187,6 +187,13 @@ enum class KaptCliOption( cliToolOption = CliToolOption("-Kapt-dump-processor-stats", VALUE) ), + DUMP_FILE_READ_HISTORY( + "dumpFileReadHistory", + "", + "Dump list of files read by processors to the specified file", + cliToolOption = CliToolOption("-Kapt-dump-file-read-history", VALUE) + ), + STRICT_MODE_OPTION( "strict", "true | false", diff --git a/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt b/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt index ffac676d6a0..7287c64ae49 100644 --- a/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt +++ b/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt @@ -128,6 +128,7 @@ class Kapt3CommandLineProcessor : CommandLineProcessor { SHOW_PROCESSOR_STATS -> setFlag(KaptFlag.SHOW_PROCESSOR_STATS, value) DUMP_PROCESSOR_STATS -> processorsStatsReportFile = File(value) + DUMP_FILE_READ_HISTORY -> fileReadHistoryReportFile = File(value) INCLUDE_COMPILE_CLASSPATH -> setFlag(KaptFlag.INCLUDE_COMPILE_CLASSPATH, value) DETECT_MEMORY_LEAKS_OPTION -> setSelector(enumValues(), value) { detectMemoryLeaks = it }