Kapt3: Generate incremental compilation metadata (light classes with Kotlin metadata)

compiler/cli/src/org/jetbrains/kotlin/cli/common/output/outputUtils.kt

@@ -23,24 +23,25 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import java.io.File
 import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.backend.common.output.OutputFile
 
-fun OutputFileCollection.writeAll(outputDir: File, report: (sources: List<File>, output: File) -> Unit) {
+fun OutputFileCollection.writeAll(outputDir: File, report: (file: OutputFile, sources: List<File>, output: File) -> Unit) {
     for (file in asList()) {
         val sources = file.sourceFiles
         val output = File(outputDir, file.relativePath)
-        report(sources, output)
+        report(file, sources, output)
         FileUtil.writeToFile(output, file.asByteArray())
     }
 }
 
-private val REPORT_NOTHING = { sources: List<File>, output: File -> }
+private val REPORT_NOTHING = { file: OutputFile, sources: List<File>, output: File -> }
 
 fun OutputFileCollection.writeAllTo(outputDir: File) {
     writeAll(outputDir, REPORT_NOTHING)
 }
 
 fun OutputFileCollection.writeAll(outputDir: File, messageCollector: MessageCollector) {
-    writeAll(outputDir) { sources, output ->
+    writeAll(outputDir) { file, sources, output ->
         messageCollector.report(CompilerMessageSeverity.OUTPUT, OutputMessageUtil.formatOutputMessage(sources, output), CompilerMessageLocation.NO_LOCATION)
     }
 }

plugins/kapt3/src/org/jetbrains/kotlin/kapt3/Kapt3BuilderFactory.kt

@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.codegen.ClassBuilder
 import org.jetbrains.kotlin.codegen.ClassBuilderFactory
 import org.jetbrains.kotlin.codegen.ClassBuilderMode
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.JvmDeclarationOrigin
+import org.jetbrains.org.objectweb.asm.ClassWriter
 import org.jetbrains.org.objectweb.asm.FieldVisitor
 import org.jetbrains.org.objectweb.asm.MethodVisitor
 import org.jetbrains.org.objectweb.asm.tree.ClassNode
@@ -37,37 +38,44 @@ internal class Kapt3BuilderFactory : ClassBuilderFactory {
         val classNode = ClassNode()
         compiledClasses += classNode
         origins.put(classNode, origin)
+        return Kapt3ClassBuilder(classNode)
+    }
 
-        return object : AbstractClassBuilder.Concrete(classNode) {
+    private inner class Kapt3ClassBuilder(val classNode: ClassNode) : AbstractClassBuilder.Concrete(classNode) {
-            override fun newField(
+        override fun newField(
-                    origin: JvmDeclarationOrigin,
+                origin: JvmDeclarationOrigin,
-                    access: Int,
+                access: Int,
-                    name: String,
+                name: String,
-                    desc: String,
+                desc: String,
-                    signature: String?,
+                signature: String?,
-                    value: Any?
+                value: Any?
-            ): FieldVisitor {
+        ): FieldVisitor {
-                val fieldNode = super.newField(origin, access, name, desc, signature, value) as FieldNode
+            val fieldNode = super.newField(origin, access, name, desc, signature, value) as FieldNode
-                origins.put(fieldNode, origin)
+            origins.put(fieldNode, origin)
-                return fieldNode
+            return fieldNode
-            }
+        }
 
-            override fun newMethod(
+        override fun newMethod(
-                    origin: JvmDeclarationOrigin,
+                origin: JvmDeclarationOrigin,
-                    access: Int,
+                access: Int,
-                    name: String,
+                name: String,
-                    desc: String,
+                desc: String,
-                    signature: String?,
+                signature: String?,
-                    exceptions: Array<out String>?
+                exceptions: Array<out String>?
-            ): MethodVisitor {
+        ): MethodVisitor {
-                val methodNode = super.newMethod(origin, access, name, desc, signature, exceptions) as MethodNode
+            val methodNode = super.newMethod(origin, access, name, desc, signature, exceptions) as MethodNode
-                origins.put(methodNode, origin)
+            origins.put(methodNode, origin)
-                return methodNode
+            return methodNode
-            }
         }
     }
 
+    override fun asBytes(builder: ClassBuilder): ByteArray {
+        val classWriter = ClassWriter(0)
+        (builder as Kapt3ClassBuilder).classNode.accept(classWriter)
+        return classWriter.toByteArray()
+    }
+
     override fun asText(builder: ClassBuilder) = throw UnsupportedOperationException()
-    override fun asBytes(builder: ClassBuilder) = throw UnsupportedOperationException()
+
     override fun close() {}
 }

plugins/kapt3/src/org/jetbrains/kotlin/kapt3/Kapt3Extension.kt

@@ -20,7 +20,10 @@ import com.intellij.openapi.project.Project
 import com.sun.tools.javac.tree.JCTree
 import com.sun.tools.javac.tree.JCTree.JCCompilationUnit
 import org.jetbrains.kotlin.analyzer.AnalysisResult
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil
 import org.jetbrains.kotlin.cli.common.output.outputUtils.writeAll
 import org.jetbrains.kotlin.codegen.*
 import org.jetbrains.kotlin.codegen.state.GenerationState
@@ -46,15 +49,15 @@ class ClasspathBasedKapt3Extension(
         javaSourceRoots: List<File>,
         sourcesOutputDir: File,
         classFilesOutputDir: File,
-        val stubsOutputDir: File?,
+        stubsOutputDir: File?,
-        val incrementalDataOutputDir: File?,
+        incrementalDataOutputDir: File?,
         options: Map<String, String>,
         aptOnly: Boolean,
         val useLightAnalysis: Boolean,
         pluginInitializedTime: Long,
         logger: KaptLogger
 ) : AbstractKapt3Extension(compileClasspath, annotationProcessingClasspath, javaSourceRoots, sourcesOutputDir,
-                           classFilesOutputDir, options, aptOnly, pluginInitializedTime, logger) {
+                           classFilesOutputDir, stubsOutputDir, incrementalDataOutputDir, options, aptOnly, pluginInitializedTime, logger) {
     override val analyzePartially: Boolean
         get() = useLightAnalysis
 
@@ -87,24 +90,6 @@ class ClasspathBasedKapt3Extension(
 
         return processors
     }
-
-    override fun saveStubs(stubs: JavacList<JCCompilationUnit>) {
-        val outputDir = stubsOutputDir ?: return
-        for (stub in stubs) {
-            val className = (stub.defs.first { it is JCTree.JCClassDecl } as JCTree.JCClassDecl).simpleName.toString()
-
-            val packageName = stub.packageName?.toString() ?: ""
-            val packageDir = if (packageName.isEmpty()) outputDir else File(outputDir, packageName.replace('.', '/'))
-            packageDir.mkdirs()
-            File(packageDir, className + ".java").writeText(stub.toString())
-        }
-    }
-
-    override fun saveIncrementalData(generationState: GenerationState, messageCollector: MessageCollector) {
-        if (incrementalDataOutputDir != null) {
-            generationState.factory.writeAll(incrementalDataOutputDir, messageCollector)
-        }
-    }
 }
 
 abstract class AbstractKapt3Extension(
@@ -113,6 +98,8 @@ abstract class AbstractKapt3Extension(
         val javaSourceRoots: List<File>,
         val sourcesOutputDir: File,
         val classFilesOutputDir: File,
+        val stubsOutputDir: File?,
+        val incrementalDataOutputDir: File?,
         val options: Map<String, String>,
         val aptOnly: Boolean,
         val pluginInitializedTime: Long,
@@ -216,15 +203,17 @@ abstract class AbstractKapt3Extension(
     }
 
     private fun generateKotlinSourceStubs(kaptContext: KaptContext, generationState: GenerationState): JavacList<JCCompilationUnit> {
+        val converter = ClassFileToSourceStubConverter(kaptContext, generationState.typeMapper, generateNonExistentClass = true)
+
         val (stubGenerationTime, kotlinSourceStubs) = measureTimeMillis {
-            ClassFileToSourceStubConverter(kaptContext, generationState.typeMapper, generateNonExistentClass = true).convert()
+            converter.convert()
         }
 
         logger.info { "Java stub generation took $stubGenerationTime ms" }
         logger.info { "Stubs for Kotlin classes: " + kotlinSourceStubs.joinToString { it.sourcefile.name } }
 
         saveStubs(kotlinSourceStubs)
-        saveIncrementalData(generationState, logger.messageCollector)
+        saveIncrementalData(generationState, logger.messageCollector, converter)
         return kotlinSourceStubs
     }
 
@@ -237,9 +226,39 @@ abstract class AbstractKapt3Extension(
         return javaFilesFromJavaSourceRoots
     }
 
-    protected abstract fun saveStubs(stubs: JavacList<JCTree.JCCompilationUnit>)
+    protected open fun saveStubs(stubs: JavacList<JCTree.JCCompilationUnit>) {
+        val outputDir = stubsOutputDir ?: return
+        for (stub in stubs) {
+            val className = (stub.defs.first { it is JCTree.JCClassDecl } as JCTree.JCClassDecl).simpleName.toString()
 
-    protected abstract fun saveIncrementalData(generationState: GenerationState, messageCollector: MessageCollector)
+            val packageName = stub.packageName?.toString() ?: ""
+            val packageDir = if (packageName.isEmpty()) outputDir else File(outputDir, packageName.replace('.', '/'))
+            packageDir.mkdirs()
+            File(packageDir, className + ".java").writeText(stub.toString())
+        }
+    }
+
+    protected open fun saveIncrementalData(
+            generationState: GenerationState,
+            messageCollector: MessageCollector,
+            converter: ClassFileToSourceStubConverter) {
+        val incrementalDataOutputDir = this.incrementalDataOutputDir ?: return
+
+        generationState.factory.writeAll(incrementalDataOutputDir) { file, sources, output ->
+            val stubFileObject = converter.bindings[file.relativePath.substringBeforeLast(".class", missingDelimiterValue = "")]
+            if (stubFileObject != null && stubsOutputDir != null) {
+                val stubFile = File(stubsOutputDir, stubFileObject.name)
+                if (stubFile.exists()) {
+                    messageCollector.report(CompilerMessageSeverity.OUTPUT,
+                                            OutputMessageUtil.formatOutputMessage(sources, stubFile), CompilerMessageLocation.NO_LOCATION)
+                }
+            }
+
+            messageCollector.report(CompilerMessageSeverity.OUTPUT,
+                                    OutputMessageUtil.formatOutputMessage(sources, output), CompilerMessageLocation.NO_LOCATION)
+        }
+
+    }
 
     protected abstract fun loadProcessors(): List<Processor>
 }

plugins/kapt3/src/org/jetbrains/kotlin/kapt3/stubs/ClassFileToSourceStubConverter.kt

@@ -70,6 +70,11 @@ class ClassFileToSourceStubConverter(
         )
     }
 
+    private val _bindings = mutableMapOf<String, KaptJavaFileObject>()
+
+    val bindings: Map<String, KaptJavaFileObject>
+        get() = _bindings
+
     private val fileManager = kaptContext.context.get(JavaFileManager::class.java) as JavacFileManager
     private val treeMaker = TreeMaker.instance(kaptContext.context) as KaptTreeMaker
     private val signatureParser = SignatureParser(treeMaker)
@@ -100,6 +105,8 @@ class ClassFileToSourceStubConverter(
 
         val topLevel = treeMaker.TopLevel(JavacList.nil(), treeMaker.SimpleName("error"), JavacList.of(nonExistentClass))
         topLevel.sourcefile = KaptJavaFileObject(topLevel, nonExistentClass, fileManager)
+
+        // We basically don't need to add binding for NonExistentClass
         return topLevel
     }
 
@@ -121,7 +128,12 @@ class ClassFileToSourceStubConverter(
         val classes = JavacList.of<JCTree>(classDeclaration)
 
         val topLevel = treeMaker.TopLevel(packageAnnotations, packageClause, imports + classes)
-        topLevel.sourcefile = KaptJavaFileObject(topLevel, classDeclaration, fileManager)
+
+        KaptJavaFileObject(topLevel, classDeclaration, fileManager).apply {
+            topLevel.sourcefile = this
+            _bindings[clazz.name] = this
+        }
+
         return topLevel
     }
 

plugins/kapt3/test/org/jetbrains/kotlin/kapt3/test/AbstractKotlinKapt3IntegrationTest.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.kapt3.test
 
+import com.intellij.openapi.extensions.Extensions
 import com.intellij.openapi.util.text.StringUtil
 import com.sun.tools.javac.tree.JCTree
 import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
@@ -24,8 +25,12 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.codegen.CodegenTestCase
 import org.jetbrains.kotlin.codegen.CodegenTestUtil
 import org.jetbrains.kotlin.codegen.state.GenerationState
+import org.jetbrains.kotlin.diagnostics.rendering.DefaultErrorMessages
 import org.jetbrains.kotlin.kapt3.AbstractKapt3Extension
 import org.jetbrains.kotlin.kapt3.Kapt3BuilderFactory
+import org.jetbrains.kotlin.kapt3.diagnostic.DefaultErrorMessagesKapt3
+import org.jetbrains.kotlin.kapt3.javac.KaptJavaFileObject
+import org.jetbrains.kotlin.kapt3.stubs.ClassFileToSourceStubConverter
 import org.jetbrains.kotlin.kapt3.util.KaptLogger
 import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisHandlerExtension
 import org.jetbrains.kotlin.test.ConfigurationKind
@@ -121,11 +126,18 @@ abstract class AbstractKotlinKapt3IntegrationTest : CodegenTestCase() {
 
         val txtFile = File(wholeFile.parentFile, wholeFile.nameWithoutExtension + ".it.txt")
         val sourceOutputDir = Files.createTempDirectory("kaptRunner").toFile()
+        val stubsDir = Files.createTempDirectory("kaptStubs").toFile()
+        val incrementalDataDir = Files.createTempDirectory("kaptIncrementalData").toFile()
 
         createEnvironmentWithMockJdkAndIdeaAnnotations(ConfigurationKind.ALL, *javaSources)
-        val kapt3Extension = Kapt3ExtensionForTests(processors, javaSources.toList(), sourceOutputDir, this.options)
+
+        val kapt3Extension = Kapt3ExtensionForTests(processors, javaSources.toList(), sourceOutputDir, this.options,
+                                                    stubsOutputDir = stubsDir, incrementalDataOutputDir = incrementalDataDir)
+
         AnalysisHandlerExtension.registerExtension(myEnvironment.project, kapt3Extension)
 
+        Extensions.getRootArea().getExtensionPoint(DefaultErrorMessages.Extension.EP_NAME).registerExtension(DefaultErrorMessagesKapt3())
+
         try {
             loadMultiFiles(files)
 
@@ -137,18 +149,22 @@ abstract class AbstractKotlinKapt3IntegrationTest : CodegenTestCase() {
             KotlinTestUtils.assertEqualsToFile(txtFile, actual)
         } finally {
             sourceOutputDir.deleteRecursively()
+            incrementalDataDir.deleteRecursively()
         }
     }
 
-    private class Kapt3ExtensionForTests(
+    protected class Kapt3ExtensionForTests(
             private val processors: List<Processor>,
             javaSourceRoots: List<File>,
             outputDir: File,
-            options: Map<String, String>
+            options: Map<String, String>,
+            stubsOutputDir: File,
+            incrementalDataOutputDir: File
     ) : AbstractKapt3Extension(PathUtil.getJdkClassesRoots(), emptyList(), javaSourceRoots, outputDir, outputDir,
-                               options, true, System.currentTimeMillis(), KaptLogger(true, messageCollector)
+                               stubsOutputDir, incrementalDataOutputDir, options, true, System.currentTimeMillis(), KaptLogger(true)
     ) {
         internal var savedStubs: String? = null
+        internal var savedBindings: Map<String, KaptJavaFileObject>? = null
 
         override fun loadProcessors() = processors
 
@@ -161,8 +177,22 @@ abstract class AbstractKotlinKapt3IntegrationTest : CodegenTestCase() {
                     .map { it.toString() }
                     .sortedBy(String::hashCode)
                     .joinToString(AbstractKotlinKapt3Test.FILE_SEPARATOR)
+
+            super.saveStubs(stubs)
         }
 
-        override fun saveIncrementalData(generationState: GenerationState, messageCollector: MessageCollector) {}
+        override fun saveIncrementalData(
+                generationState: GenerationState,
+                messageCollector: MessageCollector,
+                converter: ClassFileToSourceStubConverter
+        ) {
+            if (this.savedBindings != null) {
+                error("Bindings are already saved")
+            }
+
+            this.savedBindings = converter.bindings
+
+            super.saveIncrementalData(generationState, messageCollector, converter)
+        }
     }
 }

plugins/kapt3/test/org/jetbrains/kotlin/kapt3/test/KotlinKapt3IntegrationTests.kt

@@ -16,7 +16,11 @@
 
 package org.jetbrains.kotlin.kapt3.test
 
+import org.jetbrains.kotlin.kapt3.javac.KaptJavaFileObject
+import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisHandlerExtension
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
 import org.junit.Test
+import java.io.File
 import javax.lang.model.element.ElementKind
 import javax.lang.model.element.ExecutableElement
 
@@ -29,6 +33,48 @@ class KotlinKapt3IntegrationTests : AbstractKotlinKapt3IntegrationTest() {
         assertEquals("myMethod", annotatedElements.single().simpleName.toString())
     }
 
+    @Test
+    fun testSimpleStubsAndIncrementalData() = bindingsTest("Simple") { stubsOutputDir, incrementalDataOutputDir, bindings ->
+        assert(File(stubsOutputDir, "error/NonExistentClass.java").exists())
+        assert(File(stubsOutputDir, "test/Simple.java").exists())
+        assert(File(stubsOutputDir, "test/EnumClass.java").exists())
+
+        assert(File(incrementalDataOutputDir, "test/Simple.class").exists())
+        assert(File(incrementalDataOutputDir, "test/EnumClass.class").exists())
+
+        assert(bindings.any { it.key == "test/Simple" && it.value.name == "test/Simple.java" })
+        assert(bindings.any { it.key == "test/EnumClass" && it.value.name == "test/EnumClass.java" })
+    }
+
+    @Test
+    fun testStubsAndIncrementalDataForNestedClasses() = bindingsTest("NestedClasses") { stubsOutputDir, incrementalDataOutputDir, bindings ->
+        assert(File(stubsOutputDir, "test/Simple.java").exists())
+        assert(!File(stubsOutputDir, "test/Simple/InnerClass.java").exists())
+
+        assert(File(incrementalDataOutputDir, "test/Simple.class").exists())
+        assert(File(incrementalDataOutputDir, "test/Simple/Companion.class").exists())
+        assert(File(incrementalDataOutputDir, "test/Simple/InnerClass.class").exists())
+        assert(File(incrementalDataOutputDir, "test/Simple/NestedClass.class").exists())
+        assert(File(incrementalDataOutputDir, "test/Simple/NestedClass/NestedNestedClass.class").exists())
+
+        assert(bindings.any { it.key == "test/Simple" && it.value.name == "test/Simple.java" })
+        assert(bindings.none { it.key.contains("Companion") })
+        assert(bindings.none { it.key.contains("InnerClass") })
+    }
+
+    private fun bindingsTest(name: String, test: (File, File, Map<String, KaptJavaFileObject>) -> Unit) {
+        test(name, "test.MyAnnotation") { set, roundEnv, env ->
+            val kaptExtension = AnalysisHandlerExtension.getInstances(myEnvironment.project).firstIsInstance<Kapt3ExtensionForTests>()
+
+            val stubsOutputDir = kaptExtension.stubsOutputDir
+            val incrementalDataOutputDir = kaptExtension.incrementalDataOutputDir
+
+            val bindings = kaptExtension.savedBindings!!
+
+            test(stubsOutputDir!!, incrementalDataOutputDir!!, bindings)
+        }
+    }
+
     @Test
     fun testOptions() = test(
             "Simple", "test.MyAnnotation",

plugins/kapt3/test/org/jetbrains/kotlin/kapt3/test/KotlinKaptContextTestGenerated.java

@@ -36,6 +36,12 @@ public class KotlinKaptContextTestGenerated extends AbstractKotlinKaptContextTes
         KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("plugins/kapt3/testData/kotlinRunner"), Pattern.compile("^(.+)\\.kt$"), TargetBackend.ANY, true);
     }
 
+    @TestMetadata("NestedClasses.kt")
+    public void testNestedClasses() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("plugins/kapt3/testData/kotlinRunner/NestedClasses.kt");
+        doTest(fileName);
+    }
+
     @TestMetadata("Overloads.kt")
     public void testOverloads() throws Exception {
         String fileName = KotlinTestUtils.navigationMetadata("plugins/kapt3/testData/kotlinRunner/Overloads.kt");

plugins/kapt3/testData/kotlinRunner/NestedClasses.it.txt

@@ -0,0 +1,55 @@
+package test;
+
+public abstract @interface MyAnnotation {
+}
+
+////////////////////
+
+package test;
+
+public final class Simple {
+    public static final test.Simple.Companion Companion = null;
+
+    @MyAnnotation()
+    public final void myMethod() {
+    }
+
+    public Simple() {
+        super();
+    }
+
+    public static final class NestedClass {
+
+        public NestedClass() {
+            super();
+        }
+
+        public static final class NestedNestedClass {
+
+            public NestedNestedClass() {
+                super();
+            }
+        }
+    }
+
+    public final class InnerClass {
+
+        public InnerClass() {
+            super();
+        }
+    }
+
+    public static final class Companion {
+
+        private Companion() {
+            super();
+        }
+    }
+}
+
+////////////////////
+
+package error;
+
+public final class NonExistentClass {
+}

plugins/kapt3/testData/kotlinRunner/NestedClasses.kt

@@ -0,0 +1,15 @@
+package test
+
+internal class Simple {
+    @MyAnnotation
+    fun myMethod() {}
+
+    class NestedClass {
+        class NestedNestedClass
+    }
+
+    inner class InnerClass
+    companion object
+}
+
+internal annotation class MyAnnotation

plugins/kapt3/testData/kotlinRunner/NestedClasses.txt

@@ -0,0 +1,4 @@
+MyMethodMyAnnotation.java:
+
+package generated;
+class MyMethodMyAnnotation {}