Gradle IC: support multifile classes

build-common/src/org/jetbrains/kotlin/incremental/CacheVersion.kt

@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.load.kotlin.JvmMetadataVersion
 import java.io.File
 
 private val NORMAL_VERSION = 8
-private val EXPERIMENTAL_VERSION = 3
+private val EXPERIMENTAL_VERSION = 4
 private val DATA_CONTAINER_VERSION = 1
 
 private val NORMAL_VERSION_FILE_NAME = "format-version.txt"

build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt

@@ -16,10 +16,12 @@
 
 package org.jetbrains.kotlin.incremental
 
+import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName
 import com.intellij.util.SmartList
 import com.intellij.util.io.BooleanDataDescriptor
 import com.intellij.util.io.EnumeratorStringDescriptor
+import gnu.trove.THashSet
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.build.GeneratedJvmClass
 import org.jetbrains.kotlin.config.IncrementalCompilation
@@ -66,6 +68,7 @@ open class IncrementalCacheImpl<Target>(
         private val SUBTYPES = "subtypes"
         private val SUPERTYPES = "supertypes"
         private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
+        private val INTERNAL_NAME_TO_SOURCE = "internal-name-to-source"
 
         private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
     }
@@ -92,6 +95,8 @@ open class IncrementalCacheImpl<Target>(
     private val subtypesMap = registerExperimentalMap(SubtypesMap(SUBTYPES.storageFile))
     private val supertypesMap = registerExperimentalMap(SupertypesMap(SUPERTYPES.storageFile))
     private val classFqNameToSourceMap = registerExperimentalMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile))
+    // todo: try to use internal names only?
+    private val internalNameToSource = registerExperimentalMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile))
 
     private val dependents = arrayListOf<IncrementalCacheImpl<Target>>()
     private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory: $target" } }
@@ -130,7 +135,11 @@ open class IncrementalCacheImpl<Target>(
     fun getSubtypesOf(className: FqName): Sequence<FqName> =
             subtypesMap[className].asSequence()
 
-    fun getSourceFileIfClass(fqName: FqName): File? = classFqNameToSourceMap[fqName]
+    fun getSourceFileIfClass(fqName: FqName): File? =
+            classFqNameToSourceMap[fqName]
+
+    fun sourcesByInternalName(internalName: String): Collection<File> =
+            internalNameToSource[internalName]
 
     fun isMultifileFacade(className: JvmClassName): Boolean =
             className.internalName in multifileFacadeToParts
@@ -157,6 +166,10 @@ open class IncrementalCacheImpl<Target>(
             sourceToClassesMap.add(it, className)
         }
 
+        if (IncrementalCompilation.isExperimental()) {
+            internalNameToSource[className.internalName] = sourceFiles
+        }
+
         if (kotlinClass.classId.isLocal) {
             return CompilationResult.NO_CHANGES
         }
@@ -181,6 +194,7 @@ open class IncrementalCacheImpl<Target>(
                 // As a workaround we can remove proto values for multifile facades.
                 protoMap.remove(className)
                 classFqNameToSourceMap.remove(className.fqNameForClassNameWithoutDollars)
+                internalNameToSource.remove(className.internalName)
 
                 // TODO NO_CHANGES? (delegates only)
                 constantsMap.process(kotlinClass, isPackage = true) +
@@ -310,6 +324,7 @@ open class IncrementalCacheImpl<Target>(
             partToMultifileFacade.remove(it)
             constantsMap.remove(it)
             inlineFunctionsMap.remove(it)
+            internalNameToSource.remove(it.internalName)
         }
 
         removeAllFromClassStorage(dirtyClasses)
@@ -575,6 +590,23 @@ open class IncrementalCacheImpl<Target>(
         override fun dumpValue(value: String) = value
     }
 
+
+    inner class InternalNameToSourcesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer) {
+        operator fun set(internalName: String, sourceFiles: Iterable<File>) {
+            storage[internalName] = sourceFiles.map { it.canonicalPath }
+        }
+
+        operator fun get(internalName: String): Collection<File> =
+                (storage[internalName] ?: emptyList()).map(::File)
+
+        fun remove(internalName: String) {
+            storage.remove(internalName)
+        }
+
+        override fun dumpValue(value: Collection<String>): String =
+                value.dumpCollection()
+    }
+
     private fun addToClassStorage(kotlinClass: LocalFileKotlinClass, srcFile: File) {
         if (!IncrementalCompilation.isExperimental()) return
 
@@ -727,6 +759,8 @@ open class IncrementalCacheImpl<Target>(
     }
 }
 
+private object PathCollectionExternalizer : CollectionExternalizer<String>(PathStringDescriptor, { THashSet(FileUtil.PATH_HASHING_STRATEGY) })
+
 sealed class ChangeInfo(val fqName: FqName) {
     open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName) {
         override fun toStringProperties(): String = super.toStringProperties() + ", names = $names"

jps-plugin/testData/incremental/changeIncrementalOption/experimentalOn/expected-kotlin-caches.txt

@@ -8,6 +8,7 @@ Module 'module1' production
     experimental-format-version.txt
     format-version.txt
     class-fq-name-to-source.tab
+    internal-name-to-source.tab
     proto.tab
     source-to-classes.tab
 Module 'module1' tests
@@ -15,6 +16,7 @@ Module 'module2' production
     experimental-format-version.txt
     format-version.txt
     class-fq-name-to-source.tab
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
@@ -25,6 +27,7 @@ Module 'module3' production
     experimental-format-version.txt
     format-version.txt
     class-fq-name-to-source.tab
+    internal-name-to-source.tab
     proto.tab
     source-to-classes.tab
 Module 'module3' tests
@@ -32,6 +35,7 @@ Module 'module4' production
     experimental-format-version.txt
     format-version.txt
     class-fq-name-to-source.tab
+    internal-name-to-source.tab
     proto.tab
     source-to-classes.tab
 Module 'module4' tests

jps-plugin/testData/incremental/changeIncrementalOption/experimentalOnJavaChanged/expected-kotlin-caches.txt

@@ -7,6 +7,7 @@ kotlin-data-container
 Module 'module1' production
     experimental-format-version.txt
     format-version.txt
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
@@ -14,6 +15,7 @@ Module 'module1' tests
 Module 'module2' production
     experimental-format-version.txt
     format-version.txt
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
@@ -21,6 +23,7 @@ Module 'module2' tests
 Module 'module3' production
     experimental-format-version.txt
     format-version.txt
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab

jps-plugin/testData/incremental/classHierarchyAffected/inferredTypeArgumentChanged/gradle-build.log

@@ -0,0 +1,52 @@
+================ Step #1 =================
+
+Cleaning output files:
+  out/production/module/B.class
+End of files
+Compiling files:
+  src/B.kt
+End of files
+Marked as dirty by Kotlin:
+  src/C.kt
+  src/getListOfB.kt
+  src/getListOfC.kt
+  src/useListOfAWithListOfB.kt
+  src/useListOfAWithListOfC.kt
+Exit code: ADDITIONAL_PASS_REQUIRED
+------------------------------------------
+Cleaning output files:
+  out/production/module/C.class
+  out/production/module/GetListOfBKt.class
+  out/production/module/GetListOfCKt.class
+  out/production/module/META-INF/module.kotlin_module
+  out/production/module/UseListOfAWithListOfBKt.class
+  out/production/module/UseListOfAWithListOfCKt.class
+End of files
+Compiling files:
+  src/C.kt
+  src/getListOfB.kt
+  src/getListOfC.kt
+  src/useListOfAWithListOfB.kt
+  src/useListOfAWithListOfC.kt
+End of files
+Exit code: ABORT
+------------------------------------------
+COMPILATION FAILED
+Type mismatch: inferred type is List<B> but List<A> was expected
+Type mismatch: inferred type is List<C> but List<A> was expected
+
+================ Step #2 =================
+
+Cleaning output files:
+  out/production/module/B.class
+End of files
+Compiling files:
+  src/B.kt
+  src/C.kt
+  src/getListOfB.kt
+  src/getListOfC.kt
+  src/useListOfAWithListOfB.kt
+  src/useListOfAWithListOfC.kt
+End of files
+Exit code: OK
+------------------------------------------

jps-plugin/testData/incremental/lazyKotlinCaches/class/experimental-expected-kotlin-caches.txt

@@ -8,6 +8,7 @@ Module 'module' production
     experimental-format-version.txt
     format-version.txt
     class-fq-name-to-source.tab
+    internal-name-to-source.tab
     proto.tab
     source-to-classes.tab
 Module 'module' tests

jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/experimental-expected-kotlin-caches.txt

@@ -8,6 +8,7 @@ Module 'module' production
     experimental-format-version.txt
     format-version.txt
     class-fq-name-to-source.tab
+    internal-name-to-source.tab
     proto.tab
     source-to-classes.tab
     subtypes.tab

jps-plugin/testData/incremental/lazyKotlinCaches/constant/experimental-expected-kotlin-caches.txt

@@ -8,7 +8,8 @@ Module 'module' production
     experimental-format-version.txt
     format-version.txt
     constants.tab
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
-Module 'module' tests
+Module 'module' tests

jps-plugin/testData/incremental/lazyKotlinCaches/function/experimental-expected-kotlin-caches.txt

@@ -7,7 +7,8 @@ kotlin-data-container
 Module 'module' production
     experimental-format-version.txt
     format-version.txt
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
-Module 'module' tests
+Module 'module' tests

jps-plugin/testData/incremental/lazyKotlinCaches/inlineFunctionWithUsage/experimental-expected-kotlin-caches.txt

@@ -8,6 +8,7 @@ Module 'module' production
     experimental-format-version.txt
     format-version.txt
     inline-functions.tab
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab

jps-plugin/testData/incremental/lazyKotlinCaches/inlineFunctionWithoutUsage/experimental-expected-kotlin-caches.txt

@@ -8,7 +8,8 @@ Module 'module' production
     experimental-format-version.txt
     format-version.txt
     inline-functions.tab
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
-Module 'module' tests
+Module 'module' tests

jps-plugin/testData/incremental/lazyKotlinCaches/topLevelPropertyAccess/experimental-expected-kotlin-caches.txt

@@ -7,7 +7,8 @@ kotlin-data-container
 Module 'module' production
     experimental-format-version.txt
     format-version.txt
+    internal-name-to-source.tab
     package-parts.tab
     proto.tab
     source-to-classes.tab
-Module 'module' tests
+Module 'module' tests

jps-plugin/testData/incremental/pureKotlin/moveFileWithoutChangingPackage/gradle-build.log

@@ -0,0 +1,12 @@
+================ Step #1 =================
+
+Cleaning output files:
+  out/production/module/META-INF/module.kotlin_module
+  out/production/module/baz/AKt.class
+  out/production/module/baz/Foo.class
+End of files
+Compiling files:
+  src/bar/a.kt
+End of files
+Exit code: OK
+------------------------------------------

jps-plugin/testData/incremental/pureKotlin/multifileClassFileAdded/gradle-build.log

@@ -0,0 +1,8 @@
+================ Step #1 =================
+
+Compiling files:
+  src/a.kt
+  src/b.kt
+End of files
+Exit code: OK
+------------------------------------------

jps-plugin/testData/incremental/pureKotlin/multifileClassFileChanged/gradle-build.log

@@ -0,0 +1,8 @@
+================ Step #1 =================
+
+Compiling files:
+  src/a.kt
+  src/b.kt
+End of files
+Exit code: OK
+------------------------------------------

jps-plugin/testData/incremental/pureKotlin/multifileClassRemoved/dummy.kt

@@ -0,0 +1,3 @@
+package foo
+
+fun dummy() {}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -321,10 +321,14 @@ internal class IncrementalJvmCompilerRunner(
             }
 
             val generatedClassFiles = compilerOutput.generatedFiles
-            val sourcesWithPossibleRedeclarations = getSourcesWithPossibleRedeclarations(caches, generatedClassFiles)
+
-            if (sourcesWithPossibleRedeclarations.isNotEmpty()) {
+            if (compilationMode is CompilationMode.Incremental) {
-                dirtySources.addAll(sourcesWithPossibleRedeclarations)
+                val dirtySourcesSet = dirtySources.toHashSet()
-                continue
+                val additionalDirtyFiles = additionalDirtyFiles(caches, generatedClassFiles).filter { it !in dirtySourcesSet }
+                if (additionalDirtyFiles.isNotEmpty()) {
+                    dirtySources.addAll(additionalDirtyFiles)
+                    continue
+                }
             }
 
             allGeneratedFiles.addAll(generatedClassFiles)
@@ -404,23 +408,42 @@ internal class IncrementalJvmCompilerRunner(
         return exitCode
     }
 
-    private fun getSourcesWithPossibleRedeclarations(caches: IncrementalCachesManager, generatedFiles: List<GeneratedFile<TargetId>>): ArrayList<File> {
+    private fun additionalDirtyFiles(
-        val result = ArrayList<File>()
+            caches: IncrementalCachesManager,
+            generatedFiles: List<GeneratedFile<TargetId>>
+    ): Collection<File> {
+        val result = HashSet<File>()
+
+        fun partsByFacadeName(facadeInternalName: String): List<File> {
+            val parts = caches.incrementalCache.getStableMultifileFacadeParts(facadeInternalName) ?: emptyList()
+            return parts.flatMap { caches.incrementalCache.sourcesByInternalName(it) }
+        }
+
         for (generatedFile in generatedFiles) {
             if (generatedFile !is GeneratedJvmClass<*>) continue
 
             val outputClass = generatedFile.outputClass
-            if (outputClass.classHeader.kind != KotlinClassHeader.Kind.CLASS) continue
-            assert(generatedFile.sourceFiles.size == 1) { "KotlinClassHeader.Kind.CLASS cannot be generated from multiple files" }
 
-            val fqName = outputClass.className.fqNameForClassNameWithoutDollars
+            when (outputClass.classHeader.kind) {
-            val currentSourceFile = generatedFile.sourceFiles.single()
+                KotlinClassHeader.Kind.CLASS -> {
-            val cachedSourceFile = caches.incrementalCache.getSourceFileIfClass(fqName)
+                    val fqName = outputClass.className.fqNameForClassNameWithoutDollars
+                    val cachedSourceFile = caches.incrementalCache.getSourceFileIfClass(fqName)
+
+                    if (cachedSourceFile != null) {
+                        result.add(cachedSourceFile)
+                    }
+                }
+                // todo: more optimal is to check if public API or parts list changed
+                KotlinClassHeader.Kind.MULTIFILE_CLASS -> {
+                    result.addAll(partsByFacadeName(outputClass.className.internalName))
+                }
+                KotlinClassHeader.Kind.MULTIFILE_CLASS_PART -> {
+                    result.addAll(partsByFacadeName(outputClass.classHeader.multifileClassName!!))
+                }
 
-            if (cachedSourceFile != null && cachedSourceFile != currentSourceFile) {
-                result.add(cachedSourceFile)
             }
         }
+
         return result
     }