Gradle IC: support multifile classes

2016-10-27 23:52:21 +03:00
parent f25d948b3f
commit 151cf7d073
17 changed files with 173 additions and 19 deletions
@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.load.kotlin.JvmMetadataVersion
 import java.io.File

 private val NORMAL_VERSION = 8
-private val EXPERIMENTAL_VERSION = 3
+private val EXPERIMENTAL_VERSION = 4
 private val DATA_CONTAINER_VERSION = 1

 private val NORMAL_VERSION_FILE_NAME = "format-version.txt"
@@ -16,10 +16,12 @@

 package org.jetbrains.kotlin.incremental

+import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName
 import com.intellij.util.SmartList
 import com.intellij.util.io.BooleanDataDescriptor
 import com.intellij.util.io.EnumeratorStringDescriptor
+import gnu.trove.THashSet
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.build.GeneratedJvmClass
 import org.jetbrains.kotlin.config.IncrementalCompilation
@@ -66,6 +68,7 @@ open class IncrementalCacheImpl<Target>(
        private val SUBTYPES = "subtypes"
        private val SUPERTYPES = "supertypes"
        private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
+        private val INTERNAL_NAME_TO_SOURCE = "internal-name-to-source"

        private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
    }
@@ -92,6 +95,8 @@ open class IncrementalCacheImpl<Target>(
    private val subtypesMap = registerExperimentalMap(SubtypesMap(SUBTYPES.storageFile))
    private val supertypesMap = registerExperimentalMap(SupertypesMap(SUPERTYPES.storageFile))
    private val classFqNameToSourceMap = registerExperimentalMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile))
+    // todo: try to use internal names only?
+    private val internalNameToSource = registerExperimentalMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile))

    private val dependents = arrayListOf<IncrementalCacheImpl<Target>>()
    private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory: $target" } }
@@ -130,7 +135,11 @@ open class IncrementalCacheImpl<Target>(
    fun getSubtypesOf(className: FqName): Sequence<FqName> =
            subtypesMap[className].asSequence()

-    fun getSourceFileIfClass(fqName: FqName): File? = classFqNameToSourceMap[fqName]
+    fun getSourceFileIfClass(fqName: FqName): File? =
+            classFqNameToSourceMap[fqName]
+
+    fun sourcesByInternalName(internalName: String): Collection<File> =
+            internalNameToSource[internalName]

    fun isMultifileFacade(className: JvmClassName): Boolean =
            className.internalName in multifileFacadeToParts
@@ -157,6 +166,10 @@ open class IncrementalCacheImpl<Target>(
            sourceToClassesMap.add(it, className)
        }

+        if (IncrementalCompilation.isExperimental()) {
+            internalNameToSource[className.internalName] = sourceFiles
+        }
+
        if (kotlinClass.classId.isLocal) {
            return CompilationResult.NO_CHANGES
        }
@@ -181,6 +194,7 @@ open class IncrementalCacheImpl<Target>(
                // As a workaround we can remove proto values for multifile facades.
                protoMap.remove(className)
                classFqNameToSourceMap.remove(className.fqNameForClassNameWithoutDollars)
+                internalNameToSource.remove(className.internalName)

                // TODO NO_CHANGES? (delegates only)
                constantsMap.process(kotlinClass, isPackage = true) +
@@ -310,6 +324,7 @@ open class IncrementalCacheImpl<Target>(
            partToMultifileFacade.remove(it)
            constantsMap.remove(it)
            inlineFunctionsMap.remove(it)
+            internalNameToSource.remove(it.internalName)
        }

        removeAllFromClassStorage(dirtyClasses)
@@ -575,6 +590,23 @@ open class IncrementalCacheImpl<Target>(
        override fun dumpValue(value: String) = value
    }

+
+    inner class InternalNameToSourcesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer) {
+        operator fun set(internalName: String, sourceFiles: Iterable<File>) {
+            storage[internalName] = sourceFiles.map { it.canonicalPath }
+        }
+
+        operator fun get(internalName: String): Collection<File> =
+                (storage[internalName] ?: emptyList()).map(::File)
+
+        fun remove(internalName: String) {
+            storage.remove(internalName)
+        }
+
+        override fun dumpValue(value: Collection<String>): String =
+                value.dumpCollection()
+    }
+
    private fun addToClassStorage(kotlinClass: LocalFileKotlinClass, srcFile: File) {
        if (!IncrementalCompilation.isExperimental()) return

@@ -727,6 +759,8 @@ open class IncrementalCacheImpl<Target>(
    }
 }

+private object PathCollectionExternalizer : CollectionExternalizer<String>(PathStringDescriptor, { THashSet(FileUtil.PATH_HASHING_STRATEGY) })
+
 sealed class ChangeInfo(val fqName: FqName) {
    open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName) {
        override fun toStringProperties(): String = super.toStringProperties() + ", names = $names"
@@ -8,6 +8,7 @@ Module 'module1' production
    experimental-format-version.txt
    format-version.txt
    class-fq-name-to-source.tab
+    internal-name-to-source.tab
    proto.tab
    source-to-classes.tab
 Module 'module1' tests
@@ -15,6 +16,7 @@ Module 'module2' production
    experimental-format-version.txt
    format-version.txt
    class-fq-name-to-source.tab
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
@@ -25,6 +27,7 @@ Module 'module3' production
    experimental-format-version.txt
    format-version.txt
    class-fq-name-to-source.tab
+    internal-name-to-source.tab
    proto.tab
    source-to-classes.tab
 Module 'module3' tests
@@ -32,6 +35,7 @@ Module 'module4' production
    experimental-format-version.txt
    format-version.txt
    class-fq-name-to-source.tab
+    internal-name-to-source.tab
    proto.tab
    source-to-classes.tab
 Module 'module4' tests
@@ -7,6 +7,7 @@ kotlin-data-container
 Module 'module1' production
    experimental-format-version.txt
    format-version.txt
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
@@ -14,6 +15,7 @@ Module 'module1' tests
 Module 'module2' production
    experimental-format-version.txt
    format-version.txt
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
@@ -21,6 +23,7 @@ Module 'module2' tests
 Module 'module3' production
    experimental-format-version.txt
    format-version.txt
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
@@ -0,0 +1,52 @@
+================ Step #1 =================
+
+Cleaning output files:
+  out/production/module/B.class
+End of files
+Compiling files:
+  src/B.kt
+End of files
+Marked as dirty by Kotlin:
+  src/C.kt
+  src/getListOfB.kt
+  src/getListOfC.kt
+  src/useListOfAWithListOfB.kt
+  src/useListOfAWithListOfC.kt
+Exit code: ADDITIONAL_PASS_REQUIRED
+------------------------------------------
+Cleaning output files:
+  out/production/module/C.class
+  out/production/module/GetListOfBKt.class
+  out/production/module/GetListOfCKt.class
+  out/production/module/META-INF/module.kotlin_module
+  out/production/module/UseListOfAWithListOfBKt.class
+  out/production/module/UseListOfAWithListOfCKt.class
+End of files
+Compiling files:
+  src/C.kt
+  src/getListOfB.kt
+  src/getListOfC.kt
+  src/useListOfAWithListOfB.kt
+  src/useListOfAWithListOfC.kt
+End of files
+Exit code: ABORT
+------------------------------------------
+COMPILATION FAILED
+Type mismatch: inferred type is List<B> but List<A> was expected
+Type mismatch: inferred type is List<C> but List<A> was expected
+
+================ Step #2 =================
+
+Cleaning output files:
+  out/production/module/B.class
+End of files
+Compiling files:
+  src/B.kt
+  src/C.kt
+  src/getListOfB.kt
+  src/getListOfC.kt
+  src/useListOfAWithListOfB.kt
+  src/useListOfAWithListOfC.kt
+End of files
+Exit code: OK
+------------------------------------------
@@ -8,6 +8,7 @@ Module 'module' production
    experimental-format-version.txt
    format-version.txt
    class-fq-name-to-source.tab
+    internal-name-to-source.tab
    proto.tab
    source-to-classes.tab
 Module 'module' tests
@@ -8,6 +8,7 @@ Module 'module' production
    experimental-format-version.txt
    format-version.txt
    class-fq-name-to-source.tab
+    internal-name-to-source.tab
    proto.tab
    source-to-classes.tab
    subtypes.tab
@@ -8,7 +8,8 @@ Module 'module' production
    experimental-format-version.txt
    format-version.txt
    constants.tab
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
-Module 'module' tests
+Module 'module' tests
@@ -7,7 +7,8 @@ kotlin-data-container
 Module 'module' production
    experimental-format-version.txt
    format-version.txt
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
-Module 'module' tests
+Module 'module' tests
@@ -8,6 +8,7 @@ Module 'module' production
    experimental-format-version.txt
    format-version.txt
    inline-functions.tab
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
@@ -8,7 +8,8 @@ Module 'module' production
    experimental-format-version.txt
    format-version.txt
    inline-functions.tab
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
-Module 'module' tests
+Module 'module' tests
@@ -7,7 +7,8 @@ kotlin-data-container
 Module 'module' production
    experimental-format-version.txt
    format-version.txt
+    internal-name-to-source.tab
    package-parts.tab
    proto.tab
    source-to-classes.tab
-Module 'module' tests
+Module 'module' tests
@@ -0,0 +1,12 @@
+================ Step #1 =================
+
+Cleaning output files:
+  out/production/module/META-INF/module.kotlin_module
+  out/production/module/baz/AKt.class
+  out/production/module/baz/Foo.class
+End of files
+Compiling files:
+  src/bar/a.kt
+End of files
+Exit code: OK
+------------------------------------------
@@ -0,0 +1,8 @@
+================ Step #1 =================
+
+Compiling files:
+  src/a.kt
+  src/b.kt
+End of files
+Exit code: OK
+------------------------------------------
@@ -0,0 +1,8 @@
+================ Step #1 =================
+
+Compiling files:
+  src/a.kt
+  src/b.kt
+End of files
+Exit code: OK
+------------------------------------------
@@ -0,0 +1,3 @@
+package foo
+
+fun dummy() {}
@@ -321,10 +321,14 @@ internal class IncrementalJvmCompilerRunner(
            }

            val generatedClassFiles = compilerOutput.generatedFiles
-            val sourcesWithPossibleRedeclarations = getSourcesWithPossibleRedeclarations(caches, generatedClassFiles)
-            if (sourcesWithPossibleRedeclarations.isNotEmpty()) {
-                dirtySources.addAll(sourcesWithPossibleRedeclarations)
-                continue
+
+            if (compilationMode is CompilationMode.Incremental) {
+                val dirtySourcesSet = dirtySources.toHashSet()
+                val additionalDirtyFiles = additionalDirtyFiles(caches, generatedClassFiles).filter { it !in dirtySourcesSet }
+                if (additionalDirtyFiles.isNotEmpty()) {
+                    dirtySources.addAll(additionalDirtyFiles)
+                    continue
+                }
            }

            allGeneratedFiles.addAll(generatedClassFiles)
@@ -404,23 +408,42 @@ internal class IncrementalJvmCompilerRunner(
        return exitCode
    }

-    private fun getSourcesWithPossibleRedeclarations(caches: IncrementalCachesManager, generatedFiles: List<GeneratedFile<TargetId>>): ArrayList<File> {
-        val result = ArrayList<File>()
+    private fun additionalDirtyFiles(
+            caches: IncrementalCachesManager,
+            generatedFiles: List<GeneratedFile<TargetId>>
+    ): Collection<File> {
+        val result = HashSet<File>()
+
+        fun partsByFacadeName(facadeInternalName: String): List<File> {
+            val parts = caches.incrementalCache.getStableMultifileFacadeParts(facadeInternalName) ?: emptyList()
+            return parts.flatMap { caches.incrementalCache.sourcesByInternalName(it) }
+        }
+
        for (generatedFile in generatedFiles) {
            if (generatedFile !is GeneratedJvmClass<*>) continue

            val outputClass = generatedFile.outputClass
-            if (outputClass.classHeader.kind != KotlinClassHeader.Kind.CLASS) continue
-            assert(generatedFile.sourceFiles.size == 1) { "KotlinClassHeader.Kind.CLASS cannot be generated from multiple files" }

-            val fqName = outputClass.className.fqNameForClassNameWithoutDollars
-            val currentSourceFile = generatedFile.sourceFiles.single()
-            val cachedSourceFile = caches.incrementalCache.getSourceFileIfClass(fqName)
+            when (outputClass.classHeader.kind) {
+                KotlinClassHeader.Kind.CLASS -> {
+                    val fqName = outputClass.className.fqNameForClassNameWithoutDollars
+                    val cachedSourceFile = caches.incrementalCache.getSourceFileIfClass(fqName)
+
+                    if (cachedSourceFile != null) {
+                        result.add(cachedSourceFile)
+                    }
+                }
+                // todo: more optimal is to check if public API or parts list changed
+                KotlinClassHeader.Kind.MULTIFILE_CLASS -> {
+                    result.addAll(partsByFacadeName(outputClass.className.internalName))
+                }
+                KotlinClassHeader.Kind.MULTIFILE_CLASS_PART -> {
+                    result.addAll(partsByFacadeName(outputClass.classHeader.multifileClassName!!))
+                }

-            if (cachedSourceFile != null && cachedSourceFile != currentSourceFile) {
-                result.add(cachedSourceFile)
            }
        }
+
        return result
    }