Fix incremental compilation for new package parts

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackageFragmentProvider.kt

@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
+import org.jetbrains.kotlin.load.kotlin.ModuleMapping
 import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.name.FqName
@@ -33,6 +34,7 @@ import org.jetbrains.kotlin.resolve.scopes.JetScope
 import org.jetbrains.kotlin.serialization.PackageData
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedNewPackageMemberScope
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBuf
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
@@ -40,7 +42,6 @@ import org.jetbrains.kotlin.storage.NotNullLazyValue
 import org.jetbrains.kotlin.storage.StorageManager
 import org.jetbrains.kotlin.utils.addToStdlib.singletonOrEmptyList
 import java.util.*
-import kotlin.reflect.jvm.*
 
 public class IncrementalPackageFragmentProvider(
         sourceFiles: Collection<JetFile>,
@@ -101,12 +102,30 @@ public class IncrementalPackageFragmentProvider(
                 JetScope.Empty
             }
             else {
-                val packageDataBytes = incrementalCache.getPackageData(fqName.asString())
+                val moduleMapping = incrementalCache.getModuleMappingData()?.let { ModuleMapping(it) }
-                if (packageDataBytes == null) {
+
+                val actualPackagePartFiles =
+                        moduleMapping?.findPackageParts(fqName.asString())?.let {
+                            val allParts =
+                                    if (it.packageFqName.isEmpty()) {
+                                        it.parts
+                                    }
+                                    else {
+                                        val packageFqName = it.packageFqName.replace('.', '/')
+                                        it.parts.map { packageFqName + "/" + it }
+                                    }
+
+                            allParts.filterNot { it in obsoletePackageParts }
+                        } ?: emptyList<String>()
+
+                val dataOfPackageParts = actualPackagePartFiles.map { incrementalCache.getPackagePartData(it) }.filterNotNull()
+
+                if (dataOfPackageParts.isEmpty()) {
                     JetScope.Empty
                 }
                 else {
-                    DecapitalizedAnnotationScope.wrapIfNeeded(IncrementalPackageScope(JvmProtoBufUtil.readPackageDataFrom(packageDataBytes)), fqName)
+                    val scopes = dataOfPackageParts.map { IncrementalPackageScope(JvmProtoBufUtil.readPackageDataFrom(it)) }
+                    DecapitalizedAnnotationScope.wrapIfNeeded(DeserializedNewPackageMemberScope(this, scopes), fqName)
                 }
             }
         }

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/components/IncrementalCache.kt

@@ -16,12 +16,12 @@
 
 package org.jetbrains.kotlin.load.kotlin.incremental.components
 
-import java.io.File
-
 public interface IncrementalCache {
     public fun getObsoletePackageParts(): Collection<String>
 
-    public fun getPackageData(fqName: String): ByteArray?
+    public fun getPackagePartData(fqName: String): ByteArray?
+
+    public fun getModuleMappingData(): ByteArray?
 
     public fun close()
 }

compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt

@@ -16,8 +16,8 @@
 
 package org.jetbrains.kotlin.rmi.kotlinr
 
+import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.rmi.CompileService
-import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache;
 import java.rmi.server.UnicastRemoteObject
 
 
@@ -29,7 +29,9 @@ public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : Compile
 
     override fun getObsoletePackageParts(): Collection<String> = cache.getObsoletePackageParts()
 
-    override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName)
+    override fun getPackagePartData(fqName: String): ByteArray? = cache.getPackagePartData(fqName)
+
+    override fun getModuleMappingData(): ByteArray? = cache.getModuleMappingData()
 
     override fun close() {
         cache.close()
@@ -38,4 +40,4 @@ public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : Compile
     public fun disconnect() {
         UnicastRemoteObject.unexportObject(this, true)
     }
-}
+}

compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt

@@ -31,7 +31,10 @@ public interface CompileService : Remote {
         public fun getObsoletePackageParts(): Collection<String>
 
         throws(RemoteException::class)
-        public fun getPackageData(fqName: String): ByteArray?
+        public fun getPackagePartData(fqName: String): ByteArray?
+
+        throws(RemoteException::class)
+        public fun getModuleMappingData(): ByteArray?
 
         throws(RemoteException::class)
         public fun close()

compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt

@@ -22,7 +22,9 @@ import org.jetbrains.kotlin.rmi.CompileService
 public class RemoteIncrementalCacheClient(val cache: CompileService.RemoteIncrementalCache): IncrementalCache {
     override fun getObsoletePackageParts(): Collection<String> = cache.getObsoletePackageParts()
 
-    override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName)
+    override fun getPackagePartData(fqName: String): ByteArray? = cache.getPackagePartData(fqName)
+
+    override fun getModuleMappingData(): ByteArray? = cache.getModuleMappingData()
 
     override fun close(): Unit = cache.close()
 }

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -57,6 +57,7 @@ import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDe
 import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL_IN_CHUNK_AND_DEPENDANTS
 import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_OTHER_IN_CHUNK_AND_DEPENDANTS
 import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_OTHER_KOTLIN_IN_CHUNK
+import org.jetbrains.kotlin.load.kotlin.ModuleMapping
 import org.jetbrains.kotlin.load.kotlin.PackageClassUtils
 import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
@@ -198,7 +199,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
         }
         else {
             val generatedClasses = generatedFiles.filterIsInstance<GeneratedJvmClass>()
-            recompilationDecision = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedClasses)
+            recompilationDecision = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedFiles)
             updateJavaMappings(chunk, compilationErrors, context, dirtyFilesHolder, filesToCompile, generatedClasses)
         }
 
@@ -410,7 +411,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
     private fun updateKotlinIncrementalCache(
             compilationErrors: Boolean,
             incrementalCaches: Map<ModuleBuildTarget, IncrementalCacheImpl>,
-            generatedClasses: List<GeneratedJvmClass>
+            generatedFiles: List<GeneratedFile>
     ): IncrementalCacheImpl.RecompilationDecision {
         incrementalCaches.values().forEach { it.saveCacheFormatVersion() }
 
@@ -419,8 +420,19 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
         }
 
         var recompilationDecision = DO_NOTHING
-        for (generatedClass in generatedClasses) {
+        for (generatedFile in generatedFiles) {
-            val newDecision = incrementalCaches[generatedClass.target]!!.saveFileToCache(generatedClass.sourceFiles, generatedClass.outputClass)
+            val ic = incrementalCaches[generatedFile.target]!!
+            val newDecision =
+                    if (generatedFile is GeneratedJvmClass) {
+                        ic.saveFileToCache(generatedFile.sourceFiles, generatedFile.outputClass)
+                    }
+                    else if (generatedFile.outputFile.isModuleMappingFile()) {
+                        ic.saveModuleMappingToCache(generatedFile.outputFile)
+                    }
+                    else {
+                        continue
+                    }
+
             recompilationDecision = recompilationDecision.merge(newDecision)
         }
 
@@ -434,6 +446,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
         return recompilationDecision
     }
 
+    private fun File.isModuleMappingFile() = extension == ModuleMapping.MAPPING_FILE_EXT && parentFile.name == "META-INF"
+
     // if null is returned, nothing was done
     private fun compileToJs(chunk: ModuleChunk,
                             commonArguments: CommonCompilerArguments,

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -33,25 +33,22 @@ import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDe
 import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_OTHER_KOTLIN_IN_CHUNK
 import org.jetbrains.kotlin.load.java.JvmAbi
 import org.jetbrains.kotlin.load.java.JvmAnnotationNames
-import org.jetbrains.kotlin.load.kotlin.PackageClassUtils
+import org.jetbrains.kotlin.load.kotlin.ModuleMapping
 import org.jetbrains.kotlin.load.kotlin.header.isCompatibleClassKind
 import org.jetbrains.kotlin.load.kotlin.header.isCompatibleFileFacadeKind
 import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
-import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import org.jetbrains.kotlin.serialization.Flags
 import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.visibility
 import org.jetbrains.kotlin.serialization.jvm.BitEncoding
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
-import org.jetbrains.kotlin.serialization.deserialization.visibility
 import org.jetbrains.kotlin.utils.Printer
 import org.jetbrains.org.objectweb.asm.*
 import java.io.*
 import java.security.MessageDigest
-import java.util.ArrayList
+import java.util.*
-import java.util.Arrays
-import java.util.HashMap
 
 val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
 
@@ -105,6 +102,8 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         val PACKAGE_PARTS = "package-parts.tab"
         val SOURCE_TO_CLASSES = "source-to-classes.tab"
         val DIRTY_OUTPUT_CLASSES = "dirty-output-classes.tab"
+
+        private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
     }
 
     private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME)
@@ -145,6 +144,12 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         cacheFormatVersion.saveIfNeeded()
     }
 
+    public fun saveModuleMappingToCache(file: File): RecompilationDecision {
+        protoMap.put(JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME), file.readBytes(), isPackage = false, checkChangesIsOpenPart = false)
+        dirtyOutputClassesMap.notDirty(MODULE_MAPPING_FILE_NAME)
+        return DO_NOTHING
+    }
+
     public fun saveFileToCache(sourceFiles: Collection<File>, kotlinClass: LocalFileKotlinClass): RecompilationDecision {
         val fileBytes = kotlinClass.getFileContents()
         val className = JvmClassName.byClassId(kotlinClass.getClassId())
@@ -160,6 +165,15 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
                         constantsChanged = false,
                         inlinesChanged = false
                 )
+            header.isCompatibleFileFacadeKind() -> {
+                assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" }
+                packagePartMap.addPackagePart(className)
+                getRecompilationDecision(
+                        protoChanged = protoMap.put(className, BitEncoding.decodeBytes(header.annotationData!!), isPackage = true),
+                        constantsChanged = constantsMap.process(className, fileBytes),
+                        inlinesChanged = inlineFunctionsMap.process(className, fileBytes)
+                )
+            }
             header.isCompatibleClassKind() ->
                 when (header.classKind!!) {
                     JvmAnnotationNames.KotlinClass.Kind.CLASS -> getRecompilationDecision(
@@ -170,7 +184,6 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
 
                     JvmAnnotationNames.KotlinClass.Kind.LOCAL_CLASS, JvmAnnotationNames.KotlinClass.Kind.ANONYMOUS_OBJECT -> DO_NOTHING
                 }
-            header.isCompatibleFileFacadeKind() ||
             header.syntheticClassKind == JvmAnnotationNames.KotlinSyntheticClass.Kind.PACKAGE_PART -> {
                 assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" }
 
@@ -217,15 +230,19 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         return recompilationDecision
     }
 
-    public override fun getObsoletePackageParts(): Collection<String> {
+    override fun getObsoletePackageParts(): Collection<String> {
         val obsoletePackageParts =
                 dirtyOutputClassesMap.getDirtyOutputClasses().filter { packagePartMap.isPackagePart(JvmClassName.byInternalName(it)) }
         KotlinBuilder.LOG.debug("Obsolete package parts: ${obsoletePackageParts}")
         return obsoletePackageParts
     }
 
-    public override fun getPackageData(fqName: String): ByteArray? {
+    override fun getPackagePartData(fqName: String): ByteArray? {
-        return protoMap[JvmClassName.byFqNameWithoutInnerClasses(PackageClassUtils.getPackageClassFqName(FqName(fqName)))]
+        return protoMap[JvmClassName.byInternalName(fqName)]
+    }
+
+    override fun getModuleMappingData(): ByteArray? {
+        return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]
     }
 
     override fun flush(memoryCachesOnly: Boolean) {
@@ -306,7 +323,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
                 ByteArrayExternalizer
         )
 
-        public fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean): Boolean {
+        public fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): Boolean {
             val key = className.getInternalName()
             val oldData = storage[key]
             if (Arrays.equals(data, oldData)) {
@@ -314,7 +331,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
             }
             storage.put(key, data)
 
-            if (oldData != null && isOpenPartNotChanged(oldData, data, isPackage)) {
+            if (oldData != null && checkChangesIsOpenPart && isOpenPartNotChanged(oldData, data, isPackage)) {
                 return false
             }