From ec62cccb1a379971eb7f55c1083bca611c926b7f Mon Sep 17 00:00:00 2001 From: Zalim Bashorov Date: Fri, 4 Sep 2015 18:30:04 +0300 Subject: [PATCH] Fix incremental compilation for new package parts --- .../IncrementalPackageFragmentProvider.kt | 27 ++++++++++-- .../components/IncrementalCache.kt | 6 +-- .../kotlinr/RemoteIncrementalCacheServer.kt | 8 ++-- .../jetbrains/kotlin/rmi/CompileService.kt | 5 ++- .../service/RemoteIncrementalCacheClient.kt | 4 +- .../kotlin/jps/build/KotlinBuilder.kt | 22 ++++++++-- .../jps/incremental/IncrementalCacheImpl.kt | 41 +++++++++++++------ 7 files changed, 85 insertions(+), 28 deletions(-) diff --git a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackageFragmentProvider.kt b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackageFragmentProvider.kt index c01b45f344a..eb2be0b2111 100644 --- a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackageFragmentProvider.kt +++ b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackageFragmentProvider.kt @@ -22,6 +22,7 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl +import org.jetbrains.kotlin.load.kotlin.ModuleMapping import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.name.FqName @@ -33,6 +34,7 @@ import org.jetbrains.kotlin.resolve.scopes.JetScope import org.jetbrains.kotlin.serialization.PackageData import org.jetbrains.kotlin.serialization.ProtoBuf import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedNewPackageMemberScope import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope import org.jetbrains.kotlin.serialization.jvm.JvmProtoBuf import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil @@ -40,7 +42,6 @@ import org.jetbrains.kotlin.storage.NotNullLazyValue import org.jetbrains.kotlin.storage.StorageManager import org.jetbrains.kotlin.utils.addToStdlib.singletonOrEmptyList import java.util.* -import kotlin.reflect.jvm.* public class IncrementalPackageFragmentProvider( sourceFiles: Collection, @@ -101,12 +102,30 @@ public class IncrementalPackageFragmentProvider( JetScope.Empty } else { - val packageDataBytes = incrementalCache.getPackageData(fqName.asString()) - if (packageDataBytes == null) { + val moduleMapping = incrementalCache.getModuleMappingData()?.let { ModuleMapping(it) } + + val actualPackagePartFiles = + moduleMapping?.findPackageParts(fqName.asString())?.let { + val allParts = + if (it.packageFqName.isEmpty()) { + it.parts + } + else { + val packageFqName = it.packageFqName.replace('.', '/') + it.parts.map { packageFqName + "/" + it } + } + + allParts.filterNot { it in obsoletePackageParts } + } ?: emptyList() + + val dataOfPackageParts = actualPackagePartFiles.map { incrementalCache.getPackagePartData(it) }.filterNotNull() + + if (dataOfPackageParts.isEmpty()) { JetScope.Empty } else { - DecapitalizedAnnotationScope.wrapIfNeeded(IncrementalPackageScope(JvmProtoBufUtil.readPackageDataFrom(packageDataBytes)), fqName) + val scopes = dataOfPackageParts.map { IncrementalPackageScope(JvmProtoBufUtil.readPackageDataFrom(it)) } + DecapitalizedAnnotationScope.wrapIfNeeded(DeserializedNewPackageMemberScope(this, scopes), fqName) } } } diff --git a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/components/IncrementalCache.kt b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/components/IncrementalCache.kt index c67a424517f..73ac7ad1e86 100644 --- a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/components/IncrementalCache.kt +++ b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/components/IncrementalCache.kt @@ -16,12 +16,12 @@ package org.jetbrains.kotlin.load.kotlin.incremental.components -import java.io.File - public interface IncrementalCache { public fun getObsoletePackageParts(): Collection - public fun getPackageData(fqName: String): ByteArray? + public fun getPackagePartData(fqName: String): ByteArray? + + public fun getModuleMappingData(): ByteArray? public fun close() } diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt index c2baf48baf0..4c3de63fccb 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt @@ -16,8 +16,8 @@ package org.jetbrains.kotlin.rmi.kotlinr +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.rmi.CompileService -import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; import java.rmi.server.UnicastRemoteObject @@ -29,7 +29,9 @@ public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : Compile override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() - override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) + override fun getPackagePartData(fqName: String): ByteArray? = cache.getPackagePartData(fqName) + + override fun getModuleMappingData(): ByteArray? = cache.getModuleMappingData() override fun close() { cache.close() @@ -38,4 +40,4 @@ public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : Compile public fun disconnect() { UnicastRemoteObject.unexportObject(this, true) } -} \ No newline at end of file +} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt index 5d5cd9c7530..8316e18b5ee 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt @@ -31,7 +31,10 @@ public interface CompileService : Remote { public fun getObsoletePackageParts(): Collection throws(RemoteException::class) - public fun getPackageData(fqName: String): ByteArray? + public fun getPackagePartData(fqName: String): ByteArray? + + throws(RemoteException::class) + public fun getModuleMappingData(): ByteArray? throws(RemoteException::class) public fun close() diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt index 900c1e088d2..29d6e952f46 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt @@ -22,7 +22,9 @@ import org.jetbrains.kotlin.rmi.CompileService public class RemoteIncrementalCacheClient(val cache: CompileService.RemoteIncrementalCache): IncrementalCache { override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() - override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) + override fun getPackagePartData(fqName: String): ByteArray? = cache.getPackagePartData(fqName) + + override fun getModuleMappingData(): ByteArray? = cache.getModuleMappingData() override fun close(): Unit = cache.close() } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 99f08de277d..e32d9645cb2 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -57,6 +57,7 @@ import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDe import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL_IN_CHUNK_AND_DEPENDANTS import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_OTHER_IN_CHUNK_AND_DEPENDANTS import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_OTHER_KOTLIN_IN_CHUNK +import org.jetbrains.kotlin.load.kotlin.ModuleMapping import org.jetbrains.kotlin.load.kotlin.PackageClassUtils import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache @@ -198,7 +199,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR } else { val generatedClasses = generatedFiles.filterIsInstance() - recompilationDecision = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedClasses) + recompilationDecision = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedFiles) updateJavaMappings(chunk, compilationErrors, context, dirtyFilesHolder, filesToCompile, generatedClasses) } @@ -410,7 +411,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR private fun updateKotlinIncrementalCache( compilationErrors: Boolean, incrementalCaches: Map, - generatedClasses: List + generatedFiles: List ): IncrementalCacheImpl.RecompilationDecision { incrementalCaches.values().forEach { it.saveCacheFormatVersion() } @@ -419,8 +420,19 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR } var recompilationDecision = DO_NOTHING - for (generatedClass in generatedClasses) { - val newDecision = incrementalCaches[generatedClass.target]!!.saveFileToCache(generatedClass.sourceFiles, generatedClass.outputClass) + for (generatedFile in generatedFiles) { + val ic = incrementalCaches[generatedFile.target]!! + val newDecision = + if (generatedFile is GeneratedJvmClass) { + ic.saveFileToCache(generatedFile.sourceFiles, generatedFile.outputClass) + } + else if (generatedFile.outputFile.isModuleMappingFile()) { + ic.saveModuleMappingToCache(generatedFile.outputFile) + } + else { + continue + } + recompilationDecision = recompilationDecision.merge(newDecision) } @@ -434,6 +446,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR return recompilationDecision } + private fun File.isModuleMappingFile() = extension == ModuleMapping.MAPPING_FILE_EXT && parentFile.name == "META-INF" + // if null is returned, nothing was done private fun compileToJs(chunk: ModuleChunk, commonArguments: CommonCompilerArguments, diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index 98d10853da4..eb0cc335fd1 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -33,25 +33,22 @@ import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDe import org.jetbrains.kotlin.jps.incremental.IncrementalCacheImpl.RecompilationDecision.RECOMPILE_OTHER_KOTLIN_IN_CHUNK import org.jetbrains.kotlin.load.java.JvmAbi import org.jetbrains.kotlin.load.java.JvmAnnotationNames -import org.jetbrains.kotlin.load.kotlin.PackageClassUtils +import org.jetbrains.kotlin.load.kotlin.ModuleMapping import org.jetbrains.kotlin.load.kotlin.header.isCompatibleClassKind import org.jetbrains.kotlin.load.kotlin.header.isCompatibleFileFacadeKind import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.serialization.Flags import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.visibility import org.jetbrains.kotlin.serialization.jvm.BitEncoding import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil -import org.jetbrains.kotlin.serialization.deserialization.visibility import org.jetbrains.kotlin.utils.Printer import org.jetbrains.org.objectweb.asm.* import java.io.* import java.security.MessageDigest -import java.util.ArrayList -import java.util.Arrays -import java.util.HashMap +import java.util.* val INLINE_ANNOTATION_DESC = "Lkotlin/inline;" @@ -105,6 +102,8 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen val PACKAGE_PARTS = "package-parts.tab" val SOURCE_TO_CLASSES = "source-to-classes.tab" val DIRTY_OUTPUT_CLASSES = "dirty-output-classes.tab" + + private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT } private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME) @@ -145,6 +144,12 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen cacheFormatVersion.saveIfNeeded() } + public fun saveModuleMappingToCache(file: File): RecompilationDecision { + protoMap.put(JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME), file.readBytes(), isPackage = false, checkChangesIsOpenPart = false) + dirtyOutputClassesMap.notDirty(MODULE_MAPPING_FILE_NAME) + return DO_NOTHING + } + public fun saveFileToCache(sourceFiles: Collection, kotlinClass: LocalFileKotlinClass): RecompilationDecision { val fileBytes = kotlinClass.getFileContents() val className = JvmClassName.byClassId(kotlinClass.getClassId()) @@ -160,6 +165,15 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen constantsChanged = false, inlinesChanged = false ) + header.isCompatibleFileFacadeKind() -> { + assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" } + packagePartMap.addPackagePart(className) + getRecompilationDecision( + protoChanged = protoMap.put(className, BitEncoding.decodeBytes(header.annotationData!!), isPackage = true), + constantsChanged = constantsMap.process(className, fileBytes), + inlinesChanged = inlineFunctionsMap.process(className, fileBytes) + ) + } header.isCompatibleClassKind() -> when (header.classKind!!) { JvmAnnotationNames.KotlinClass.Kind.CLASS -> getRecompilationDecision( @@ -170,7 +184,6 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen JvmAnnotationNames.KotlinClass.Kind.LOCAL_CLASS, JvmAnnotationNames.KotlinClass.Kind.ANONYMOUS_OBJECT -> DO_NOTHING } - header.isCompatibleFileFacadeKind() || header.syntheticClassKind == JvmAnnotationNames.KotlinSyntheticClass.Kind.PACKAGE_PART -> { assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" } @@ -217,15 +230,19 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return recompilationDecision } - public override fun getObsoletePackageParts(): Collection { + override fun getObsoletePackageParts(): Collection { val obsoletePackageParts = dirtyOutputClassesMap.getDirtyOutputClasses().filter { packagePartMap.isPackagePart(JvmClassName.byInternalName(it)) } KotlinBuilder.LOG.debug("Obsolete package parts: ${obsoletePackageParts}") return obsoletePackageParts } - public override fun getPackageData(fqName: String): ByteArray? { - return protoMap[JvmClassName.byFqNameWithoutInnerClasses(PackageClassUtils.getPackageClassFqName(FqName(fqName)))] + override fun getPackagePartData(fqName: String): ByteArray? { + return protoMap[JvmClassName.byInternalName(fqName)] + } + + override fun getModuleMappingData(): ByteArray? { + return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)] } override fun flush(memoryCachesOnly: Boolean) { @@ -306,7 +323,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen ByteArrayExternalizer ) - public fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean): Boolean { + public fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): Boolean { val key = className.getInternalName() val oldData = storage[key] if (Arrays.equals(data, oldData)) { @@ -314,7 +331,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen } storage.put(key, data) - if (oldData != null && isOpenPartNotChanged(oldData, data, isPackage)) { + if (oldData != null && checkChangesIsOpenPart && isOpenPartNotChanged(oldData, data, isPackage)) { return false }