From c3170d19087b51640a93d042ac78b24348a67ee1 Mon Sep 17 00:00:00 2001 From: Anton Bannykh Date: Fri, 12 Apr 2019 20:54:49 +0300 Subject: [PATCH] JS: cache metadata, grouped by package FQN --- .../kotlin/incremental/IncrementalJsCache.kt | 63 ++++++++++++++++++- .../js/IncrementalDataProviderFromCache.kt | 3 + .../jetbrains/kotlin/cli/js/K2JSCompiler.java | 13 +++- .../CompilerCallbackServicesFacadeServer.kt | 9 +++ .../common/CompilerCallbackServicesFacade.kt | 17 +++++ .../daemon/RemoteIncrementalDataProvider.kt | 10 +++ .../RemoteIncrementalResultsConsumer.kt | 7 +++ .../jps/build/AbstractLookupTrackerTest.kt | 2 +- .../incremental/js/IncrementalDataProvider.kt | 7 ++- .../js/IncrementalResultsConsumer.kt | 11 ++++ .../js/KotlinJavascriptSerializationUtil.kt | 49 +++++++++------ .../jetbrains/kotlin/js/test/BasicBoxTest.kt | 22 +++++-- .../kotlin/js/facade/K2JSTranslator.kt | 46 ++++++++------ 13 files changed, 214 insertions(+), 45 deletions(-) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt index daac22e4956..fb1636305b5 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt @@ -27,6 +27,7 @@ import org.jetbrains.kotlin.metadata.js.JsProtoBuf import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.name.parentOrNull import org.jetbrains.kotlin.serialization.deserialization.getClassId import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol import java.io.DataInput @@ -41,6 +42,7 @@ open class IncrementalJsCache( private const val TRANSLATION_RESULT_MAP = "translation-result" private const val INLINE_FUNCTIONS = "inline-functions" private const val HEADER_FILE_NAME = "header.meta" + private const val PACKAGE_META_FILE = "packages-meta" fun hasHeaderFile(cachesDir: File) = File(cachesDir, HEADER_FILE_NAME).exists() } @@ -49,8 +51,10 @@ open class IncrementalJsCache( override val dirtyOutputClassesMap = registerMap(DirtyClassesFqNameMap(DIRTY_OUTPUT_CLASSES.storageFile)) private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, pathConverter)) private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter)) + private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile)) private val dirtySources = hashSetOf() + private val dirtyPackages = hashSetOf() private val headerFile: File get() = File(cachesDir, HEADER_FILE_NAME) @@ -63,6 +67,11 @@ open class IncrementalJsCache( } override fun markDirty(removedAndCompiledSources: Collection) { + removedAndCompiledSources.forEach { sourceFile -> + sourceToClassesMap[sourceFile].forEach { + dirtyPackages += it.parentOrNull()?.asString() ?: "" + } + } super.markDirty(removedAndCompiledSources) dirtySources.addAll(removedAndCompiledSources) } @@ -95,6 +104,11 @@ open class IncrementalJsCache( for ((srcFile, inlineDeclarations) in incrementalResults.inlineFunctions) { inlineFunctions.process(srcFile, inlineDeclarations, changesCollector) } + + for ((packageName, metadata) in incrementalResults.packageMetadata) { + packageMetadata.put(packageName, metadata) + dirtyPackages.remove(packageName) + } } private fun registerOutputForFile(srcFile: File, name: FqName) { @@ -108,18 +122,31 @@ open class IncrementalJsCache( inlineFunctions.remove(it) } removeAllFromClassStorage(dirtyOutputClassesMap.getDirtyOutputClasses(), changesCollector) + dirtyPackages.forEach { + packageMetadata.remove(it) + } dirtySources.clear() dirtyOutputClassesMap.clean() + dirtyPackages.clear() } fun nonDirtyPackageParts(): Map = hashMapOf().apply { for (file in translationResults.keys()) { + if (file !in dirtySources) { put(file, translationResults[file]!!) } } } + + fun packageMetadata(): Map = hashMapOf().apply { + for (fqNameString in packageMetadata.keys()) { + if (fqNameString !in dirtyPackages) { + put(fqNameString, packageMetadata[fqNameString]!!) + } + } + } } private object TranslationResultValueExternalizer : DataExternalizer { @@ -154,7 +181,8 @@ private object TranslationResultValueExternalizer : DataExternalizer(storageFile, TranslationResultValueExternalizer) { +) : + BasicStringMap(storageFile, TranslationResultValueExternalizer) { override fun dumpValue(value: TranslationResultValue): String = "Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}, InlineData: ${value.inlineData.md5()}" @@ -181,7 +209,7 @@ private class TranslationResultMap( } } -fun getProtoData(sourceFile: File, metadata: ByteArray): Map { +fun getProtoData(sourceFile: File, metadata: ByteArray): Map { val classes = hashMapOf() val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry) val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames) @@ -227,4 +255,35 @@ private class InlineFunctionsMap( override fun dumpValue(value: Map): String = value.dumpMap { java.lang.Long.toHexString(it) } +} + +private object ByteArrayExternalizer : DataExternalizer { + override fun save(output: DataOutput, value: ByteArray) { + output.writeInt(value.size) + output.write(value) + } + + override fun read(input: DataInput): ByteArray { + val size = input.readInt() + val array = ByteArray(size) + input.readFully(array) + return array + } +} + + +private class PackageMetadataMap(storageFile: File) : BasicStringMap(storageFile, ByteArrayExternalizer) { + fun put(packageName: String, newMetadata: ByteArray) { + storage[packageName] = newMetadata + } + + fun remove(packageName: String) { + storage.remove(packageName) + } + + fun keys() = storage.keys + + operator fun get(packageName: String) = storage[packageName] + + override fun dumpValue(value: ByteArray): String = "Package metadata: ${value.md5()}" } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt index b30d6d47c5d..9375c946515 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt @@ -18,4 +18,7 @@ class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) : override val metadataVersion: IntArray get() = JsMetadataVersion.INSTANCE.toArray() // TODO: store and load correct metadata version + + override val packageMetadata: Map + get() = cache.packageMetadata() } diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java index e466922e259..253b2946ff8 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java @@ -65,6 +65,7 @@ import org.jetbrains.kotlin.js.facade.TranslationUnit; import org.jetbrains.kotlin.js.facade.exceptions.TranslationException; import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver; import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion; +import org.jetbrains.kotlin.name.FqName; import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus; import org.jetbrains.kotlin.psi.KtFile; import org.jetbrains.kotlin.serialization.js.ModuleKind; @@ -73,6 +74,7 @@ import org.jetbrains.kotlin.utils.*; import java.io.File; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; import static org.jetbrains.kotlin.cli.common.ExitCode.COMPILATION_ERROR; import static org.jetbrains.kotlin.cli.common.ExitCode.OK; @@ -144,6 +146,15 @@ public class K2JSCompiler extends CLICompiler { } Arrays.sort(allSources); + Set dirtyPackages = nonCompiledSources.values().stream().map(KtFile::getPackageFqName).collect(Collectors.toSet()); + Map packageMetadata = new HashMap<>(); + for (Map.Entry e : incrementalDataProvider.getPackageMetadata().entrySet()) { + FqName name = new FqName(e.getKey()); + if (!dirtyPackages.contains(name)) { + packageMetadata.put(name, e.getValue()); + } + } + List translationUnits = new ArrayList<>(); for (i = 0; i < allSources.length; i++) { KtFile nonCompiled = nonCompiledSources.get(allSources[i]); @@ -155,7 +166,7 @@ public class K2JSCompiler extends CLICompiler { translationUnits.add(new TranslationUnit.BinaryAst(translatedValue.getBinaryAst(), translatedValue.getInlineData())); } } - return translator.translateUnits(reporter, translationUnits, mainCallParameters, jsAnalysisResult); + return translator.translateUnits(reporter, translationUnits, mainCallParameters, jsAnalysisResult, packageMetadata); } CollectionsKt.sortBy(allKotlinFiles, ktFile -> VfsUtilCore.virtualToIoFile(ktFile.getVirtualFile())); diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/CompilerCallbackServicesFacadeServer.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/CompilerCallbackServicesFacadeServer.kt index f699b876abd..af4539901d3 100644 --- a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/CompilerCallbackServicesFacadeServer.kt +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/CompilerCallbackServicesFacadeServer.kt @@ -134,6 +134,10 @@ open class CompilerCallbackServicesFacadeServer( incrementalResultsConsumer!!.processInlineFunctions(functions) } + override fun incrementalResultsConsumer_processPackageMetadata(packageName: String, metadata: ByteArray) { + incrementalResultsConsumer!!.processPackageMetadata(packageName, metadata) + } + override fun incrementalDataProvider_getHeaderMetadata(): ByteArray = incrementalDataProvider!!.headerMetadata override fun incrementalDataProvider_getMetadataVersion(): IntArray = incrementalDataProvider!!.metadataVersion @@ -142,4 +146,9 @@ open class CompilerCallbackServicesFacadeServer( incrementalDataProvider!!.compiledPackageParts.entries.map { CompiledPackagePart(it.key.path, it.value.metadata, it.value.binaryAst, it.value.inlineData) } + + override fun incrementalDataProvider_getPackageMetadata(): Collection = + incrementalDataProvider!!.packageMetadata.entries.map { (fqName, metadata) -> + PackageMetadata(fqName, metadata) + } } diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacade.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacade.kt index 99331bceb33..c16c3e5e3c8 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacade.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacade.kt @@ -112,6 +112,9 @@ interface CompilerCallbackServicesFacade : Remote { @Throws(RemoteException::class) fun incrementalResultsConsumer_processInlineFunctions(functions: Collection) + @Throws(RemoteException::class) + fun incrementalResultsConsumer_processPackageMetadata(packageName: String, metadata: ByteArray) + // --------------------------------------------------- // IncrementalDataProvider (js) @Throws(RemoteException::class) @@ -122,6 +125,9 @@ interface CompilerCallbackServicesFacade : Remote { @Throws(RemoteException::class) fun incrementalDataProvider_getMetadataVersion(): IntArray + + @Throws(RemoteException::class) + fun incrementalDataProvider_getPackageMetadata(): Collection } class CompiledPackagePart( @@ -129,6 +135,17 @@ class CompiledPackagePart( val metadata: ByteArray, val binaryAst: ByteArray, val inlineData: ByteArray ) : Serializable +class PackageMetadata( + val packageName: String, + val metadata: ByteArray +) : Serializable { + companion object { + // just a random number, but should never be changed to avoid deserialization problems + private val serialVersionUID: Long = 54021986502349756L + } +} + + class RmiFriendlyCompilationCanceledException : Exception(), Serializable { companion object { // just a random number, but should never be changed to avoid deserialization problems diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt index 35e1276b9c5..891b14d7fa5 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt @@ -33,4 +33,14 @@ class RemoteIncrementalDataProvider(val facade: CompilerCallbackServicesFacade, get() = rpcProfiler.withMeasure(this) { facade.incrementalDataProvider_getMetadataVersion() } + + override val packageMetadata: Map + get() = rpcProfiler.withMeasure(this) { + val result = mutableMapOf() + facade.incrementalDataProvider_getPackageMetadata().forEach { + val prev = result.put(it.packageName, it.metadata) + check(prev == null) { "packageMetadata: duplicated entry for package `${it.packageName}`" } + } + result + } } diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt index c7cc6af584a..7e34ba05cd5 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt @@ -15,6 +15,7 @@ import java.io.File class RemoteIncrementalResultsConsumer(val facade: CompilerCallbackServicesFacade, eventManager: EventManager, val rpcProfiler: Profiler) : IncrementalResultsConsumer { + init { eventManager. onCompilationFinished(this::flush) @@ -42,6 +43,12 @@ class RemoteIncrementalResultsConsumer(val facade: CompilerCallbackServicesFacad override fun processInlineFunctions(functions: Collection) = error("Should not be called in Daemon Server") + override fun processPackageMetadata(packageName: String, metadata: ByteArray) { + rpcProfiler.withMeasure(this) { + facade.incrementalResultsConsumer_processPackageMetadata(packageName, metadata) + } + } + fun flush() { rpcProfiler.withMeasure(this) { facade.incrementalResultsConsumer_processInlineFunctions(deferInlineFuncs.map { diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt index 137ed8513fc..614a9ea9c1e 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt @@ -123,7 +123,7 @@ abstract class AbstractJsLookupTrackerTest : AbstractLookupTrackerTest() { if (header != null) { register( IncrementalDataProvider::class.java, - IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray()) + IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray(), emptyMap()) // TODO pass correct metadata ) } diff --git a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt index 7d94bfdba72..5dbbb0512b6 100644 --- a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt +++ b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.incremental.js +import org.jetbrains.kotlin.name.FqName import java.io.File // byte arrays are used to simplify passing to different classloaders @@ -27,10 +28,14 @@ interface IncrementalDataProvider { val compiledPackageParts: Map val metadataVersion: IntArray + + /** gets non-dirty package metadata from previous compilation */ + val packageMetadata: Map } class IncrementalDataProviderImpl( override val headerMetadata: ByteArray, override val compiledPackageParts: Map, - override val metadataVersion: IntArray + override val metadataVersion: IntArray, + override val packageMetadata: Map ) : IncrementalDataProvider diff --git a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt index 0ef5e4530d2..869eb49c238 100644 --- a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt +++ b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.incremental.js +import org.jetbrains.kotlin.name.FqName import java.io.File import java.io.Serializable import java.security.MessageDigest @@ -39,6 +40,8 @@ interface IncrementalResultsConsumer { * Used in daemon RPC. */ fun processInlineFunctions(functions: Collection) + + fun processPackageMetadata(packageName: String, metadata: ByteArray) } class JsInlineFunctionHash(val sourceFilePath: String, val fqName: String, val inlineFunctionMd5Hash: Long): Serializable @@ -97,6 +100,14 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer { val mapForSource = _deferInlineFuncs.getOrPut(sourceFile) { hashMapOf() } mapForSource[fqName] = FunctionWithSourceInfo(inlineFunction, line, column) } + + private val _packageMetadata = hashMapOf() + val packageMetadata: Map + get() = _packageMetadata + + override fun processPackageMetadata(packageName: String, metadata: ByteArray) { + _packageMetadata[packageName] = metadata + } } private fun ByteArray.md5(): Long { diff --git a/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt b/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt index 965effa32f4..04df9b5f2ee 100644 --- a/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt +++ b/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt @@ -75,20 +75,8 @@ object KotlinJavascriptSerializationUtil { languageVersionSettings: LanguageVersionSettings, metadataVersion: JsMetadataVersion ): SerializedMetadata { - val serializedFragments = HashMap() - val module = jsDescriptor.data - - for (fqName in getPackagesFqNames(module).sortedBy { it.asString() }) { - val fragment = serializeDescriptors( - bindingContext, module, - module.getPackage(fqName).memberScope.getContributedDescriptors(), - fqName, languageVersionSettings, metadataVersion - ) - - if (!fragment.isEmpty()) { - serializedFragments[fqName] = fragment.toByteArray() - } - } + val serializedFragments = + emptyMap().missingMetadata(bindingContext, jsDescriptor.data, languageVersionSettings, metadataVersion) return SerializedMetadata(serializedFragments, jsDescriptor, languageVersionSettings, metadataVersion) } @@ -258,9 +246,6 @@ object KotlinJavascriptSerializationUtil { return filesProto.build() } - private fun ProtoBuf.PackageFragment.isEmpty(): Boolean = - class_Count == 0 && `package`.let { it.functionCount == 0 && it.propertyCount == 0 && it.typeAliasCount == 0 } - fun serializeHeader( module: ModuleDescriptor, packageFqName: FqName?, languageVersionSettings: LanguageVersionSettings ): JsProtoBuf.Header { @@ -293,7 +278,7 @@ object KotlinJavascriptSerializationUtil { return header.build() } - private fun getPackagesFqNames(module: ModuleDescriptor): Set { + fun getPackagesFqNames(module: ModuleDescriptor): Set { return mutableSetOf().apply { getSubPackagesFqNames(module.getPackage(FqName.ROOT), this) add(FqName.ROOT) @@ -331,6 +316,34 @@ object KotlinJavascriptSerializationUtil { } } +fun Map.missingMetadata( + bindingContext: BindingContext, + moduleDescriptor: ModuleDescriptor, + languageVersionSettings: LanguageVersionSettings, + metadataVersion: JsMetadataVersion +): Map { + val serializedFragments = HashMap() + + for (fqName in KotlinJavascriptSerializationUtil.getPackagesFqNames(moduleDescriptor).sortedBy { it.asString() }) { + if (fqName in this) continue + + val fragment = KotlinJavascriptSerializationUtil.serializeDescriptors( + bindingContext, moduleDescriptor, + moduleDescriptor.getPackage(fqName).memberScope.getContributedDescriptors(), + fqName, languageVersionSettings, metadataVersion + ) + + if (!fragment.isEmpty()) { + serializedFragments[fqName] = fragment.toByteArray() + } + } + + return serializedFragments +} + +private fun ProtoBuf.PackageFragment.isEmpty(): Boolean = + class_Count == 0 && `package`.let { it.functionCount == 0 && it.propertyCount == 0 && it.typeAliasCount == 0 } + data class KotlinJavaScriptLibraryParts( val header: JsProtoBuf.Header, val body: List, diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt index 585b1baef21..676e8c50227 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt @@ -21,6 +21,7 @@ import org.jetbrains.kotlin.cli.common.output.writeAllTo import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.* +import org.jetbrains.kotlin.daemon.common.PackageMetadata import org.jetbrains.kotlin.idea.KotlinFileType import org.jetbrains.kotlin.incremental.js.IncrementalDataProviderImpl import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl @@ -48,6 +49,7 @@ import org.jetbrains.kotlin.js.test.interop.ScriptEngineV8Lazy import org.jetbrains.kotlin.js.test.utils.* import org.jetbrains.kotlin.js.util.TextOutputImpl import org.jetbrains.kotlin.metadata.DebugProtoBuf +import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.KtNamedFunction import org.jetbrains.kotlin.psi.KtPsiFactory @@ -395,6 +397,7 @@ abstract class BasicBoxTest( val sourceFile = File(testFile.fileName) incrementalData.translatedFiles.remove(sourceFile) sourceToTranslationUnit[sourceFile] = TranslationUnit.SourceFile(createPsiFile(testFile.fileName)) + incrementalData.packageMetadata.remove(testFile.packageName) } } for ((sourceFile, data) in incrementalData.translatedFiles) { @@ -461,7 +464,11 @@ abstract class BasicBoxTest( private fun removeRecompiledSuffix(text: String): String = text.replace("-recompiled.js", ".js") - class IncrementalData(var header: ByteArray? = null, val translatedFiles: MutableMap = hashMapOf()) + class IncrementalData( + var header: ByteArray? = null, + val translatedFiles: MutableMap = hashMapOf(), + val packageMetadata: MutableMap = hashMapOf() + ) protected open fun translateFiles( units: List, @@ -505,6 +512,8 @@ abstract class BasicBoxTest( incrementalData.translatedFiles[srcFile] = data } + incrementalData.packageMetadata += incrementalService.packageMetadata + incrementalData.header = incrementalService.headerMetadata } @@ -640,7 +649,7 @@ abstract class BasicBoxTest( if (header != null) { configuration.put( JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER, - IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray()) + IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray(), incrementalData.packageMetadata) ) } @@ -768,7 +777,12 @@ abstract class BasicBoxTest( currentModule.sourceMapSourceEmbedding = SourceMapSourceEmbedding.valueOf(match.groupValues[1]) } - return TestFile(temporaryFile.absolutePath, currentModule, recompile = RECOMPILE_PATTERN.matcher(text).find()) + return TestFile( + temporaryFile.absolutePath, + currentModule, + recompile = RECOMPILE_PATTERN.matcher(text).find(), + packageName = ktFile.packageFqName.asString() + ) } override fun createModule(name: String, dependencies: List, friends: List) = @@ -779,7 +793,7 @@ abstract class BasicBoxTest( } } - private class TestFile(val fileName: String, val module: TestModule, val recompile: Boolean) { + private class TestFile(val fileName: String, val module: TestModule, val recompile: Boolean, val packageName: String) { init { module.files += this } diff --git a/js/js.translator/src/org/jetbrains/kotlin/js/facade/K2JSTranslator.kt b/js/js.translator/src/org/jetbrains/kotlin/js/facade/K2JSTranslator.kt index d729159dc3f..d42920af994 100644 --- a/js/js.translator/src/org/jetbrains/kotlin/js/facade/K2JSTranslator.kt +++ b/js/js.translator/src/org/jetbrains/kotlin/js/facade/K2JSTranslator.kt @@ -54,6 +54,7 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.resolve.BindingTrace import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf +import org.jetbrains.kotlin.serialization.js.missingMetadata /** * An entry point of translator. @@ -140,8 +141,8 @@ class K2JSTranslator @JvmOverloads constructor( expandIsCalls(translationResult.newFragments) checkCanceled() - updatePackageMetadata(translationResult, bindingTrace, moduleDescriptor, packageMetadata) - trySaveIncrementalData(translationResult, pathResolver, bindingTrace, moduleDescriptor, packageMetadata) + updateMetadataMap(bindingTrace.bindingContext, moduleDescriptor, packageMetadata) + trySaveIncrementalData(translationResult, pathResolver, bindingTrace, moduleDescriptor) checkCanceled() // Global phases @@ -167,11 +168,34 @@ class K2JSTranslator @JvmOverloads constructor( } } + private fun updateMetadataMap( + bindingContext: BindingContext, + moduleDescriptor: ModuleDescriptor, + packageMetadata: MutableMap + ) { + val additionalMetadata = packageMetadata.missingMetadata( + bindingContext, + moduleDescriptor, + config.configuration.languageVersionSettings, + config.configuration.get(CommonConfigurationKeys.METADATA_VERSION) as? JsMetadataVersion ?: JsMetadataVersion.INSTANCE) + + for ((packageName, metadata) in additionalMetadata) { + incrementalResults?.processPackageMetadata(packageName.asString(), metadata) + } + + packageMetadata += additionalMetadata + } + private fun checkCanceled() { ProgressIndicatorAndCompilationCanceledStatus.checkCanceled() } - private fun serializeScope(bindingContext: BindingContext, moduleDescriptor: ModuleDescriptor, packageName: FqName, scope: Collection): ProtoBuf.PackageFragment { + private fun serializeScope( + bindingContext: BindingContext, + moduleDescriptor: ModuleDescriptor, + packageName: FqName, + scope: Collection + ): ProtoBuf.PackageFragment { val metadataVersion = config.configuration.get(CommonConfigurationKeys.METADATA_VERSION) return KotlinJavascriptSerializationUtil.serializeDescriptors( bindingContext, @@ -183,25 +207,11 @@ class K2JSTranslator @JvmOverloads constructor( ) } - private fun updatePackageMetadata( - translationResult: AstGenerationResult, - bindingTrace: BindingTrace, - moduleDescriptor: ModuleDescriptor, - packageMetadata: MutableMap - ) { - translationResult.translatedSourceFiles.keys.mapTo(mutableSetOf()) { it.file.packageFqName }.forEach { packageName -> - val memberScope = moduleDescriptor.getPackage(packageName).memberScope.getContributedDescriptors() - val bytes = serializeScope(bindingTrace.bindingContext, moduleDescriptor, packageName, memberScope).toByteArray() - packageMetadata[packageName] = bytes - } - } - private fun trySaveIncrementalData( translationResult: AstGenerationResult, pathResolver: SourceFilePathResolver, bindingTrace: BindingTrace, - moduleDescriptor: ModuleDescriptor, - packageMetadata: Map + moduleDescriptor: ModuleDescriptor ) { // TODO Maybe switch validation on for recompile if (incrementalResults == null && !shouldValidateJsAst) return