JS: improve binary representation format of metadata in .meta.js

Instead of writing many different files and serializing this "virtual file
system" to a byte array in a protobuf message, just write the needed stuff
directly, as fields in the Library message. Make it consist of many Part
messages, where the Part message is equivalent to the BuiltIns message in
builtins.proto. The next step would be to combine Library.Part and BuiltIns,
which will allow us to simplify some serialization-related code soon.

In this commit, no changes happened to the .kjsm format. But since the code
that serialized the abovementioned files was shared, a temporary abstraction
over two serialization formats was made, see SerializerCallbacks.

This commit temporarily breaks .kjsm decompiler and stub builder

idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/js/KotlinJavaScriptDeserializerForDecompiler.kt

@@ -31,10 +31,7 @@ import org.jetbrains.kotlin.resolve.TargetPlatform
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.*
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
-import org.jetbrains.kotlin.serialization.js.DynamicTypeDeserializer
+import org.jetbrains.kotlin.serialization.js.*
-import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol
-import org.jetbrains.kotlin.serialization.js.KotlinJavascriptClassDataFinder
-import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializedResourcePaths
 import java.io.ByteArrayInputStream
 
 class KotlinJavaScriptDeserializerForDecompiler(
@@ -52,9 +49,8 @@ class KotlinJavaScriptDeserializerForDecompiler(
     override val targetPlatform: TargetPlatform get() = JsPlatform
     override val builtIns: KotlinBuiltIns get() = DefaultBuiltIns.Instance
 
-    private val classDataFinder = KotlinJavascriptClassDataFinder(nameResolver) { path ->
+    // TODO: read metadata from .kjsm files here
-        packageDirectory.findChild(path.substringAfterLast("/"))?.inputStream
+    private val classDataFinder = KotlinJavascriptClassDataFinder(JsProtoBuf.Library.Part.getDefaultInstance(), nameResolver)
-    }
 
     override val deserializationComponents: DeserializationComponents
 

idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/js/KotlinJavaScriptStubBuilder.kt

@@ -32,6 +32,7 @@ import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.NameResolver
 import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl
 import org.jetbrains.kotlin.serialization.deserialization.TypeTable
+import org.jetbrains.kotlin.serialization.js.JsProtoBuf
 import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol
 import org.jetbrains.kotlin.serialization.js.KotlinJavascriptClassDataFinder
 import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializedResourcePaths
@@ -76,9 +77,8 @@ class KotlinJavaScriptStubBuilder : ClsStubBuilder() {
     }
 
     private fun createStubBuilderComponents(file: VirtualFile, nameResolver: NameResolver): ClsStubBuilderComponents {
-        val classDataFinder = KotlinJavascriptClassDataFinder(nameResolver) { path ->
+        // TODO: read metadata from .kjsm files here
-            file.parent.findChild(path.substringAfterLast("/"))?.inputStream
+        val classDataFinder = KotlinJavascriptClassDataFinder(JsProtoBuf.Library.Part.getDefaultInstance(), nameResolver)
-        }
         val annotationLoader = AnnotationLoaderForStubBuilderImpl(JsSerializerProtocol)
         return ClsStubBuilderComponents(classDataFinder, annotationLoader, file)
     }

js/js.serializer/src/js.proto

@@ -30,6 +30,10 @@ message Files {
   repeated File file = 1;
 }
 
+extend Package {
+  optional int32 package_fq_name = 131;
+}
+
 extend Class {
   repeated Annotation class_annotation = 130;
   optional int32 class_containing_file_id = 135;
@@ -72,11 +76,6 @@ message Classes {
 }
 
 message Library {
-  message FileEntry {
-    required string path = 1;
-    required bytes content = 2;
-  }
-
   enum Kind {
     PLAIN = 1;
     AMD = 2;
@@ -84,7 +83,22 @@ message Library {
     UMD = 4;
   }
 
-  repeated FileEntry entry = 1;
+  // This is a copy of BuiltIns in builtins.proto
-  optional Kind kind = 2 [default = PLAIN];
+  // TODO: combine the two
+  message Part {
+    optional StringTable strings = 1;
+    optional QualifiedNameTable qualified_names = 2;
+
+    optional Package package = 3;
+
+    repeated Class class = 4;
+
+    optional Files files = 130;
+  }
+
+  optional Kind kind = 1 [default = PLAIN];
+
+  repeated Part part = 2;
+
   repeated string imported_module = 3;
 }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptClassDataFinder.kt

@@ -20,18 +20,22 @@ import org.jetbrains.kotlin.descriptors.SourceElement
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.serialization.ClassData
 import org.jetbrains.kotlin.serialization.ClassDataWithSource
-import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
 import org.jetbrains.kotlin.serialization.deserialization.NameResolver
-import java.io.InputStream
 
 class KotlinJavascriptClassDataFinder(
-        private val nameResolver: NameResolver,
+        proto: JsProtoBuf.Library.Part,
-        private val loadResource: (path: String) -> InputStream?
+        private val nameResolver: NameResolver
 ) : ClassDataFinder {
+    private val classIdToProto =
+            proto.class_List.associateBy { klass ->
+                nameResolver.getClassId(klass.fqName)
+            }
+
+    internal val allClassIds: Collection<ClassId> get() = classIdToProto.keys
+
     override fun findClassData(classId: ClassId): ClassDataWithSource? {
-        val stream = loadResource(KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classId)) ?: return null
+        val classProto = classIdToProto[classId] ?: return null
-        val classProto = ProtoBuf.Class.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
         return ClassDataWithSource(ClassData(nameResolver, classProto), SourceElement.NO_SOURCE)
     }
 }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptPackageFragment.kt

@@ -20,63 +20,42 @@ import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.AnnotationDeserializer
 import org.jetbrains.kotlin.serialization.deserialization.DeserializedPackageFragment
-import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl
+import org.jetbrains.kotlin.serialization.deserialization.NameResolver
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPropertyDescriptor
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor
 import org.jetbrains.kotlin.storage.StorageManager
-import java.io.InputStream
+import org.jetbrains.kotlin.storage.getValue
 
 class KotlinJavascriptPackageFragment(
         fqName: FqName,
         storageManager: StorageManager,
         module: ModuleDescriptor,
-        private val loadResource: (path: String) -> InputStream?
+        private val proto: JsProtoBuf.Library.Part,
+        private val nameResolver: NameResolver
 ) : DeserializedPackageFragment(fqName, storageManager, module) {
-    private val nameResolver =
+    private val fileMap: Map<Int, FileHolder> by storageManager.createLazyValue {
-            loadResourceSure(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName)).use { stream ->
+        proto.files.fileList.associate { file -> file.id to FileHolder(file.annotationList) }
-                NameResolverImpl.read(stream)
-            }
-
-    private val fileMap: Map<Int, FileHolder> by lazy {
-        loadResource(KotlinJavascriptSerializedResourcePaths.getFileListFilePath(fqName))?.use { rawInput ->
-            val input = CodedInputStream.newInstance(rawInput)
-            val filesProto = JsProtoBuf.Files.parseFrom(input).fileOrBuilderList
-            filesProto.associate { it.id to FileHolder(it.annotationList) }
-        }.orEmpty()
     }
 
-    private val annotationDeserializer: AnnotationDeserializer by lazy {
+    private val annotationDeserializer: AnnotationDeserializer by storageManager.createLazyValue {
         AnnotationDeserializer(module, components.notFoundClasses)
     }
 
-    override val classDataFinder = KotlinJavascriptClassDataFinder(nameResolver, loadResource)
+    override val classDataFinder = KotlinJavascriptClassDataFinder(proto, nameResolver)
 
     override fun computeMemberScope(): DeserializedPackageMemberScope =
-            loadResourceSure(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName)).use { packageStream ->
+            DeserializedPackageMemberScope(
-                val packageProto = ProtoBuf.Package.parseFrom(packageStream, JsSerializerProtocol.extensionRegistry)
+                    this, proto.`package`, nameResolver, containerSource = null, components = components,
-                DeserializedPackageMemberScope(
+                    classNames = { classDataFinder.allClassIds.filterNot(ClassId::isNestedClass).map { it.shortClassName } }
-                        this, packageProto, nameResolver, containerSource = null, components = components,
+            )
-                        classNames = { loadClassNames() }
-                )
-            }
-
-    private fun loadClassNames(): Collection<Name> =
-            loadResourceSure(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName)).use { classesStream ->
-                val classesProto = JsProtoBuf.Classes.parseFrom(classesStream, JsSerializerProtocol.extensionRegistry)
-                classesProto.classNameList?.map { id -> nameResolver.getName(id) } ?: listOf()
-            }
-
-    private fun loadResourceSure(path: String): InputStream =
-            loadResource(path) ?: throw IllegalStateException("Resource not found in classpath: $path")
 
     fun getContainingFileAnnotations(descriptor: DeclarationDescriptor): List<AnnotationDescriptor> {
         if (DescriptorUtils.getParentOfType(descriptor, PackageFragmentDescriptor::class.java) != this) {
@@ -93,7 +72,7 @@ class KotlinJavascriptPackageFragment(
     }
 
     private inner class FileHolder(val annotationsProto: List<ProtoBuf.Annotation>) {
-        val annotations: List<AnnotationDescriptor> by lazy {
+        val annotations: List<AnnotationDescriptor> by storageManager.createLazyValue {
             annotationsProto.map { annotationDeserializer.deserializeAnnotation(it, nameResolver) }
         }
     }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt

@@ -17,8 +17,9 @@
 package org.jetbrains.kotlin.serialization.js
 
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.protobuf.ByteString
+import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.resolve.descriptorUtil.classId
@@ -40,97 +41,91 @@ import java.util.zip.GZIPOutputStream
 object KotlinJavascriptSerializationUtil {
     val CLASS_METADATA_FILE_EXTENSION: String = "kjsm"
 
-    private val PACKAGE_DEFAULT_BYTES = run {
-        val stream = ByteArrayOutputStream()
-        ProtoBuf.Package.getDefaultInstance().writeTo(stream)
-        stream.toByteArray()
-    }
-
-    private val CLASSES_IN_PACKAGE_DEFAULT_BYTES = run {
-        val stream = ByteArrayOutputStream()
-        JsProtoBuf.Classes.getDefaultInstance().writeTo(stream)
-        stream.toByteArray()
-    }
-
-    private val STRING_TABLE_DEFAULT_BYTES = run {
-        val serializer = DescriptorSerializer.createTopLevel(KotlinJavascriptSerializerExtension(KotlinFileRegistry()))
-        val stream = ByteArrayOutputStream()
-        serializer.stringTable.serializeTo(stream)
-        stream.toByteArray()
-    }
-
     @JvmStatic
     fun readModule(
-            metadata: ByteArray, storageManager: StorageManager, kotlinModule: ModuleDescriptor, configuration: DeserializationConfiguration
+            metadata: ByteArray, storageManager: StorageManager, module: ModuleDescriptor, configuration: DeserializationConfiguration
     ): JsModuleDescriptor<PackageFragmentProvider?> {
-        val jsModule = metadata.readAsContentMap(kotlinModule.name.asString())
+        val jsModule = metadata.deserializeToLibraryParts(module.name.asString())
-        return jsModule.copy(createPackageFragmentProvider(kotlinModule, jsModule.data, storageManager, configuration))
+        return jsModule.copy(createKotlinJavascriptPackageFragmentProvider(storageManager, module, jsModule.data, configuration))
     }
 
-    @JvmStatic
+    fun serializeMetadata(
-    private fun createPackageFragmentProvider(
+            bindingContext: BindingContext,
-            moduleDescriptor: ModuleDescriptor,
+            module: ModuleDescriptor,
-            contentMap: Map<String, ByteArray>,
+            moduleKind: ModuleKind,
-            storageManager: StorageManager,
+            importedModules: List<String>
-            configuration: DeserializationConfiguration
+    ): JsProtoBuf.Library {
-    ): PackageFragmentProvider? {
+        val builder = JsProtoBuf.Library.newBuilder()
-        val packageFqNames = getPackages(contentMap).map(::FqName).toSet()
-        if (packageFqNames.isEmpty()) return null
 
-        return createKotlinJavascriptPackageFragmentProvider(storageManager, moduleDescriptor, packageFqNames, configuration) { path ->
+        val moduleProtoKind = when (moduleKind) {
-            if (!contentMap.containsKey(path)) {
-                when {
-                    isPackageMetadataFile(path) ->
-                        ByteArrayInputStream(PACKAGE_DEFAULT_BYTES)
-                    isStringTableFile(path) ->
-                        ByteArrayInputStream(STRING_TABLE_DEFAULT_BYTES)
-                    isClassesInPackageFile(path) ->
-                        ByteArrayInputStream(CLASSES_IN_PACKAGE_DEFAULT_BYTES)
-                    else ->
-                        null
-                }
-            }
-            else ByteArrayInputStream(contentMap[path])
-        }
-    }
-
-    fun contentMapToByteArray(contentMap: Map<String, ByteArray>, moduleKind: ModuleKind, importedModules: List<String>): ByteArray {
-        val contentBuilder = JsProtoBuf.Library.newBuilder()
-
-        contentBuilder.kind = when (moduleKind) {
             ModuleKind.PLAIN -> JsProtoBuf.Library.Kind.PLAIN
             ModuleKind.AMD -> JsProtoBuf.Library.Kind.AMD
             ModuleKind.COMMON_JS -> JsProtoBuf.Library.Kind.COMMON_JS
             ModuleKind.UMD -> JsProtoBuf.Library.Kind.UMD
         }
-
+        if (builder.kind != moduleProtoKind) {
-        importedModules.forEach { contentBuilder.addImportedModule(it) }
+            builder.kind = moduleProtoKind
-
-        contentMap.forEach {
-            val entry = JsProtoBuf.Library.FileEntry.newBuilder().setPath(it.key).setContent(ByteString.copyFrom(it.value)).build()
-            contentBuilder.addEntry(entry)
         }
 
-        val byteStream = ByteArrayOutputStream()
+        importedModules.forEach { builder.addImportedModule(it) }
-        GZIPOutputStream(byteStream).use {
+
-            contentBuilder.build().writeTo(it)
+        for (fqName in getPackagesFqNames(module)) {
+            val part = JsProtoBuf.Library.Part.newBuilder()
+            serializePackage(bindingContext, module, fqName, object : SerializerCallbacks {
+                override fun writeClass(classId: ClassId, classProto: ProtoBuf.Class) {
+                    part.addClass_(classProto)
+                }
+
+                override fun writePackage(fqName: FqName, packageProto: ProtoBuf.Package) {
+                    part.`package` = packageProto
+                }
+
+                override fun writeFiles(fqName: FqName, filesProto: JsProtoBuf.Files) {
+                    part.files = filesProto
+                }
+
+                override fun writeStringTable(fqName: FqName, stringTable: StringTableImpl) {
+                    val (strings, qualifiedNames) = stringTable.buildProto()
+                    part.strings = strings
+                    part.qualifiedNames = qualifiedNames
+                }
+
+                override fun writeClassNames(fqName: FqName, classNames: List<Name>, stringTable: StringTableImpl) {
+                    // Do nothing
+                }
+            })
+
+            if (part.hasPackage() || part.class_Count > 0) {
+                builder.addPart(part)
+            }
         }
 
-        return byteStream.toByteArray()
+        return builder.build()
     }
 
     fun metadataAsString(bindingContext: BindingContext, jsDescriptor: JsModuleDescriptor<ModuleDescriptor>): String =
-        KotlinJavascriptMetadataUtils.formatMetadataAsString(jsDescriptor.name, jsDescriptor.toBinaryMetadata(bindingContext))
+            KotlinJavascriptMetadataUtils.formatMetadataAsString(jsDescriptor.name, jsDescriptor.serializeToBinaryMetadata(bindingContext))
 
-    fun serializePackage(bindingContext: BindingContext, module: ModuleDescriptor, fqName: FqName,
+    interface SerializerCallbacks {
-                         writeFun: (String, ByteArray) -> Unit) {
+        fun writeClass(classId: ClassId, classProto: ProtoBuf.Class)
+
+        fun writePackage(fqName: FqName, packageProto: ProtoBuf.Package)
+
+        fun writeFiles(fqName: FqName, filesProto: JsProtoBuf.Files)
+
+        fun writeStringTable(fqName: FqName, stringTable: StringTableImpl)
+
+        fun writeClassNames(fqName: FqName, classNames: List<Name>, stringTable: StringTableImpl)
+    }
+
+    fun serializePackage(bindingContext: BindingContext, module: ModuleDescriptor, fqName: FqName, callbacks: SerializerCallbacks) {
         val packageView = module.getPackage(fqName)
 
         // TODO: ModuleDescriptor should be able to return the package only with the contents of that module, without dependencies
         val skip: (DeclarationDescriptor) -> Boolean = { DescriptorUtils.getContainingModule(it) != module || (it is MemberDescriptor && it.isHeader) }
 
         val fileRegistry = KotlinFileRegistry()
-        val serializerExtension = KotlinJavascriptSerializerExtension(fileRegistry)
+        val serializerExtension = KotlinJavascriptSerializerExtension(fileRegistry, fqName)
         val serializer = DescriptorSerializer.createTopLevel(serializerExtension)
 
         val classifierDescriptors = DescriptorSerializer.sort(packageView.memberScope.getContributedDescriptors(
@@ -138,66 +133,36 @@ object KotlinJavascriptSerializationUtil {
 
         ClassSerializationUtil.serializeClasses(classifierDescriptors, serializer, object : ClassSerializationUtil.Sink {
             override fun writeClass(classDescriptor: ClassDescriptor, classProto: ProtoBuf.Class) {
-                val stream = ByteArrayOutputStream()
+                callbacks.writeClass(classDescriptor.classId!!, classProto)
-                classProto.writeTo(stream)
-                writeFun(getFileName(classDescriptor), stream.toByteArray())
             }
         }, skip)
 
-        val packageStream = ByteArrayOutputStream()
+        val stringTable = serializerExtension.stringTable
+
         val fragments = packageView.fragments
         val members = fragments
                 .flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) }
                 .filterNot(skip)
-        val packageProto = serializer.packagePartProto(members).build() ?: error("Package fragments not serialized: $fragments")
+        val packageProto = serializer.packagePartProto(members)
-        if (packageProto.functionCount > 0 || packageProto.propertyCount > 0 || packageProto.typeAliasCount > 0) {
+        packageProto.setExtension(JsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(fqName))
-            packageProto.writeTo(packageStream)
+        callbacks.writePackage(fqName, packageProto.build())
-            writeFun(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), packageStream.toByteArray())
-        }
 
-        writeFun(KotlinJavascriptSerializedResourcePaths.getFileListFilePath(fqName),
+        val fileTable = serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(stringTable))
-                 serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(serializer.stringTable)))
+        callbacks.writeFiles(fqName, fileTable)
 
-        val strings = serializerExtension.stringTable
+        val classNames = DescriptorSerializer.sort(fragments.flatMap { fragment ->
-        serializeClassNamesInPackage(fqName, fragments, strings, skip, writeFun)
+            fragment.getMemberScope().getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS).filterIsInstance<ClassDescriptor>()
+        }.filterNot(skip)).map { it.name }
+        callbacks.writeClassNames(fqName, classNames, stringTable)
 
-        val nameStream = ByteArrayOutputStream()
+        callbacks.writeStringTable(fqName, stringTable)
-        strings.serializeTo(nameStream)
-        val stringBytes = nameStream.toByteArray()
-
-        if (!stringBytes.isEmpty() && !Arrays.equals(stringBytes, STRING_TABLE_DEFAULT_BYTES)) {
-            writeFun(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName), stringBytes)
-        }
     }
 
-    private fun serializeClassNamesInPackage(
+    private fun serializeFiles(
-            fqName: FqName,
+            fileRegistry: KotlinFileRegistry,
-            packageFragments: Collection<PackageFragmentDescriptor>,
+            bindingContext: BindingContext,
-            stringTable: StringTableImpl,
+            serializer: AnnotationSerializer
-            skip: (DeclarationDescriptor) -> Boolean,
+    ): JsProtoBuf.Files {
-            writeFun: (String, ByteArray) -> Unit
-    ) {
-        val classes = packageFragments.flatMap {
-            it.getMemberScope().getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS).filterIsInstance<ClassDescriptor>()
-        }.filter { !skip(it) }
-
-        val builder = JsProtoBuf.Classes.newBuilder()
-
-        for (descriptor in DescriptorSerializer.sort(classes)) {
-            builder.addClassName(stringTable.getSimpleNameIndex(descriptor.name))
-        }
-
-        val classesProto = builder.build()
-
-        if (classesProto.classNameCount > 0) {
-            val stream = ByteArrayOutputStream()
-            classesProto.writeTo(stream)
-            writeFun(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName), stream.toByteArray())
-        }
-    }
-
-    private fun serializeFiles(fileRegistry: KotlinFileRegistry, bindingContext: BindingContext,
-                               serializer: AnnotationSerializer): ByteArray {
         val filesProto = JsProtoBuf.Files.newBuilder()
         for ((file, id) in fileRegistry.fileIds) {
             val fileProto = JsProtoBuf.File.newBuilder()
@@ -208,24 +173,52 @@ object KotlinJavascriptSerializationUtil {
             }
             filesProto.addFile(fileProto)
         }
-
+        return filesProto.build()
-        return ByteArrayOutputStream().use { out ->
-            filesProto.build().writeTo(out)
-            out
-        }.toByteArray()
-    }
-
-    private fun getFileName(classDescriptor: ClassDescriptor): String {
-        return KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classDescriptor.classId!!)
     }
 
     fun toContentMap(bindingContext: BindingContext, module: ModuleDescriptor): Map<String, ByteArray> {
         val contentMap = hashMapOf<String, ByteArray>()
 
-        getPackagesFqNames(module).forEach {
+        fun writeFile(fileName: String, bytes: ByteArray) {
-            serializePackage(bindingContext, module, it) {
+            contentMap[fileName] = bytes
-                fileName, bytes -> contentMap[fileName] = bytes
+        }
-            }
+
+        for (fqName in getPackagesFqNames(module)) {
+            serializePackage(bindingContext, module, fqName, object : SerializerCallbacks {
+                override fun writeClass(classId: ClassId, classProto: ProtoBuf.Class) {
+                    val bytes = ByteArrayOutputStream().apply(classProto::writeTo).toByteArray()
+                    writeFile(KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classId), bytes)
+                }
+
+                override fun writePackage(fqName: FqName, packageProto: ProtoBuf.Package) {
+                    if (packageProto.functionCount > 0 || packageProto.propertyCount > 0 || packageProto.typeAliasCount > 0) {
+                        val bytes = ByteArrayOutputStream().apply(packageProto::writeTo).toByteArray()
+                        writeFile(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), bytes)
+                    }
+                }
+
+                override fun writeFiles(fqName: FqName, filesProto: JsProtoBuf.Files) {
+                    val bytes = ByteArrayOutputStream().apply(filesProto::writeTo).toByteArray()
+                    writeFile(KotlinJavascriptSerializedResourcePaths.getFileListFilePath(fqName), bytes)
+                }
+
+                override fun writeStringTable(fqName: FqName, stringTable: StringTableImpl) {
+                    val bytes = ByteArrayOutputStream().apply(stringTable::serializeTo).toByteArray()
+                    if (bytes.isNotEmpty()) {
+                        writeFile(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName), bytes)
+                    }
+                }
+
+                override fun writeClassNames(fqName: FqName, classNames: List<Name>, stringTable: StringTableImpl) {
+                    val builder = JsProtoBuf.Classes.newBuilder()
+                    builder.addAllClassName(classNames.map(stringTable::getSimpleNameIndex))
+                    val classesProto = builder.build()
+                    if (classesProto.classNameCount > 0) {
+                        val bytes = ByteArrayOutputStream().apply(classesProto::writeTo).toByteArray()
+                        writeFile(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName), bytes)
+                    }
+                }
+            })
         }
 
         return contentMap
@@ -251,44 +244,28 @@ object KotlinJavascriptSerializationUtil {
         }
     }
 
-    private fun getPackages(contentMap: Map<String, ByteArray>): Set<String> {
+    private fun JsModuleDescriptor<ModuleDescriptor>.serializeToBinaryMetadata(bindingContext: BindingContext): ByteArray {
-        val keys = contentMap.keys.map { (if (it.startsWith('/')) it else "/" + it).substringBeforeLast('/') }.toSet()
+        val proto = serializeMetadata(bindingContext, data, kind, imported)
-
+        return ByteArrayOutputStream().apply {
-        val result = hashSetOf<String>()
+            GZIPOutputStream(this).use(proto::writeTo)
-
+        }.toByteArray()
-        fun addNames(name: String) {
-            result.add(name)
-            if (name != "") {
-                addNames(name.substringBeforeLast('/'))
-            }
-        }
-
-        keys.forEach { addNames(it) }
-
-        return result.map { it.substringAfter('/').replace('/', '.') }.toSet()
     }
 
-    private fun JsModuleDescriptor<ModuleDescriptor>.toBinaryMetadata(bindingContext: BindingContext) =
+    private fun ByteArray.deserializeToLibraryParts(name: String): JsModuleDescriptor<List<JsProtoBuf.Library.Part>> {
-            contentMapToByteArray(toContentMap(bindingContext, data), kind, imported)
+        val content = GZIPInputStream(ByteArrayInputStream(this)).use { stream ->
-}
+            JsProtoBuf.Library.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
-
+        }
-private fun ByteArray.readAsContentMap(name: String): JsModuleDescriptor<Map<String, ByteArray>> {
+
-    val gzipInputStream = GZIPInputStream(ByteArrayInputStream(this))
+        return JsModuleDescriptor(
-    val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
+                name = name,
-    gzipInputStream.close()
+                data = content.partList,
-
+                kind = when (content.kind) {
-    val contentMap: MutableMap<String, ByteArray> = hashMapOf()
+                    null, JsProtoBuf.Library.Kind.PLAIN -> ModuleKind.PLAIN
-    content.entryList.forEach { entry -> contentMap[entry.path] = entry.content.toByteArray() }
+                    JsProtoBuf.Library.Kind.AMD -> ModuleKind.AMD
-
+                    JsProtoBuf.Library.Kind.COMMON_JS -> ModuleKind.COMMON_JS
-    return JsModuleDescriptor(
+                    JsProtoBuf.Library.Kind.UMD -> ModuleKind.UMD
-            name = name,
+                },
-            data = contentMap,
+                imported = content.importedModuleList
-            kind = when (content.kind) {
+        )
-                null, JsProtoBuf.Library.Kind.PLAIN -> ModuleKind.PLAIN
+    }
-                JsProtoBuf.Library.Kind.AMD -> ModuleKind.AMD
-                JsProtoBuf.Library.Kind.COMMON_JS -> ModuleKind.COMMON_JS
-                JsProtoBuf.Library.Kind.UMD -> ModuleKind.UMD
-            },
-            imported = content.importedModuleList
-    )
 }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializedResourcePaths.kt

@@ -59,20 +59,6 @@ fun FqName.isPackageClassFqName(): Boolean = !this.isRoot && getPackageClassFqNa
 
 fun isDefaultPackageMetafile(fileName: String): Boolean = fileName == DEFAULT_PACKAGE_METAFILE_NAME
 
-fun isPackageMetadataFile(fileName: String): Boolean =
-        KotlinJavascriptSerializedResourcePaths.getPackageFilePath(getPackageFqName(fileName)) == fileName
-
-fun isStringTableFile(fileName: String): Boolean =
-        KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(getPackageFqName(fileName)) == fileName
-
-fun isClassesInPackageFile(fileName: String): Boolean =
-        KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(getPackageFqName(fileName)) == fileName
-
-private fun getPackageFqName(fileName: String): FqName = FqName(getPackageName(fileName))
-
-private fun getPackageName(filePath: String): String =
-        if (filePath.indexOf('/') >= 0) filePath.substringBeforeLast('/').replace('/', '.') else ""
-
 private fun getPackageClassFqName(packageFQN: FqName): FqName {
     return packageFQN.child(Name.identifier(getPackageClassName(packageFQN)))
 }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializerExtension.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.serialization.js
 
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.DescriptorUtils
@@ -24,10 +25,13 @@ import org.jetbrains.kotlin.resolve.source.PsiSourceFile
 import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
 import org.jetbrains.kotlin.types.FlexibleType
 
-class KotlinJavascriptSerializerExtension(private val fileRegistry: KotlinFileRegistry) :
+class KotlinJavascriptSerializerExtension(
-        KotlinSerializerExtensionBase(JsSerializerProtocol) {
+        private val fileRegistry: KotlinFileRegistry,
+        private val packageFqName: FqName
+) : KotlinSerializerExtensionBase(JsSerializerProtocol) {
     override val stringTable = JavaScriptStringTable()
 
     override fun serializeFlexibleType(flexibleType: FlexibleType, lowerProto: ProtoBuf.Type.Builder, upperProto: ProtoBuf.Type.Builder) {
@@ -42,6 +46,10 @@ class KotlinJavascriptSerializerExtension(private val fileRegistry: KotlinFileRe
         super.serializeClass(descriptor, proto)
     }
 
+    override fun serializePackage(proto: ProtoBuf.Package.Builder) {
+        proto.setExtension(BuiltInsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(packageFqName))
+    }
+
     override fun serializeProperty(descriptor: PropertyDescriptor, proto: ProtoBuf.Property.Builder) {
         val id = getFileId(descriptor)
         if (id != null) {

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/kotlinJavascriptPackageFragmentProvider.kt

@@ -20,21 +20,25 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.serialization.deserialization.*
 import org.jetbrains.kotlin.storage.StorageManager
-import java.io.InputStream
 
 fun createKotlinJavascriptPackageFragmentProvider(
         storageManager: StorageManager,
         module: ModuleDescriptor,
-        packageFqNames: Set<FqName>,
+        libraryParts: List<JsProtoBuf.Library.Part>,
-        configuration: DeserializationConfiguration,
+        configuration: DeserializationConfiguration
-        loadResource: (String) -> InputStream?
 ): PackageFragmentProvider {
-    val packageFragments = packageFqNames.map { fqName ->
+    val packageFragments = libraryParts.map { part ->
-        KotlinJavascriptPackageFragment(fqName, storageManager, module, loadResource)
+        val nameResolver = NameResolverImpl(part.strings, part.qualifiedNames)
+        val fqName = when {
+            part.hasPackage() -> nameResolver.getPackageFqName(part.`package`.getExtension(JsProtoBuf.packageFqName))
+            part.class_Count > 0 -> nameResolver.getClassId(part.class_OrBuilderList.first().fqName).packageFqName
+            else -> throw IllegalStateException("Invalid library part: either a Package or a Class must be present")
+        }
+        KotlinJavascriptPackageFragment(fqName, storageManager, module, part, nameResolver)
     }
+
     val provider = PackageFragmentProviderImpl(packageFragments)
 
     val notFoundClasses = NotFoundClasses(storageManager, module)