JS: improve binary representation format of metadata in .meta.js

Instead of writing many different files and serializing this "virtual file system" to a byte array in a protobuf message, just write the needed stuff directly, as fields in the Library message. Make it consist of many Part messages, where the Part message is equivalent to the BuiltIns message in builtins.proto. The next step would be to combine Library.Part and BuiltIns, which will allow us to simplify some serialization-related code soon. In this commit, no changes happened to the .kjsm format. But since the code that serialized the abovementioned files was shared, a temporary abstraction over two serialization formats was made, see SerializerCallbacks. This commit temporarily breaks .kjsm decompiler and stub builder
2017-01-24 14:05:43 +03:00
parent 77ea0614f9
commit 4b81dfb7c1
11 changed files with 2724 additions and 1327 deletions
@@ -31,10 +31,7 @@ import org.jetbrains.kotlin.resolve.TargetPlatform
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.*
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
-import org.jetbrains.kotlin.serialization.js.DynamicTypeDeserializer
-import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol
-import org.jetbrains.kotlin.serialization.js.KotlinJavascriptClassDataFinder
-import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializedResourcePaths
+import org.jetbrains.kotlin.serialization.js.*
 import java.io.ByteArrayInputStream

 class KotlinJavaScriptDeserializerForDecompiler(
@@ -52,9 +49,8 @@ class KotlinJavaScriptDeserializerForDecompiler(
    override val targetPlatform: TargetPlatform get() = JsPlatform
    override val builtIns: KotlinBuiltIns get() = DefaultBuiltIns.Instance

-    private val classDataFinder = KotlinJavascriptClassDataFinder(nameResolver) { path ->
-        packageDirectory.findChild(path.substringAfterLast("/"))?.inputStream
-    }
+    // TODO: read metadata from .kjsm files here
+    private val classDataFinder = KotlinJavascriptClassDataFinder(JsProtoBuf.Library.Part.getDefaultInstance(), nameResolver)

    override val deserializationComponents: DeserializationComponents

@@ -32,6 +32,7 @@ import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.NameResolver
 import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl
 import org.jetbrains.kotlin.serialization.deserialization.TypeTable
+import org.jetbrains.kotlin.serialization.js.JsProtoBuf
 import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol
 import org.jetbrains.kotlin.serialization.js.KotlinJavascriptClassDataFinder
 import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializedResourcePaths
@@ -76,9 +77,8 @@ class KotlinJavaScriptStubBuilder : ClsStubBuilder() {
    }

    private fun createStubBuilderComponents(file: VirtualFile, nameResolver: NameResolver): ClsStubBuilderComponents {
-        val classDataFinder = KotlinJavascriptClassDataFinder(nameResolver) { path ->
-            file.parent.findChild(path.substringAfterLast("/"))?.inputStream
-        }
+        // TODO: read metadata from .kjsm files here
+        val classDataFinder = KotlinJavascriptClassDataFinder(JsProtoBuf.Library.Part.getDefaultInstance(), nameResolver)
        val annotationLoader = AnnotationLoaderForStubBuilderImpl(JsSerializerProtocol)
        return ClsStubBuilderComponents(classDataFinder, annotationLoader, file)
    }
@@ -30,6 +30,10 @@ message Files {
  repeated File file = 1;
 }

+extend Package {
+  optional int32 package_fq_name = 131;
+}
+
 extend Class {
  repeated Annotation class_annotation = 130;
  optional int32 class_containing_file_id = 135;
@@ -72,11 +76,6 @@ message Classes {
 }

 message Library {
-  message FileEntry {
-    required string path = 1;
-    required bytes content = 2;
-  }
-
  enum Kind {
    PLAIN = 1;
    AMD = 2;
@@ -84,7 +83,22 @@ message Library {
    UMD = 4;
  }

-  repeated FileEntry entry = 1;
-  optional Kind kind = 2 [default = PLAIN];
+  // This is a copy of BuiltIns in builtins.proto
+  // TODO: combine the two
+  message Part {
+    optional StringTable strings = 1;
+    optional QualifiedNameTable qualified_names = 2;
+
+    optional Package package = 3;
+
+    repeated Class class = 4;
+
+    optional Files files = 130;
+  }
+
+  optional Kind kind = 1 [default = PLAIN];
+
+  repeated Part part = 2;
+
  repeated string imported_module = 3;
 }
@@ -20,18 +20,22 @@ import org.jetbrains.kotlin.descriptors.SourceElement
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.serialization.ClassData
 import org.jetbrains.kotlin.serialization.ClassDataWithSource
-import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
 import org.jetbrains.kotlin.serialization.deserialization.NameResolver
-import java.io.InputStream

 class KotlinJavascriptClassDataFinder(
-        private val nameResolver: NameResolver,
-        private val loadResource: (path: String) -> InputStream?
+        proto: JsProtoBuf.Library.Part,
+        private val nameResolver: NameResolver
 ) : ClassDataFinder {
+    private val classIdToProto =
+            proto.class_List.associateBy { klass ->
+                nameResolver.getClassId(klass.fqName)
+            }
+
+    internal val allClassIds: Collection<ClassId> get() = classIdToProto.keys
+
    override fun findClassData(classId: ClassId): ClassDataWithSource? {
-        val stream = loadResource(KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classId)) ?: return null
-        val classProto = ProtoBuf.Class.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
+        val classProto = classIdToProto[classId] ?: return null
        return ClassDataWithSource(ClassData(nameResolver, classProto), SourceElement.NO_SOURCE)
    }
 }
@@ -20,63 +20,42 @@ import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.AnnotationDeserializer
 import org.jetbrains.kotlin.serialization.deserialization.DeserializedPackageFragment
-import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl
+import org.jetbrains.kotlin.serialization.deserialization.NameResolver
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPropertyDescriptor
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor
 import org.jetbrains.kotlin.storage.StorageManager
-import java.io.InputStream
+import org.jetbrains.kotlin.storage.getValue

 class KotlinJavascriptPackageFragment(
        fqName: FqName,
        storageManager: StorageManager,
        module: ModuleDescriptor,
-        private val loadResource: (path: String) -> InputStream?
+        private val proto: JsProtoBuf.Library.Part,
+        private val nameResolver: NameResolver
 ) : DeserializedPackageFragment(fqName, storageManager, module) {
-    private val nameResolver =
-            loadResourceSure(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName)).use { stream ->
-                NameResolverImpl.read(stream)
-            }
-
-    private val fileMap: Map<Int, FileHolder> by lazy {
-        loadResource(KotlinJavascriptSerializedResourcePaths.getFileListFilePath(fqName))?.use { rawInput ->
-            val input = CodedInputStream.newInstance(rawInput)
-            val filesProto = JsProtoBuf.Files.parseFrom(input).fileOrBuilderList
-            filesProto.associate { it.id to FileHolder(it.annotationList) }
-        }.orEmpty()
+    private val fileMap: Map<Int, FileHolder> by storageManager.createLazyValue {
+        proto.files.fileList.associate { file -> file.id to FileHolder(file.annotationList) }
    }

-    private val annotationDeserializer: AnnotationDeserializer by lazy {
+    private val annotationDeserializer: AnnotationDeserializer by storageManager.createLazyValue {
        AnnotationDeserializer(module, components.notFoundClasses)
    }

-    override val classDataFinder = KotlinJavascriptClassDataFinder(nameResolver, loadResource)
+    override val classDataFinder = KotlinJavascriptClassDataFinder(proto, nameResolver)

    override fun computeMemberScope(): DeserializedPackageMemberScope =
-            loadResourceSure(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName)).use { packageStream ->
-                val packageProto = ProtoBuf.Package.parseFrom(packageStream, JsSerializerProtocol.extensionRegistry)
-                DeserializedPackageMemberScope(
-                        this, packageProto, nameResolver, containerSource = null, components = components,
-                        classNames = { loadClassNames() }
-                )
-            }
-
-    private fun loadClassNames(): Collection<Name> =
-            loadResourceSure(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName)).use { classesStream ->
-                val classesProto = JsProtoBuf.Classes.parseFrom(classesStream, JsSerializerProtocol.extensionRegistry)
-                classesProto.classNameList?.map { id -> nameResolver.getName(id) } ?: listOf()
-            }
-
-    private fun loadResourceSure(path: String): InputStream =
-            loadResource(path) ?: throw IllegalStateException("Resource not found in classpath: $path")
+            DeserializedPackageMemberScope(
+                    this, proto.`package`, nameResolver, containerSource = null, components = components,
+                    classNames = { classDataFinder.allClassIds.filterNot(ClassId::isNestedClass).map { it.shortClassName } }
+            )

    fun getContainingFileAnnotations(descriptor: DeclarationDescriptor): List<AnnotationDescriptor> {
        if (DescriptorUtils.getParentOfType(descriptor, PackageFragmentDescriptor::class.java) != this) {
@@ -93,7 +72,7 @@ class KotlinJavascriptPackageFragment(
    }

    private inner class FileHolder(val annotationsProto: List<ProtoBuf.Annotation>) {
-        val annotations: List<AnnotationDescriptor> by lazy {
+        val annotations: List<AnnotationDescriptor> by storageManager.createLazyValue {
            annotationsProto.map { annotationDeserializer.deserializeAnnotation(it, nameResolver) }
        }
    }
@@ -17,8 +17,9 @@
 package org.jetbrains.kotlin.serialization.js

 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.protobuf.ByteString
+import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.resolve.descriptorUtil.classId
@@ -40,97 +41,91 @@ import java.util.zip.GZIPOutputStream
 object KotlinJavascriptSerializationUtil {
    val CLASS_METADATA_FILE_EXTENSION: String = "kjsm"

-    private val PACKAGE_DEFAULT_BYTES = run {
-        val stream = ByteArrayOutputStream()
-        ProtoBuf.Package.getDefaultInstance().writeTo(stream)
-        stream.toByteArray()
-    }
-
-    private val CLASSES_IN_PACKAGE_DEFAULT_BYTES = run {
-        val stream = ByteArrayOutputStream()
-        JsProtoBuf.Classes.getDefaultInstance().writeTo(stream)
-        stream.toByteArray()
-    }
-
-    private val STRING_TABLE_DEFAULT_BYTES = run {
-        val serializer = DescriptorSerializer.createTopLevel(KotlinJavascriptSerializerExtension(KotlinFileRegistry()))
-        val stream = ByteArrayOutputStream()
-        serializer.stringTable.serializeTo(stream)
-        stream.toByteArray()
-    }
-
    @JvmStatic
    fun readModule(
-            metadata: ByteArray, storageManager: StorageManager, kotlinModule: ModuleDescriptor, configuration: DeserializationConfiguration
+            metadata: ByteArray, storageManager: StorageManager, module: ModuleDescriptor, configuration: DeserializationConfiguration
    ): JsModuleDescriptor<PackageFragmentProvider?> {
-        val jsModule = metadata.readAsContentMap(kotlinModule.name.asString())
-        return jsModule.copy(createPackageFragmentProvider(kotlinModule, jsModule.data, storageManager, configuration))
+        val jsModule = metadata.deserializeToLibraryParts(module.name.asString())
+        return jsModule.copy(createKotlinJavascriptPackageFragmentProvider(storageManager, module, jsModule.data, configuration))
    }

-    @JvmStatic
-    private fun createPackageFragmentProvider(
-            moduleDescriptor: ModuleDescriptor,
-            contentMap: Map<String, ByteArray>,
-            storageManager: StorageManager,
-            configuration: DeserializationConfiguration
-    ): PackageFragmentProvider? {
-        val packageFqNames = getPackages(contentMap).map(::FqName).toSet()
-        if (packageFqNames.isEmpty()) return null
+    fun serializeMetadata(
+            bindingContext: BindingContext,
+            module: ModuleDescriptor,
+            moduleKind: ModuleKind,
+            importedModules: List<String>
+    ): JsProtoBuf.Library {
+        val builder = JsProtoBuf.Library.newBuilder()

-        return createKotlinJavascriptPackageFragmentProvider(storageManager, moduleDescriptor, packageFqNames, configuration) { path ->
-            if (!contentMap.containsKey(path)) {
-                when {
-                    isPackageMetadataFile(path) ->
-                        ByteArrayInputStream(PACKAGE_DEFAULT_BYTES)
-                    isStringTableFile(path) ->
-                        ByteArrayInputStream(STRING_TABLE_DEFAULT_BYTES)
-                    isClassesInPackageFile(path) ->
-                        ByteArrayInputStream(CLASSES_IN_PACKAGE_DEFAULT_BYTES)
-                    else ->
-                        null
-                }
-            }
-            else ByteArrayInputStream(contentMap[path])
-        }
-    }
-
-    fun contentMapToByteArray(contentMap: Map<String, ByteArray>, moduleKind: ModuleKind, importedModules: List<String>): ByteArray {
-        val contentBuilder = JsProtoBuf.Library.newBuilder()
-
-        contentBuilder.kind = when (moduleKind) {
+        val moduleProtoKind = when (moduleKind) {
            ModuleKind.PLAIN -> JsProtoBuf.Library.Kind.PLAIN
            ModuleKind.AMD -> JsProtoBuf.Library.Kind.AMD
            ModuleKind.COMMON_JS -> JsProtoBuf.Library.Kind.COMMON_JS
            ModuleKind.UMD -> JsProtoBuf.Library.Kind.UMD
        }
-
-        importedModules.forEach { contentBuilder.addImportedModule(it) }
-
-        contentMap.forEach {
-            val entry = JsProtoBuf.Library.FileEntry.newBuilder().setPath(it.key).setContent(ByteString.copyFrom(it.value)).build()
-            contentBuilder.addEntry(entry)
+        if (builder.kind != moduleProtoKind) {
+            builder.kind = moduleProtoKind
        }

-        val byteStream = ByteArrayOutputStream()
-        GZIPOutputStream(byteStream).use {
-            contentBuilder.build().writeTo(it)
+        importedModules.forEach { builder.addImportedModule(it) }
+
+        for (fqName in getPackagesFqNames(module)) {
+            val part = JsProtoBuf.Library.Part.newBuilder()
+            serializePackage(bindingContext, module, fqName, object : SerializerCallbacks {
+                override fun writeClass(classId: ClassId, classProto: ProtoBuf.Class) {
+                    part.addClass_(classProto)
+                }
+
+                override fun writePackage(fqName: FqName, packageProto: ProtoBuf.Package) {
+                    part.`package` = packageProto
+                }
+
+                override fun writeFiles(fqName: FqName, filesProto: JsProtoBuf.Files) {
+                    part.files = filesProto
+                }
+
+                override fun writeStringTable(fqName: FqName, stringTable: StringTableImpl) {
+                    val (strings, qualifiedNames) = stringTable.buildProto()
+                    part.strings = strings
+                    part.qualifiedNames = qualifiedNames
+                }
+
+                override fun writeClassNames(fqName: FqName, classNames: List<Name>, stringTable: StringTableImpl) {
+                    // Do nothing
+                }
+            })
+
+            if (part.hasPackage() || part.class_Count > 0) {
+                builder.addPart(part)
+            }
        }

-        return byteStream.toByteArray()
+        return builder.build()
    }

    fun metadataAsString(bindingContext: BindingContext, jsDescriptor: JsModuleDescriptor<ModuleDescriptor>): String =
-        KotlinJavascriptMetadataUtils.formatMetadataAsString(jsDescriptor.name, jsDescriptor.toBinaryMetadata(bindingContext))
+            KotlinJavascriptMetadataUtils.formatMetadataAsString(jsDescriptor.name, jsDescriptor.serializeToBinaryMetadata(bindingContext))

-    fun serializePackage(bindingContext: BindingContext, module: ModuleDescriptor, fqName: FqName,
-                         writeFun: (String, ByteArray) -> Unit) {
+    interface SerializerCallbacks {
+        fun writeClass(classId: ClassId, classProto: ProtoBuf.Class)
+
+        fun writePackage(fqName: FqName, packageProto: ProtoBuf.Package)
+
+        fun writeFiles(fqName: FqName, filesProto: JsProtoBuf.Files)
+
+        fun writeStringTable(fqName: FqName, stringTable: StringTableImpl)
+
+        fun writeClassNames(fqName: FqName, classNames: List<Name>, stringTable: StringTableImpl)
+    }
+
+    fun serializePackage(bindingContext: BindingContext, module: ModuleDescriptor, fqName: FqName, callbacks: SerializerCallbacks) {
        val packageView = module.getPackage(fqName)

        // TODO: ModuleDescriptor should be able to return the package only with the contents of that module, without dependencies
        val skip: (DeclarationDescriptor) -> Boolean = { DescriptorUtils.getContainingModule(it) != module || (it is MemberDescriptor && it.isHeader) }

        val fileRegistry = KotlinFileRegistry()
-        val serializerExtension = KotlinJavascriptSerializerExtension(fileRegistry)
+        val serializerExtension = KotlinJavascriptSerializerExtension(fileRegistry, fqName)
        val serializer = DescriptorSerializer.createTopLevel(serializerExtension)

        val classifierDescriptors = DescriptorSerializer.sort(packageView.memberScope.getContributedDescriptors(
@@ -138,66 +133,36 @@ object KotlinJavascriptSerializationUtil {

        ClassSerializationUtil.serializeClasses(classifierDescriptors, serializer, object : ClassSerializationUtil.Sink {
            override fun writeClass(classDescriptor: ClassDescriptor, classProto: ProtoBuf.Class) {
-                val stream = ByteArrayOutputStream()
-                classProto.writeTo(stream)
-                writeFun(getFileName(classDescriptor), stream.toByteArray())
+                callbacks.writeClass(classDescriptor.classId!!, classProto)
            }
        }, skip)

-        val packageStream = ByteArrayOutputStream()
+        val stringTable = serializerExtension.stringTable
+
        val fragments = packageView.fragments
        val members = fragments
                .flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) }
                .filterNot(skip)
-        val packageProto = serializer.packagePartProto(members).build() ?: error("Package fragments not serialized: $fragments")
-        if (packageProto.functionCount > 0 || packageProto.propertyCount > 0 || packageProto.typeAliasCount > 0) {
-            packageProto.writeTo(packageStream)
-            writeFun(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), packageStream.toByteArray())
-        }
+        val packageProto = serializer.packagePartProto(members)
+        packageProto.setExtension(JsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(fqName))
+        callbacks.writePackage(fqName, packageProto.build())

-        writeFun(KotlinJavascriptSerializedResourcePaths.getFileListFilePath(fqName),
-                 serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(serializer.stringTable)))
+        val fileTable = serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(stringTable))
+        callbacks.writeFiles(fqName, fileTable)

-        val strings = serializerExtension.stringTable
-        serializeClassNamesInPackage(fqName, fragments, strings, skip, writeFun)
+        val classNames = DescriptorSerializer.sort(fragments.flatMap { fragment ->
+            fragment.getMemberScope().getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS).filterIsInstance<ClassDescriptor>()
+        }.filterNot(skip)).map { it.name }
+        callbacks.writeClassNames(fqName, classNames, stringTable)

-        val nameStream = ByteArrayOutputStream()
-        strings.serializeTo(nameStream)
-        val stringBytes = nameStream.toByteArray()
-
-        if (!stringBytes.isEmpty() && !Arrays.equals(stringBytes, STRING_TABLE_DEFAULT_BYTES)) {
-            writeFun(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName), stringBytes)
-        }
+        callbacks.writeStringTable(fqName, stringTable)
    }

-    private fun serializeClassNamesInPackage(
-            fqName: FqName,
-            packageFragments: Collection<PackageFragmentDescriptor>,
-            stringTable: StringTableImpl,
-            skip: (DeclarationDescriptor) -> Boolean,
-            writeFun: (String, ByteArray) -> Unit
-    ) {
-        val classes = packageFragments.flatMap {
-            it.getMemberScope().getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS).filterIsInstance<ClassDescriptor>()
-        }.filter { !skip(it) }
-
-        val builder = JsProtoBuf.Classes.newBuilder()
-
-        for (descriptor in DescriptorSerializer.sort(classes)) {
-            builder.addClassName(stringTable.getSimpleNameIndex(descriptor.name))
-        }
-
-        val classesProto = builder.build()
-
-        if (classesProto.classNameCount > 0) {
-            val stream = ByteArrayOutputStream()
-            classesProto.writeTo(stream)
-            writeFun(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName), stream.toByteArray())
-        }
-    }
-
-    private fun serializeFiles(fileRegistry: KotlinFileRegistry, bindingContext: BindingContext,
-                               serializer: AnnotationSerializer): ByteArray {
+    private fun serializeFiles(
+            fileRegistry: KotlinFileRegistry,
+            bindingContext: BindingContext,
+            serializer: AnnotationSerializer
+    ): JsProtoBuf.Files {
        val filesProto = JsProtoBuf.Files.newBuilder()
        for ((file, id) in fileRegistry.fileIds) {
            val fileProto = JsProtoBuf.File.newBuilder()
@@ -208,24 +173,52 @@ object KotlinJavascriptSerializationUtil {
            }
            filesProto.addFile(fileProto)
        }
-
-        return ByteArrayOutputStream().use { out ->
-            filesProto.build().writeTo(out)
-            out
-        }.toByteArray()
-    }
-
-    private fun getFileName(classDescriptor: ClassDescriptor): String {
-        return KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classDescriptor.classId!!)
+        return filesProto.build()
    }

    fun toContentMap(bindingContext: BindingContext, module: ModuleDescriptor): Map<String, ByteArray> {
        val contentMap = hashMapOf<String, ByteArray>()

-        getPackagesFqNames(module).forEach {
-            serializePackage(bindingContext, module, it) {
-                fileName, bytes -> contentMap[fileName] = bytes
-            }
+        fun writeFile(fileName: String, bytes: ByteArray) {
+            contentMap[fileName] = bytes
+        }
+
+        for (fqName in getPackagesFqNames(module)) {
+            serializePackage(bindingContext, module, fqName, object : SerializerCallbacks {
+                override fun writeClass(classId: ClassId, classProto: ProtoBuf.Class) {
+                    val bytes = ByteArrayOutputStream().apply(classProto::writeTo).toByteArray()
+                    writeFile(KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classId), bytes)
+                }
+
+                override fun writePackage(fqName: FqName, packageProto: ProtoBuf.Package) {
+                    if (packageProto.functionCount > 0 || packageProto.propertyCount > 0 || packageProto.typeAliasCount > 0) {
+                        val bytes = ByteArrayOutputStream().apply(packageProto::writeTo).toByteArray()
+                        writeFile(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), bytes)
+                    }
+                }
+
+                override fun writeFiles(fqName: FqName, filesProto: JsProtoBuf.Files) {
+                    val bytes = ByteArrayOutputStream().apply(filesProto::writeTo).toByteArray()
+                    writeFile(KotlinJavascriptSerializedResourcePaths.getFileListFilePath(fqName), bytes)
+                }
+
+                override fun writeStringTable(fqName: FqName, stringTable: StringTableImpl) {
+                    val bytes = ByteArrayOutputStream().apply(stringTable::serializeTo).toByteArray()
+                    if (bytes.isNotEmpty()) {
+                        writeFile(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName), bytes)
+                    }
+                }
+
+                override fun writeClassNames(fqName: FqName, classNames: List<Name>, stringTable: StringTableImpl) {
+                    val builder = JsProtoBuf.Classes.newBuilder()
+                    builder.addAllClassName(classNames.map(stringTable::getSimpleNameIndex))
+                    val classesProto = builder.build()
+                    if (classesProto.classNameCount > 0) {
+                        val bytes = ByteArrayOutputStream().apply(classesProto::writeTo).toByteArray()
+                        writeFile(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName), bytes)
+                    }
+                }
+            })
        }

        return contentMap
@@ -251,44 +244,28 @@ object KotlinJavascriptSerializationUtil {
        }
    }

-    private fun getPackages(contentMap: Map<String, ByteArray>): Set<String> {
-        val keys = contentMap.keys.map { (if (it.startsWith('/')) it else "/" + it).substringBeforeLast('/') }.toSet()
-
-        val result = hashSetOf<String>()
-
-        fun addNames(name: String) {
-            result.add(name)
-            if (name != "") {
-                addNames(name.substringBeforeLast('/'))
-            }
-        }
-
-        keys.forEach { addNames(it) }
-
-        return result.map { it.substringAfter('/').replace('/', '.') }.toSet()
+    private fun JsModuleDescriptor<ModuleDescriptor>.serializeToBinaryMetadata(bindingContext: BindingContext): ByteArray {
+        val proto = serializeMetadata(bindingContext, data, kind, imported)
+        return ByteArrayOutputStream().apply {
+            GZIPOutputStream(this).use(proto::writeTo)
+        }.toByteArray()
    }

-    private fun JsModuleDescriptor<ModuleDescriptor>.toBinaryMetadata(bindingContext: BindingContext) =
-            contentMapToByteArray(toContentMap(bindingContext, data), kind, imported)
-}
-
-private fun ByteArray.readAsContentMap(name: String): JsModuleDescriptor<Map<String, ByteArray>> {
-    val gzipInputStream = GZIPInputStream(ByteArrayInputStream(this))
-    val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
-    gzipInputStream.close()
-
-    val contentMap: MutableMap<String, ByteArray> = hashMapOf()
-    content.entryList.forEach { entry -> contentMap[entry.path] = entry.content.toByteArray() }
-
-    return JsModuleDescriptor(
-            name = name,
-            data = contentMap,
-            kind = when (content.kind) {
-                null, JsProtoBuf.Library.Kind.PLAIN -> ModuleKind.PLAIN
-                JsProtoBuf.Library.Kind.AMD -> ModuleKind.AMD
-                JsProtoBuf.Library.Kind.COMMON_JS -> ModuleKind.COMMON_JS
-                JsProtoBuf.Library.Kind.UMD -> ModuleKind.UMD
-            },
-            imported = content.importedModuleList
-    )
+    private fun ByteArray.deserializeToLibraryParts(name: String): JsModuleDescriptor<List<JsProtoBuf.Library.Part>> {
+        val content = GZIPInputStream(ByteArrayInputStream(this)).use { stream ->
+            JsProtoBuf.Library.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
+        }
+
+        return JsModuleDescriptor(
+                name = name,
+                data = content.partList,
+                kind = when (content.kind) {
+                    null, JsProtoBuf.Library.Kind.PLAIN -> ModuleKind.PLAIN
+                    JsProtoBuf.Library.Kind.AMD -> ModuleKind.AMD
+                    JsProtoBuf.Library.Kind.COMMON_JS -> ModuleKind.COMMON_JS
+                    JsProtoBuf.Library.Kind.UMD -> ModuleKind.UMD
+                },
+                imported = content.importedModuleList
+        )
+    }
 }
@@ -59,20 +59,6 @@ fun FqName.isPackageClassFqName(): Boolean = !this.isRoot && getPackageClassFqNa

 fun isDefaultPackageMetafile(fileName: String): Boolean = fileName == DEFAULT_PACKAGE_METAFILE_NAME

-fun isPackageMetadataFile(fileName: String): Boolean =
-        KotlinJavascriptSerializedResourcePaths.getPackageFilePath(getPackageFqName(fileName)) == fileName
-
-fun isStringTableFile(fileName: String): Boolean =
-        KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(getPackageFqName(fileName)) == fileName
-
-fun isClassesInPackageFile(fileName: String): Boolean =
-        KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(getPackageFqName(fileName)) == fileName
-
-private fun getPackageFqName(fileName: String): FqName = FqName(getPackageName(fileName))
-
-private fun getPackageName(filePath: String): String =
-        if (filePath.indexOf('/') >= 0) filePath.substringBeforeLast('/').replace('/', '.') else ""
-
 private fun getPackageClassFqName(packageFQN: FqName): FqName {
    return packageFQN.child(Name.identifier(getPackageClassName(packageFQN)))
 }
@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.serialization.js

 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.DescriptorUtils
@@ -24,10 +25,13 @@ import org.jetbrains.kotlin.resolve.source.PsiSourceFile
 import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
 import org.jetbrains.kotlin.types.FlexibleType

-class KotlinJavascriptSerializerExtension(private val fileRegistry: KotlinFileRegistry) :
-        KotlinSerializerExtensionBase(JsSerializerProtocol) {
+class KotlinJavascriptSerializerExtension(
+        private val fileRegistry: KotlinFileRegistry,
+        private val packageFqName: FqName
+) : KotlinSerializerExtensionBase(JsSerializerProtocol) {
    override val stringTable = JavaScriptStringTable()

    override fun serializeFlexibleType(flexibleType: FlexibleType, lowerProto: ProtoBuf.Type.Builder, upperProto: ProtoBuf.Type.Builder) {
@@ -42,6 +46,10 @@ class KotlinJavascriptSerializerExtension(private val fileRegistry: KotlinFileRe
        super.serializeClass(descriptor, proto)
    }

+    override fun serializePackage(proto: ProtoBuf.Package.Builder) {
+        proto.setExtension(BuiltInsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(packageFqName))
+    }
+
    override fun serializeProperty(descriptor: PropertyDescriptor, proto: ProtoBuf.Property.Builder) {
        val id = getFileId(descriptor)
        if (id != null) {
@@ -20,21 +20,25 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.serialization.deserialization.*
 import org.jetbrains.kotlin.storage.StorageManager
-import java.io.InputStream

 fun createKotlinJavascriptPackageFragmentProvider(
        storageManager: StorageManager,
        module: ModuleDescriptor,
-        packageFqNames: Set<FqName>,
-        configuration: DeserializationConfiguration,
-        loadResource: (String) -> InputStream?
+        libraryParts: List<JsProtoBuf.Library.Part>,
+        configuration: DeserializationConfiguration
 ): PackageFragmentProvider {
-    val packageFragments = packageFqNames.map { fqName ->
-        KotlinJavascriptPackageFragment(fqName, storageManager, module, loadResource)
+    val packageFragments = libraryParts.map { part ->
+        val nameResolver = NameResolverImpl(part.strings, part.qualifiedNames)
+        val fqName = when {
+            part.hasPackage() -> nameResolver.getPackageFqName(part.`package`.getExtension(JsProtoBuf.packageFqName))
+            part.class_Count > 0 -> nameResolver.getClassId(part.class_OrBuilderList.first().fqName).packageFqName
+            else -> throw IllegalStateException("Invalid library part: either a Package or a Class must be present")
+        }
+        KotlinJavascriptPackageFragment(fqName, storageManager, module, part, nameResolver)
    }
+
    val provider = PackageFragmentProviderImpl(packageFragments)

    val notFoundClasses = NotFoundClasses(storageManager, module)