Serialize .kotlin_metadata binary files per source file

Do not serialize everything in the same package to the same file (as is done
for built-ins) because this approach is unfriendly to incremental compilation,
which is going to be supported in the future. Instead, similarly to JVM
serialize each class to its own file, and each source file with top-level
callables/typealiases to its own file.

E.g. if a file named test.kt contains a class Foo and some
functions/properties, the output will contain two files: TestKt.kotlin_metadata
and Foo.kotlin_metadata. Each one of this files contains the serialized
BuiltIns message (see builtins.proto)

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt

@@ -18,7 +18,6 @@ package org.jetbrains.kotlin.serialization
 
 import org.jetbrains.kotlin.analyzer.AnalysisResult
 import org.jetbrains.kotlin.analyzer.common.DefaultAnalyzerFacade
-import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
 import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
@@ -28,12 +27,15 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
 import org.jetbrains.kotlin.descriptors.ClassKind
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
-import org.jetbrains.kotlin.descriptors.PackageViewDescriptor
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.psi.*
+import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
-import org.jetbrains.kotlin.resolve.scopes.MemberScope
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension
 import java.io.ByteArrayOutputStream
@@ -44,9 +46,6 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
     protected var totalSize = 0
     protected var totalFiles = 0
 
-    protected open fun getPackageFilePath(fqName: FqName): String =
-            fqName.asString().replace('.', '/') + "/" + (if (fqName.isRoot) "default-package" else fqName.shortName().asString()) + "." + METADATA_FILE_EXTENSION
-
     fun serialize(environment: KotlinCoreEnvironment) {
         val configuration = environment.configuration
         val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
@@ -63,36 +62,69 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
             override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns)
         })
 
-        val moduleDescriptor = analyzer.analysisResult.moduleDescriptor
+        if (analyzer.hasErrors()) return
 
-        destDir.deleteRecursively()
+        val (bindingContext, moduleDescriptor) = analyzer.analysisResult
 
-        if (!destDir.mkdirs()) {
+        performSerialization(files, bindingContext, moduleDescriptor, destDir)
-            throw AssertionError("Could not make directories: " + destDir)
+    }
-        }
 
-        files.map { it.packageFqName }.toSet().forEach {
+    protected open fun performSerialization(
-            fqName ->
+            files: Collection<KtFile>, bindingContext: BindingContext, module: ModuleDescriptor, destDir: File
-            PackageSerializer(moduleDescriptor.getPackage(fqName), destDir, { bytesWritten ->
+    ) {
-                totalSize += bytesWritten
+        for (file in files) {
-                totalFiles++
+            val packageFqName = file.packageFqName
-            }).run()
+            val members = arrayListOf<DeclarationDescriptor>()
+            for (declaration in file.declarations) {
+                declaration.accept(object : KtVisitorVoid() {
+                    override fun visitNamedFunction(function: KtNamedFunction) {
+                        members.add(bindingContext.get(BindingContext.FUNCTION, function)
+                                    ?: error("No descriptor found for function ${function.fqName}"))
+                    }
+
+                    override fun visitProperty(property: KtProperty) {
+                        members.add(bindingContext.get(BindingContext.VARIABLE, property)
+                                    ?: error("No descriptor found for property ${property.fqName}"))
+                    }
+
+                    override fun visitTypeAlias(typeAlias: KtTypeAlias) {
+                        members.add(bindingContext.get(BindingContext.TYPE_ALIAS, typeAlias)
+                                    ?: error("No descriptor found for type alias ${typeAlias.fqName}"))
+                    }
+
+                    override fun visitClassOrObject(classOrObject: KtClassOrObject) {
+                        val classDescriptor = bindingContext.get(BindingContext.CLASS, classOrObject)
+                                              ?: error("No descriptor found for class ${classOrObject.fqName}")
+                        val destFile = File(destDir, getClassFilePath(ClassId(packageFqName, classDescriptor.name)))
+                        PackageSerializer(listOf(classDescriptor), emptyList(), packageFqName, destFile).run()
+                    }
+                })
+            }
+            // TODO (!): store list of all such files somewhere
+            val destFile = File(destDir, getPackageFilePath(packageFqName, file.name))
+            PackageSerializer(emptyList(), members, packageFqName, destFile).run()
         }
     }
 
-    private inner class PackageSerializer(
+    private fun getPackageFilePath(packageFqName: FqName, fileName: String): String =
-            private val packageView: PackageViewDescriptor,
+            packageFqName.asString().replace('.', '/') + "/" +
-            private val destDir: File,
+            PackagePartClassUtils.getPartClassName(fileName.substringBeforeLast(".kt")) + METADATA_FILE_EXTENSION
-            private val onFileWrite: (bytesWritten: Int) -> Unit
+
+    private fun getClassFilePath(classId: ClassId): String =
+            classId.asSingleFqName().asString().replace('.', '/') + METADATA_FILE_EXTENSION
+
+    protected inner class PackageSerializer(
+            private val classes: Collection<DeclarationDescriptor>,
+            private val members: Collection<DeclarationDescriptor>,
+            packageFqName: FqName,
+            private val destFile: File
     ) {
-        private val fqName = packageView.fqName
-        private val fragments = packageView.fragments
         private val proto = BuiltInsProtoBuf.BuiltIns.newBuilder()
-        private val extension = BuiltInsSerializerExtension(fragments)
+        private val extension = BuiltInsSerializerExtension(packageFqName)
 
         fun run() {
-            serializeClasses(packageView.memberScope)
+            serializeClasses(classes)
-            serializePackageFragments(fragments)
+            serializeMembers(members)
             serializeStringTable()
             serializeBuiltInsFile()
         }
@@ -101,19 +133,19 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
             val classProto = DescriptorSerializer.createTopLevel(extension).classProto(classDescriptor).build()
             proto.addClass_(classProto)
 
-            serializeClasses(classDescriptor.unsubstitutedInnerClassesScope)
+            serializeClasses(classDescriptor.unsubstitutedInnerClassesScope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))
         }
 
-        private fun serializeClasses(scope: MemberScope) {
+        private fun serializeClasses(classes: Collection<DeclarationDescriptor>) {
-            for (descriptor in DescriptorSerializer.sort(scope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))) {
+            for (descriptor in DescriptorSerializer.sort(classes)) {
                 if (descriptor is ClassDescriptor && descriptor.kind != ClassKind.ENUM_ENTRY) {
                     serializeClass(descriptor)
                 }
             }
         }
 
-        private fun serializePackageFragments(fragments: List<PackageFragmentDescriptor>) {
+        private fun serializeMembers(members: Collection<DeclarationDescriptor>) {
-            proto.`package` = DescriptorSerializer.createTopLevel(extension).packageProto(fragments).build()
+            proto.`package` = DescriptorSerializer.createTopLevel(extension).packagePartProto(members).build()
         }
 
         private fun serializeStringTable() {
@@ -130,14 +162,15 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
                 version.forEach { writeInt(it) }
             }
             proto.build().writeTo(stream)
-            write(getPackageFilePath(fqName), stream)
+            write(stream)
         }
 
-        private fun write(fileName: String, stream: ByteArrayOutputStream) {
+        private fun write(stream: ByteArrayOutputStream) {
-            onFileWrite(stream.size())
+            totalSize += stream.size()
-            val file = File(destDir, fileName)
+            totalFiles++
-            file.parentFile.mkdirs()
+            assert(!destFile.isDirectory) { "Cannot write because output destination is a directory: $destFile" }
-            file.writeBytes(stream.toByteArray())
+            destFile.parentFile.mkdirs()
+            destFile.writeBytes(stream.toByteArray())
         }
     }
 

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt

@@ -26,7 +26,11 @@ import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.addKotlinSourceRoots
-import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.resolve.BindingContext
+import org.jetbrains.kotlin.resolve.DescriptorUtils
+import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
 import org.jetbrains.kotlin.serialization.MetadataSerializer
 import java.io.File
 
@@ -74,5 +78,21 @@ class BuiltInsSerializer(dependOnOldBuiltIns: Boolean) : MetadataSerializer(depe
         }
     }
 
-    override fun getPackageFilePath(fqName: FqName): String = BuiltInSerializerProtocol.getBuiltInsFilePath(fqName)
+    override fun performSerialization(files: Collection<KtFile>, bindingContext: BindingContext, module: ModuleDescriptor, destDir: File) {
+        destDir.deleteRecursively()
+        if (!destDir.mkdirs()) {
+            throw AssertionError("Could not make directories: " + destDir)
+        }
+
+        files.map { it.packageFqName }.toSet().forEach {
+            fqName ->
+            val packageView = module.getPackage(fqName)
+            PackageSerializer(
+                    packageView.memberScope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS),
+                    packageView.fragments.flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) },
+                    packageView.fqName,
+                    File(destDir, BuiltInSerializerProtocol.getBuiltInsFilePath(packageView.fqName))
+            ).run()
+        }
+    }
 }

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializerExtension.kt

@@ -17,18 +17,16 @@
 package org.jetbrains.kotlin.serialization.builtins
 
 import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase
 import org.jetbrains.kotlin.serialization.ProtoBuf
 
 class BuiltInsSerializerExtension(
-        private val packageFragments: Collection<PackageFragmentDescriptor>
+        private val packageFqName: FqName
 ) : KotlinSerializerExtensionBase(BuiltInSerializerProtocol) {
     override fun shouldUseTypeTable(): Boolean = true
 
     override fun serializePackage(proto: ProtoBuf.Package.Builder) {
-        if (packageFragments.isEmpty()) return
+        proto.setExtension(BuiltInsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(packageFqName))
-
-        proto.setExtension(BuiltInsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(packageFragments.first().fqName))
     }
 }

compiler/serialization/src/org/jetbrains/kotlin/serialization/DescriptorSerializer.kt

@@ -495,17 +495,6 @@ class DescriptorSerializer private constructor(
         return builder
     }
 
-    @JvmOverloads
-    fun packageProto(
-            fragments: Collection<PackageFragmentDescriptor>,
-            skip: (DeclarationDescriptor) -> Boolean = { false }
-    ): ProtoBuf.Package.Builder {
-        val members = fragments
-                .flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) }
-                .filter { !skip(it) }
-        return packagePartProto(members)
-    }
-
     fun packagePartProto(members: Collection<DeclarationDescriptor>): ProtoBuf.Package.Builder {
         val builder = ProtoBuf.Package.newBuilder()
 

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt

@@ -124,6 +124,7 @@ object KotlinJavascriptSerializationUtil {
     fun serializePackage(module: ModuleDescriptor, fqName: FqName, writeFun: (String, ByteArray) -> Unit) {
         val packageView = module.getPackage(fqName)
 
+        // TODO: ModuleDescriptor should be able to return the package only with the contents of that module, without dependencies
         val skip: (DeclarationDescriptor) -> Boolean = { DescriptorUtils.getContainingModule(it) != module }
 
         val serializerExtension = KotlinJavascriptSerializerExtension()
@@ -142,7 +143,10 @@ object KotlinJavascriptSerializationUtil {
 
         val packageStream = ByteArrayOutputStream()
         val fragments = packageView.fragments
-        val packageProto = serializer.packageProto(fragments, skip).build() ?: error("Package fragments not serialized: $fragments")
+        val members = fragments
+                .flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) }
+                .filterNot(skip)
+        val packageProto = serializer.packagePartProto(members).build() ?: error("Package fragments not serialized: $fragments")
         if (packageProto.functionCount > 0 || packageProto.propertyCount > 0 || packageProto.typeAliasCount > 0) {
             packageProto.writeTo(packageStream)
             writeFun(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), packageStream.toByteArray())