From 1d71c19b4206cff7f283f52097e81f7e582bbc4d Mon Sep 17 00:00:00 2001 From: Alexander Udalov Date: Fri, 11 Nov 2016 18:56:17 +0300 Subject: [PATCH] Serialize .kotlin_metadata binary files per source file Do not serialize everything in the same package to the same file (as is done for built-ins) because this approach is unfriendly to incremental compilation, which is going to be supported in the future. Instead, similarly to JVM serialize each class to its own file, and each source file with top-level callables/typealiases to its own file. E.g. if a file named test.kt contains a class Foo and some functions/properties, the output will contain two files: TestKt.kotlin_metadata and Foo.kotlin_metadata. Each one of this files contains the serialized BuiltIns message (see builtins.proto) --- .../serialization/MetadataSerializer.kt | 109 ++++++++++++------ .../builtins/BuiltInsSerializer.kt | 24 +++- .../builtins/BuiltInsSerializerExtension.kt | 8 +- .../serialization/DescriptorSerializer.kt | 11 -- .../js/KotlinJavascriptSerializationUtil.kt | 6 +- 5 files changed, 101 insertions(+), 57 deletions(-) diff --git a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt index c443f72f433..3eb9f1c0d7b 100644 --- a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt +++ b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt @@ -18,7 +18,6 @@ package org.jetbrains.kotlin.serialization import org.jetbrains.kotlin.analyzer.AnalysisResult import org.jetbrains.kotlin.analyzer.common.DefaultAnalyzerFacade -import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport @@ -28,12 +27,15 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.descriptors.ClassKind -import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor -import org.jetbrains.kotlin.descriptors.PackageViewDescriptor +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils +import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.psi.* +import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter -import org.jetbrains.kotlin.resolve.scopes.MemberScope import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension import java.io.ByteArrayOutputStream @@ -44,9 +46,6 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { protected var totalSize = 0 protected var totalFiles = 0 - protected open fun getPackageFilePath(fqName: FqName): String = - fqName.asString().replace('.', '/') + "/" + (if (fqName.isRoot) "default-package" else fqName.shortName().asString()) + "." + METADATA_FILE_EXTENSION - fun serialize(environment: KotlinCoreEnvironment) { val configuration = environment.configuration val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) @@ -63,36 +62,69 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns) }) - val moduleDescriptor = analyzer.analysisResult.moduleDescriptor + if (analyzer.hasErrors()) return - destDir.deleteRecursively() + val (bindingContext, moduleDescriptor) = analyzer.analysisResult - if (!destDir.mkdirs()) { - throw AssertionError("Could not make directories: " + destDir) - } + performSerialization(files, bindingContext, moduleDescriptor, destDir) + } - files.map { it.packageFqName }.toSet().forEach { - fqName -> - PackageSerializer(moduleDescriptor.getPackage(fqName), destDir, { bytesWritten -> - totalSize += bytesWritten - totalFiles++ - }).run() + protected open fun performSerialization( + files: Collection, bindingContext: BindingContext, module: ModuleDescriptor, destDir: File + ) { + for (file in files) { + val packageFqName = file.packageFqName + val members = arrayListOf() + for (declaration in file.declarations) { + declaration.accept(object : KtVisitorVoid() { + override fun visitNamedFunction(function: KtNamedFunction) { + members.add(bindingContext.get(BindingContext.FUNCTION, function) + ?: error("No descriptor found for function ${function.fqName}")) + } + + override fun visitProperty(property: KtProperty) { + members.add(bindingContext.get(BindingContext.VARIABLE, property) + ?: error("No descriptor found for property ${property.fqName}")) + } + + override fun visitTypeAlias(typeAlias: KtTypeAlias) { + members.add(bindingContext.get(BindingContext.TYPE_ALIAS, typeAlias) + ?: error("No descriptor found for type alias ${typeAlias.fqName}")) + } + + override fun visitClassOrObject(classOrObject: KtClassOrObject) { + val classDescriptor = bindingContext.get(BindingContext.CLASS, classOrObject) + ?: error("No descriptor found for class ${classOrObject.fqName}") + val destFile = File(destDir, getClassFilePath(ClassId(packageFqName, classDescriptor.name))) + PackageSerializer(listOf(classDescriptor), emptyList(), packageFqName, destFile).run() + } + }) + } + // TODO (!): store list of all such files somewhere + val destFile = File(destDir, getPackageFilePath(packageFqName, file.name)) + PackageSerializer(emptyList(), members, packageFqName, destFile).run() } } - private inner class PackageSerializer( - private val packageView: PackageViewDescriptor, - private val destDir: File, - private val onFileWrite: (bytesWritten: Int) -> Unit + private fun getPackageFilePath(packageFqName: FqName, fileName: String): String = + packageFqName.asString().replace('.', '/') + "/" + + PackagePartClassUtils.getPartClassName(fileName.substringBeforeLast(".kt")) + METADATA_FILE_EXTENSION + + private fun getClassFilePath(classId: ClassId): String = + classId.asSingleFqName().asString().replace('.', '/') + METADATA_FILE_EXTENSION + + protected inner class PackageSerializer( + private val classes: Collection, + private val members: Collection, + packageFqName: FqName, + private val destFile: File ) { - private val fqName = packageView.fqName - private val fragments = packageView.fragments private val proto = BuiltInsProtoBuf.BuiltIns.newBuilder() - private val extension = BuiltInsSerializerExtension(fragments) + private val extension = BuiltInsSerializerExtension(packageFqName) fun run() { - serializeClasses(packageView.memberScope) - serializePackageFragments(fragments) + serializeClasses(classes) + serializeMembers(members) serializeStringTable() serializeBuiltInsFile() } @@ -101,19 +133,19 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { val classProto = DescriptorSerializer.createTopLevel(extension).classProto(classDescriptor).build() proto.addClass_(classProto) - serializeClasses(classDescriptor.unsubstitutedInnerClassesScope) + serializeClasses(classDescriptor.unsubstitutedInnerClassesScope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS)) } - private fun serializeClasses(scope: MemberScope) { - for (descriptor in DescriptorSerializer.sort(scope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))) { + private fun serializeClasses(classes: Collection) { + for (descriptor in DescriptorSerializer.sort(classes)) { if (descriptor is ClassDescriptor && descriptor.kind != ClassKind.ENUM_ENTRY) { serializeClass(descriptor) } } } - private fun serializePackageFragments(fragments: List) { - proto.`package` = DescriptorSerializer.createTopLevel(extension).packageProto(fragments).build() + private fun serializeMembers(members: Collection) { + proto.`package` = DescriptorSerializer.createTopLevel(extension).packagePartProto(members).build() } private fun serializeStringTable() { @@ -130,14 +162,15 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { version.forEach { writeInt(it) } } proto.build().writeTo(stream) - write(getPackageFilePath(fqName), stream) + write(stream) } - private fun write(fileName: String, stream: ByteArrayOutputStream) { - onFileWrite(stream.size()) - val file = File(destDir, fileName) - file.parentFile.mkdirs() - file.writeBytes(stream.toByteArray()) + private fun write(stream: ByteArrayOutputStream) { + totalSize += stream.size() + totalFiles++ + assert(!destFile.isDirectory) { "Cannot write because output destination is a directory: $destFile" } + destFile.parentFile.mkdirs() + destFile.writeBytes(stream.toByteArray()) } } diff --git a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt index aa3dce53162..d16a7065f5f 100644 --- a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt +++ b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt @@ -26,7 +26,11 @@ import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.addKotlinSourceRoots -import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.resolve.BindingContext +import org.jetbrains.kotlin.resolve.DescriptorUtils +import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter import org.jetbrains.kotlin.serialization.MetadataSerializer import java.io.File @@ -74,5 +78,21 @@ class BuiltInsSerializer(dependOnOldBuiltIns: Boolean) : MetadataSerializer(depe } } - override fun getPackageFilePath(fqName: FqName): String = BuiltInSerializerProtocol.getBuiltInsFilePath(fqName) + override fun performSerialization(files: Collection, bindingContext: BindingContext, module: ModuleDescriptor, destDir: File) { + destDir.deleteRecursively() + if (!destDir.mkdirs()) { + throw AssertionError("Could not make directories: " + destDir) + } + + files.map { it.packageFqName }.toSet().forEach { + fqName -> + val packageView = module.getPackage(fqName) + PackageSerializer( + packageView.memberScope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS), + packageView.fragments.flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) }, + packageView.fqName, + File(destDir, BuiltInSerializerProtocol.getBuiltInsFilePath(packageView.fqName)) + ).run() + } + } } diff --git a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializerExtension.kt b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializerExtension.kt index 398c20625d8..68a3522ec97 100644 --- a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializerExtension.kt +++ b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializerExtension.kt @@ -17,18 +17,16 @@ package org.jetbrains.kotlin.serialization.builtins import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol -import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor +import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase import org.jetbrains.kotlin.serialization.ProtoBuf class BuiltInsSerializerExtension( - private val packageFragments: Collection + private val packageFqName: FqName ) : KotlinSerializerExtensionBase(BuiltInSerializerProtocol) { override fun shouldUseTypeTable(): Boolean = true override fun serializePackage(proto: ProtoBuf.Package.Builder) { - if (packageFragments.isEmpty()) return - - proto.setExtension(BuiltInsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(packageFragments.first().fqName)) + proto.setExtension(BuiltInsProtoBuf.packageFqName, stringTable.getPackageFqNameIndex(packageFqName)) } } diff --git a/compiler/serialization/src/org/jetbrains/kotlin/serialization/DescriptorSerializer.kt b/compiler/serialization/src/org/jetbrains/kotlin/serialization/DescriptorSerializer.kt index 1cda3afe502..d16a37245b5 100644 --- a/compiler/serialization/src/org/jetbrains/kotlin/serialization/DescriptorSerializer.kt +++ b/compiler/serialization/src/org/jetbrains/kotlin/serialization/DescriptorSerializer.kt @@ -495,17 +495,6 @@ class DescriptorSerializer private constructor( return builder } - @JvmOverloads - fun packageProto( - fragments: Collection, - skip: (DeclarationDescriptor) -> Boolean = { false } - ): ProtoBuf.Package.Builder { - val members = fragments - .flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) } - .filter { !skip(it) } - return packagePartProto(members) - } - fun packagePartProto(members: Collection): ProtoBuf.Package.Builder { val builder = ProtoBuf.Package.newBuilder() diff --git a/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt b/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt index a7d3918fd2d..9699ba8e7bc 100644 --- a/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt +++ b/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt @@ -124,6 +124,7 @@ object KotlinJavascriptSerializationUtil { fun serializePackage(module: ModuleDescriptor, fqName: FqName, writeFun: (String, ByteArray) -> Unit) { val packageView = module.getPackage(fqName) + // TODO: ModuleDescriptor should be able to return the package only with the contents of that module, without dependencies val skip: (DeclarationDescriptor) -> Boolean = { DescriptorUtils.getContainingModule(it) != module } val serializerExtension = KotlinJavascriptSerializerExtension() @@ -142,7 +143,10 @@ object KotlinJavascriptSerializationUtil { val packageStream = ByteArrayOutputStream() val fragments = packageView.fragments - val packageProto = serializer.packageProto(fragments, skip).build() ?: error("Package fragments not serialized: $fragments") + val members = fragments + .flatMap { fragment -> DescriptorUtils.getAllDescriptors(fragment.getMemberScope()) } + .filterNot(skip) + val packageProto = serializer.packagePartProto(members).build() ?: error("Package fragments not serialized: $fragments") if (packageProto.functionCount > 0 || packageProto.propertyCount > 0 || packageProto.typeAliasCount > 0) { packageProto.writeTo(packageStream) writeFun(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), packageStream.toByteArray())