From 2e933a165a3036d55cc17af7efbb156bf46b1eb6 Mon Sep 17 00:00:00 2001 From: Denis Zharkov Date: Wed, 17 Jan 2018 12:14:28 +0300 Subject: [PATCH] Avoid retaining all protobuf objects in DeserializedMemberScope Object model for deserialized protobuf isn't very cheap, so loading just our stdlib's packages several times (as we do in the IDE) might lead to ~ 96 MB of retained memory just by these objects. While in fact many of them remain unused #KT-21517 Fixed --- .../JvmBuiltInsPackageFragmentProvider.kt | 3 +- .../DeserializationComponentsForJava.kt | 4 +- .../kotlin/builtins/BuiltInsLoaderImpl.kt | 3 +- .../MetadataPackageFragmentProvider.kt | 4 +- .../serialization/deserialization/context.kt | 4 +- .../descriptors/DeserializedMemberScope.kt | 104 +++++++++++------- .../DeserializerForClassfileDecompiler.kt | 3 +- ...KotlinMetadataDeserializerForDecompiler.kt | 3 +- ...kotlinJavascriptPackageFragmentProvider.kt | 3 +- 9 files changed, 82 insertions(+), 49 deletions(-) diff --git a/core/descriptors.jvm/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt b/core/descriptors.jvm/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt index 92d58b3bce3..fb53ad13e2c 100644 --- a/core/descriptors.jvm/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt +++ b/core/descriptors.jvm/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt @@ -53,7 +53,8 @@ class JvmBuiltInsPackageFragmentProvider( ), notFoundClasses, ContractDeserializer.DEFAULT, - additionalClassPartsProvider, platformDependentDeclarationFilter + additionalClassPartsProvider, platformDependentDeclarationFilter, + BuiltInSerializerProtocol.extensionRegistry ) } diff --git a/core/descriptors.jvm/src/org/jetbrains/kotlin/load/kotlin/DeserializationComponentsForJava.kt b/core/descriptors.jvm/src/org/jetbrains/kotlin/load/kotlin/DeserializationComponentsForJava.kt index a45419bfed6..810882aecc8 100644 --- a/core/descriptors.jvm/src/org/jetbrains/kotlin/load/kotlin/DeserializationComponentsForJava.kt +++ b/core/descriptors.jvm/src/org/jetbrains/kotlin/load/kotlin/DeserializationComponentsForJava.kt @@ -24,6 +24,7 @@ import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.java.lazy.LazyJavaPackageFragmentProvider import org.jetbrains.kotlin.platform.JvmBuiltIns import org.jetbrains.kotlin.serialization.deserialization.* +import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil import org.jetbrains.kotlin.storage.StorageManager // This class is needed only for easier injection: exact types of needed components are specified in the constructor here. @@ -50,7 +51,8 @@ class DeserializationComponentsForJava( LocalClassifierTypeSettings.Default, errorReporter, lookupTracker, JavaFlexibleTypeDeserializer, emptyList(), notFoundClasses, contractDeserializer, additionalClassPartsProvider = jvmBuiltIns?.settings ?: AdditionalClassPartsProvider.None, - platformDependentDeclarationFilter = jvmBuiltIns?.settings ?: PlatformDependentDeclarationFilter.NoPlatformDependent + platformDependentDeclarationFilter = jvmBuiltIns?.settings ?: PlatformDependentDeclarationFilter.NoPlatformDependent, + extensionRegistryLite = JvmProtoBufUtil.EXTENSION_REGISTRY ) } } diff --git a/core/deserialization/src/org/jetbrains/kotlin/builtins/BuiltInsLoaderImpl.kt b/core/deserialization/src/org/jetbrains/kotlin/builtins/BuiltInsLoaderImpl.kt index 71a35620f56..ba88e630319 100644 --- a/core/deserialization/src/org/jetbrains/kotlin/builtins/BuiltInsLoaderImpl.kt +++ b/core/deserialization/src/org/jetbrains/kotlin/builtins/BuiltInsLoaderImpl.kt @@ -84,7 +84,8 @@ class BuiltInsLoaderImpl : BuiltInsLoader { notFoundClasses, ContractDeserializer.DEFAULT, additionalClassPartsProvider, - platformDependentDeclarationFilter + platformDependentDeclarationFilter, + BuiltInSerializerProtocol.extensionRegistry ) for (packageFragment in packageFragments) { diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt index 58d8a80172d..cb753b3c515 100644 --- a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt @@ -24,7 +24,6 @@ import org.jetbrains.kotlin.descriptors.PackagePartProvider import org.jetbrains.kotlin.descriptors.SourceElement import org.jetbrains.kotlin.descriptors.deserialization.AdditionalClassPartsProvider import org.jetbrains.kotlin.descriptors.deserialization.PlatformDependentDeclarationFilter -import org.jetbrains.kotlin.incremental.components.LookupLocation import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName @@ -60,7 +59,8 @@ class MetadataPackageFragmentProvider( emptyList(), notFoundClasses, ContractDeserializer.DEFAULT, - AdditionalClassPartsProvider.None, PlatformDependentDeclarationFilter.All + AdditionalClassPartsProvider.None, PlatformDependentDeclarationFilter.All, + BuiltInSerializerProtocol.extensionRegistry ) } diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/context.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/context.kt index f4440b4128c..60956251140 100644 --- a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/context.kt +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/context.kt @@ -24,6 +24,7 @@ import org.jetbrains.kotlin.descriptors.deserialization.ClassDescriptorFactory import org.jetbrains.kotlin.descriptors.deserialization.PlatformDependentDeclarationFilter import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.resolve.constants.ConstantValue import org.jetbrains.kotlin.serialization.ProtoBuf import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource @@ -45,7 +46,8 @@ class DeserializationComponents( val notFoundClasses: NotFoundClasses, val contractDeserializer: ContractDeserializer, val additionalClassPartsProvider: AdditionalClassPartsProvider = AdditionalClassPartsProvider.None, - val platformDependentDeclarationFilter: PlatformDependentDeclarationFilter = PlatformDependentDeclarationFilter.All + val platformDependentDeclarationFilter: PlatformDependentDeclarationFilter = PlatformDependentDeclarationFilter.All, + val extensionRegistryLite: ExtensionRegistryLite ) { val classDeserializer: ClassDeserializer = ClassDeserializer(this) diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/descriptors/DeserializedMemberScope.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/descriptors/DeserializedMemberScope.kt index 7895eafa18a..81ac2083344 100644 --- a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/descriptors/DeserializedMemberScope.kt +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/descriptors/DeserializedMemberScope.kt @@ -20,7 +20,9 @@ import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.incremental.components.LookupLocation import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.protobuf.AbstractMessageLite import org.jetbrains.kotlin.protobuf.MessageLite +import org.jetbrains.kotlin.protobuf.Parser import org.jetbrains.kotlin.resolve.MemberComparator import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter import org.jetbrains.kotlin.resolve.scopes.MemberScopeImpl @@ -30,6 +32,8 @@ import org.jetbrains.kotlin.storage.getValue import org.jetbrains.kotlin.utils.Printer import org.jetbrains.kotlin.utils.addIfNotNull import org.jetbrains.kotlin.utils.compact +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream import java.util.* abstract class DeserializedMemberScope protected constructor( @@ -40,20 +44,22 @@ abstract class DeserializedMemberScope protected constructor( classNames: () -> Collection ) : MemberScopeImpl() { - private val functionProtos by - c.storageManager.createLazyValue { - functionList.groupByName { it.name } - } - private val propertyProtos by - c.storageManager.createLazyValue { - propertyList.groupByName { it.name } - } - private val typeAliasProtos by - c.storageManager.createLazyValue { - if (c.components.configuration.typeAliasesAllowed) - typeAliasList.groupByName { it.name } - else emptyMap() - } + private val functionProtosBytes = functionList.groupByName { it.name }.packToByteArray() + + private val propertyProtosBytes = propertyList.groupByName { it.name }.packToByteArray() + + private val typeAliasBytes = + if (c.components.configuration.typeAliasesAllowed) + typeAliasList.groupByName { it.name }.packToByteArray() + else + emptyMap() + + private fun Map>.packToByteArray(): Map = + mapValues { entry -> + val byteArrayOutputStream = ByteArrayOutputStream() + entry.value.map { proto -> proto.writeDelimitedTo(byteArrayOutputStream) } + byteArrayOutputStream.toByteArray() + } private val functions = c.storageManager.createMemoizedFunction> { computeFunctions(it) } @@ -63,14 +69,14 @@ abstract class DeserializedMemberScope protected constructor( c.storageManager.createMemoizedFunctionWithNullableValues { createTypeAlias(it) } private val functionNamesLazy by c.storageManager.createLazyValue { - functionProtos.keys + getNonDeclaredFunctionNames() + functionProtosBytes.keys + getNonDeclaredFunctionNames() } private val variableNamesLazy by c.storageManager.createLazyValue { - propertyProtos.keys + getNonDeclaredVariableNames() + propertyProtosBytes.keys + getNonDeclaredVariableNames() } - private val typeAliasNames: Set get() = typeAliasProtos.keys + private val typeAliasNames: Set get() = typeAliasBytes.keys internal val classNames by c.storageManager.createLazyValue { classNames().toSet() } @@ -87,21 +93,37 @@ abstract class DeserializedMemberScope protected constructor( ) = groupBy { c.nameResolver.getName(getNameIndex(it)) } private fun computeFunctions(name: Name) = - computeDescriptors( - name, - functionProtos, - { c.memberDeserializer.loadFunction(it) }, - { computeNonDeclaredFunctions(name, it)} - ) + computeDescriptors( + name, + functionProtosBytes, + ProtoBuf.Function.PARSER, + { c.memberDeserializer.loadFunction(it) }, + { computeNonDeclaredFunctions(name, it) } + ) - inline private fun computeDescriptors( - name: Name, - protosByName: Map>, - factory: (M) -> D, - computeNonDeclared: (MutableCollection) -> Unit + private inline fun computeDescriptors( + name: Name, + bytesByName: Map, + parser: Parser, + factory: (M) -> D, + computeNonDeclared: (MutableCollection) -> Unit + ): Collection = + computeDescriptors( + bytesByName[name]?.let { + val inputStream = ByteArrayInputStream(it) + generateSequence { + parser.parseDelimitedFrom(inputStream, c.components.extensionRegistryLite) + }.toList() + } ?: emptyList(), + factory, + computeNonDeclared + ) + + private inline fun computeDescriptors( + protos: Collection, + factory: (M) -> D, + computeNonDeclared: (MutableCollection) -> Unit ): Collection { - val protos = protosByName[name].orEmpty() - val descriptors = protos.mapTo(arrayListOf(), factory) computeNonDeclared(descriptors) @@ -117,20 +139,22 @@ abstract class DeserializedMemberScope protected constructor( } private fun computeProperties(name: Name) = - computeDescriptors( - name, - propertyProtos, - { c.memberDeserializer.loadProperty(it) }, - { computeNonDeclaredProperties(name, it) } - ) + computeDescriptors( + name, + propertyProtosBytes, + ProtoBuf.Property.PARSER, + { c.memberDeserializer.loadProperty(it) }, + { computeNonDeclaredProperties(name, it) } + ) protected open fun computeNonDeclaredProperties(name: Name, descriptors: MutableCollection) { } - private fun createTypeAlias(name: Name) = - typeAliasProtos[name]?.singleOrNull()?.let { - c.memberDeserializer.loadTypeAlias(it) - } + private fun createTypeAlias(name: Name): TypeAliasDescriptor? { + val byteArray = typeAliasBytes[name] ?: return null + val proto = ProtoBuf.TypeAlias.parseDelimitedFrom(ByteArrayInputStream(byteArray)) ?: return null + return c.memberDeserializer.loadTypeAlias(proto) + } override fun getContributedVariables(name: Name, location: LookupLocation): Collection { if (name !in getVariableNames()) return emptyList() diff --git a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt index c0aee144d17..a5953894001 100644 --- a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt +++ b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt @@ -71,7 +71,8 @@ class DeserializerForClassfileDecompiler( storageManager, moduleDescriptor, DeserializationConfiguration.Default, classDataFinder, annotationAndConstantLoader, packageFragmentProvider, ResolveEverythingToKotlinAnyLocalClassifierResolver(builtIns), LoggingErrorReporter(LOG), LookupTracker.DO_NOTHING, JavaFlexibleTypeDeserializer, emptyList(), notFoundClasses, - ContractDeserializer.DEFAULT + ContractDeserializer.DEFAULT, + extensionRegistryLite = JvmProtoBufUtil.EXTENSION_REGISTRY ) } diff --git a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/common/KotlinMetadataDeserializerForDecompiler.kt b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/common/KotlinMetadataDeserializerForDecompiler.kt index 4bbbe35094a..b78a29e3ac4 100644 --- a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/common/KotlinMetadataDeserializerForDecompiler.kt +++ b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/common/KotlinMetadataDeserializerForDecompiler.kt @@ -52,7 +52,8 @@ class KotlinMetadataDeserializerForDecompiler( AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, serializerProtocol), packageFragmentProvider, ResolveEverythingToKotlinAnyLocalClassifierResolver(builtIns), LoggingErrorReporter(LOG), LookupTracker.DO_NOTHING, flexibleTypeDeserializer, emptyList(), notFoundClasses, - ContractDeserializer.DEFAULT + ContractDeserializer.DEFAULT, + extensionRegistryLite = serializerProtocol.extensionRegistry ) } diff --git a/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/kotlinJavascriptPackageFragmentProvider.kt b/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/kotlinJavascriptPackageFragmentProvider.kt index ca484411922..85514ef33ca 100644 --- a/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/kotlinJavascriptPackageFragmentProvider.kt +++ b/js/js.serializer/src/org/jetbrains/kotlin/serialization/js/kotlinJavascriptPackageFragmentProvider.kt @@ -73,7 +73,8 @@ fun createKotlinJavascriptPackageFragmentProvider( emptyList(), notFoundClasses, ContractDeserializerImpl(configuration), - platformDependentDeclarationFilter = PlatformDependentDeclarationFilter.NoPlatformDependent + platformDependentDeclarationFilter = PlatformDependentDeclarationFilter.NoPlatformDependent, + extensionRegistryLite = JsSerializerProtocol.extensionRegistry ) for (packageFragment in packageFragments.filterIsInstance()) {