Refactor deserialization initialization in built-ins and JS

BuiltinsPackageFragment was only designed to work with a single instance alive. Because of this, it was creating deserialization subsystem upon initialization. This was not working perfectly in JS where it was used, because multiple storage managers, class caches and other components were created for each package, leading to different concurrency errors and performance hits. Also in the near future another package fragment will be needed to represent the built-in package "kotlin.reflect"
2015-04-17 21:19:19 +03:00
parent 19ddb4b510
commit 3d928a0842
10 changed files with 183 additions and 102 deletions
@@ -16,8 +16,8 @@

 package org.jetbrains.kotlin.serialization.builtins

-import org.jetbrains.kotlin.builtins.BuiltinsPackageFragment
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
+import org.jetbrains.kotlin.builtins.createBuiltInPackageFragmentProvider
 import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime
 import org.jetbrains.kotlin.jvm.compiler.LoadDescriptorUtil.TEST_PACKAGE_FQNAME
 import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeCapabilitiesDeserializer
@@ -40,19 +40,21 @@ public class BuiltInsSerializerTest : TestCaseWithTmpdir() {

        val module = JetTestUtils.createEmptyModule("<module>")

-        val packageFragment =
-            BuiltinsPackageFragment(TEST_PACKAGE_FQNAME, LockBasedStorageManager(), module, FlexibleTypeCapabilitiesDeserializer.ThrowException) {
-                val file = File(tmpdir, it)
-                if (file.exists()) FileInputStream(file) else null
-            }
+        val packageFragmentProvider = createBuiltInPackageFragmentProvider(
+                LockBasedStorageManager(), module, setOf(TEST_PACKAGE_FQNAME),
+                FlexibleTypeCapabilitiesDeserializer.ThrowException
+        ) {
+            val file = File(tmpdir, it)
+            if (file.exists()) FileInputStream(file) else null
+        }

-        module.initialize(packageFragment.provider)
+        module.initialize(packageFragmentProvider)
        module.addDependencyOnModule(module)
        module.addDependencyOnModule(KotlinBuiltIns.getInstance().getBuiltInsModule())
        module.seal()

        RecursiveDescriptorComparator.validateAndCompareDescriptorWithFile(
-                module.getPackage(TEST_PACKAGE_FQNAME),
+                module.getPackage(TEST_PACKAGE_FQNAME)!!,
                RecursiveDescriptorComparator.DONT_INCLUDE_METHODS_OF_OBJECT,
                File(source.replace(".kt", ".txt"))
        )
@@ -25,7 +25,6 @@ import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.addKotlinSourceRoots
-import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.js.analyze.TopDownAnalyzerFacadeForJS
 import org.jetbrains.kotlin.js.config.LibrarySourcesConfig
@@ -35,6 +34,7 @@ import org.jetbrains.kotlin.test.JetTestUtils
 import org.jetbrains.kotlin.test.TestCaseWithTmpdir
 import org.jetbrains.kotlin.test.util.RecursiveDescriptorComparator
 import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
+import org.jetbrains.kotlin.utils.sure
 import java.io.File

 public class KotlinJavascriptSerializerTest : TestCaseWithTmpdir() {
@@ -81,7 +81,8 @@ public class KotlinJavascriptSerializerTest : TestCaseWithTmpdir() {
        val metadata = KotlinJavascriptMetadataUtils.loadMetadata(metaFile)
        assert(metadata.size() == 1)

-        val provider = CompositePackageFragmentProvider(KotlinJavascriptSerializationUtil.getPackageFragmentProviders(module, metadata[0].body))
+        val provider = KotlinJavascriptSerializationUtil.createPackageFragmentProvider(module, metadata[0].body)
+                .sure { "No package fragment provider was created" }

        module.initialize(provider)
        module.addDependencyOnModule(module)
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.builtins
+
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.serialization.ClassData
+import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
+import java.io.InputStream
+
+public class BuiltInsClassDataFinder(
+        private val packageFragmentProvider: PackageFragmentProvider,
+        private val loadResource: (path: String) -> InputStream?
+) : ClassDataFinder {
+    override fun findClassData(classId: ClassId): ClassData? {
+        val packageFragment = packageFragmentProvider.getPackageFragments(classId.getPackageFqName()).singleOrNull() ?: return null
+
+        val stream = loadResource(BuiltInsSerializationUtil.getClassMetadataPath(classId)) ?: return null
+
+        val classProto = ProtoBuf.Class.parseFrom(stream, BuiltInsSerializationUtil.EXTENSION_REGISTRY)
+        val nameResolver =
+                (packageFragment as? BuiltinsPackageFragment ?: error("Not a built-in package fragment: $packageFragment")).nameResolver
+
+        val expectedShortName = classId.getShortClassName()
+        val actualShortName = nameResolver.getClassId(classProto.getFqName()).getShortClassName()
+        if (!actualShortName.isSpecial() && actualShortName != expectedShortName) {
+            // Workaround for case-insensitive file systems,
+            // otherwise we'd find "Collection" for "collection" etc
+            return null
+        }
+
+        return ClassData(nameResolver, classProto)
+    }
+}
@@ -16,14 +16,23 @@

 package org.jetbrains.kotlin.builtins

-import org.jetbrains.kotlin.name.*
+import com.google.protobuf.ExtensionRegistryLite
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
 import kotlin.platform.platformStatic

 public object BuiltInsSerializationUtil {
+    public val EXTENSION_REGISTRY: ExtensionRegistryLite
+
+    init {
+        EXTENSION_REGISTRY = ExtensionRegistryLite.newInstance()
+        BuiltInsProtoBuf.registerAllExtensions(EXTENSION_REGISTRY)
+    }
+
    private val CLASS_METADATA_FILE_EXTENSION = "kotlin_class"
    private val PACKAGE_FILE_NAME = ".kotlin_package"
    private val STRING_TABLE_FILE_NAME = ".kotlin_string_table"
-    private val CLASS_NAMES_FILE_NAME = ".kotlin_class_names"

    platformStatic public fun getClassMetadataPath(classId: ClassId): String {
        return packageFqNameToPath(classId.getPackageFqName()) + "/" + classId.getRelativeClassName().asString() +
@@ -36,9 +45,6 @@ public object BuiltInsSerializationUtil {
    platformStatic public fun getStringTableFilePath(fqName: FqName): String =
            packageFqNameToPath(fqName) + "/" + STRING_TABLE_FILE_NAME

-    platformStatic public fun getClassNamesFilePath(fqName: FqName): String =
-            packageFqNameToPath(fqName) + "/" + CLASS_NAMES_FILE_NAME
-
    private fun packageFqNameToPath(fqName: FqName): String =
            fqName.asString().replace('.', '/')
 }
@@ -16,86 +16,45 @@

 package org.jetbrains.kotlin.builtins

-import com.google.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
-import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
 import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
-import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.serialization.ClassData
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
-import org.jetbrains.kotlin.serialization.deserialization.*
+import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents
+import org.jetbrains.kotlin.serialization.deserialization.NameResolver
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
 import org.jetbrains.kotlin.storage.StorageManager
 import java.io.InputStream
+import javax.inject.Inject
+import kotlin.properties.Delegates

 public class BuiltinsPackageFragment(
        fqName: FqName,
        storageManager: StorageManager,
        module: ModuleDescriptor,
-        flexibleTypeCapabilitiesDeserializer: FlexibleTypeCapabilitiesDeserializer,
        private val loadResource: (path: String) -> InputStream?
 ) : PackageFragmentDescriptorImpl(module, fqName) {

-    private val extensionRegistry: ExtensionRegistryLite
+    val nameResolver = NameResolver.read(loadResourceSure(BuiltInsSerializationUtil.getStringTableFilePath(fqName)))

-    init {
-        extensionRegistry = ExtensionRegistryLite.newInstance()
-        BuiltInsProtoBuf.registerAllExtensions(extensionRegistry)
-        extensionRegistry
+    private var components: DeserializationComponents by Delegates.notNull()
+
+    Inject
+    public fun setDeserializationComponents(components: DeserializationComponents) {
+        this.components = components
    }

-    private val nameResolver = NameResolver.read(
-            getStream(BuiltInsSerializationUtil.getStringTableFilePath(fqName))
-    )
-
-    public val provider: PackageFragmentProvider = PackageFragmentProviderImpl(listOf(this))
-
-    private val members: DeserializedPackageMemberScope = run {
-        val proto = loadPackage()
-        val localClassResolver = LocalClassResolverImpl()
-        val components = DeserializationComponents(
-                storageManager, module, BuiltInsClassDataFinder(),
-                BuiltInsAnnotationAndConstantLoader(getContainingDeclaration()),
-                provider, localClassResolver,
-                flexibleTypeCapabilitiesDeserializer
-        )
-        localClassResolver.setDeserializationComponents(components)
-        DeserializedPackageMemberScope(this, proto, nameResolver, components, { readClassNames(proto) })
+    private val memberScope = storageManager.createLazyValue {
+        val stream = loadResourceSure(BuiltInsSerializationUtil.getPackageFilePath(fqName))
+        val proto = ProtoBuf.Package.parseFrom(stream, BuiltInsSerializationUtil.EXTENSION_REGISTRY)
+        DeserializedPackageMemberScope(this, proto, nameResolver, components, classNames = {
+            proto.getExtension(BuiltInsProtoBuf.className)?.map { id -> nameResolver.getName(id) } ?: listOf()
+        })
    }

-    private fun loadPackage(): ProtoBuf.Package {
-        val stream = getStream(BuiltInsSerializationUtil.getPackageFilePath(fqName))
-        return ProtoBuf.Package.parseFrom(stream, extensionRegistry)
-    }
+    override fun getMemberScope() = memberScope()

-    private fun readClassNames(proto: ProtoBuf.Package): List<Name> {
-        return proto.getExtension(BuiltInsProtoBuf.className)?.map { id -> nameResolver.getName(id) } ?: listOf()
-    }
-
-    override fun getMemberScope() = members
-
-    private fun getStream(path: String): InputStream =
+    private fun loadResourceSure(path: String): InputStream =
            loadResource(path) ?: throw IllegalStateException("Resource not found in classpath: $path")
-
-    private inner class BuiltInsClassDataFinder : ClassDataFinder {
-        override fun findClassData(classId: ClassId): ClassData? {
-            val stream = loadResource(BuiltInsSerializationUtil.getClassMetadataPath(classId)) ?: return null
-
-            val classProto = ProtoBuf.Class.parseFrom(stream, extensionRegistry)
-
-            val expectedShortName = classId.getShortClassName()
-            val actualShortName = nameResolver.getClassId(classProto.getFqName()).getShortClassName()
-            if (!actualShortName.isSpecial() && actualShortName != expectedShortName) {
-                // Workaround for case-insensitive file systems,
-                // otherwise we'd find "Collection" for "collection" etc
-                return null
-            }
-
-            return ClassData(nameResolver, classProto)
-        }
-    }
 }
@@ -40,6 +40,7 @@ import org.jetbrains.kotlin.types.checker.JetTypeChecker;
 import java.io.InputStream;
 import java.util.*;

+import static kotlin.KotlinPackage.single;
 import static org.jetbrains.kotlin.builtins.PrimitiveType.*;

 public class KotlinBuiltIns {
@@ -109,22 +110,25 @@ public class KotlinBuiltIns {

    private KotlinBuiltIns() {
        builtInsModule = new ModuleDescriptorImpl(
-                Name.special("<built-ins lazy module>"), Collections.<ImportPath>emptyList(), PlatformToKotlinClassMap.EMPTY
+                Name.special("<built-ins module>"), Collections.<ImportPath>emptyList(), PlatformToKotlinClassMap.EMPTY
        );
-        builtinsPackageFragment = new BuiltinsPackageFragment(
-                BUILT_INS_PACKAGE_FQ_NAME, new LockBasedStorageManager(), builtInsModule,
-                FlexibleTypeCapabilitiesDeserializer.ThrowException.INSTANCE$,
-                new Function1<String, InputStream>() {
+
+        PackageFragmentProvider packageFragmentProvider = BuiltinsPackage.createBuiltInPackageFragmentProvider(
+                new LockBasedStorageManager(), builtInsModule, Collections.singleton(BUILT_INS_PACKAGE_FQ_NAME),
+                FlexibleTypeCapabilitiesDeserializer.ThrowException.INSTANCE$, new Function1<String, InputStream>() {
                    @Override
                    public InputStream invoke(String path) {
                        return KotlinBuiltIns.class.getClassLoader().getResourceAsStream(path);
                    }
                }
        );
-        builtInsModule.initialize(builtinsPackageFragment.getProvider());
+
+        builtInsModule.initialize(packageFragmentProvider);
        builtInsModule.addDependencyOnModule(builtInsModule);
        builtInsModule.seal();

+        builtinsPackageFragment = (BuiltinsPackageFragment) single(packageFragmentProvider.getPackageFragments(BUILT_INS_PACKAGE_FQ_NAME));
+
        primitiveTypeToNullableJetType = new EnumMap<PrimitiveType, JetType>(PrimitiveType.class);
        primitiveTypeToArrayJetType = new EnumMap<PrimitiveType, JetType>(PrimitiveType.class);
        primitiveJetTypeToJetArrayType = new HashMap<JetType, JetType>();
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.builtins
+
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents
+import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeCapabilitiesDeserializer
+import org.jetbrains.kotlin.serialization.deserialization.LocalClassResolverImpl
+import org.jetbrains.kotlin.storage.StorageManager
+import java.io.InputStream
+
+public fun createBuiltInPackageFragmentProvider(
+        storageManager: StorageManager,
+        module: ModuleDescriptor,
+        packageFqNames: Set<FqName>,
+        flexibleTypeCapabilitiesDeserializer: FlexibleTypeCapabilitiesDeserializer,
+        loadResource: (String) -> InputStream?
+): PackageFragmentProvider {
+    val packageFragments = packageFqNames.map { fqName ->
+        BuiltinsPackageFragment(fqName, storageManager, module, loadResource)
+    }
+    val provider = PackageFragmentProviderImpl(packageFragments)
+
+    val localClassResolver = LocalClassResolverImpl()
+
+    val components = DeserializationComponents(
+            storageManager,
+            module,
+            BuiltInsClassDataFinder(provider, loadResource),
+            BuiltInsAnnotationAndConstantLoader(module),
+            provider,
+            localClassResolver,
+            flexibleTypeCapabilitiesDeserializer
+    )
+
+    localClassResolver.setDeserializationComponents(components)
+
+    for (packageFragment in packageFragments) {
+        packageFragment.setDeserializationComponents(components)
+    }
+
+    return provider
+}
@@ -66,7 +66,8 @@ public object JsAnalyzerFacade : AnalyzerFacade<JsResolverForModule, PlatformAna
            if (!JsHeaderLibraryDetectionUtil.isJsHeaderLibraryWithSources(files.toList())) {
                val providers = moduleInfo.library.getFiles(OrderRootType.CLASSES)
                        .flatMap { KotlinJavascriptMetadataUtils.loadMetadata(PathUtil.getLocalPath(it)!!) }
-                        .flatMap { KotlinJavascriptSerializationUtil.getPackageFragmentProviders(moduleDescriptor, it.body) }
+                        .map { KotlinJavascriptSerializationUtil.createPackageFragmentProvider(moduleDescriptor, it.body) }
+                        .filterNotNull()

                if (providers.isNotEmpty()) {
                    packageFragmentProvider = CompositePackageFragmentProvider(listOf(packageFragmentProvider) + providers)
@@ -33,6 +33,7 @@ import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil;
 import org.jetbrains.kotlin.utils.KotlinJavascriptMetadata;

 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;

 /**
@@ -107,7 +108,7 @@ public abstract class Config {
        return moduleId;
    }

-    public abstract  boolean checkLibFilesAndReportErrors(@NotNull Function1<String, Unit> report);
+    public abstract boolean checkLibFilesAndReportErrors(@NotNull Function1<String, Unit> report);

    protected abstract void init(@NotNull List<JetFile> sourceFilesInLibraries, @NotNull List<KotlinJavascriptMetadata> metadata);

@@ -117,10 +118,10 @@ public abstract class Config {
        if (moduleDescriptors != null) return moduleDescriptors;

        moduleDescriptors = new SmartList<ModuleDescriptorImpl>();
-        for(KotlinJavascriptMetadata metadataEntry : metadata) {
+        for (KotlinJavascriptMetadata metadataEntry : metadata) {
            moduleDescriptors.add(createModuleDescriptor(metadataEntry));
        }
-        for(ModuleDescriptorImpl module : moduleDescriptors) {
+        for (ModuleDescriptorImpl module : moduleDescriptors) {
            setDependencies(module, moduleDescriptors);
            module.seal();
        }
@@ -149,18 +150,19 @@ public abstract class Config {
    private static ModuleDescriptorImpl createModuleDescriptor(KotlinJavascriptMetadata metadata) {
        ModuleDescriptorImpl moduleDescriptor = TopDownAnalyzerFacadeForJS.createJsModule("<" + metadata.getModuleName() + ">");

-        List<PackageFragmentProvider> providers = KotlinJavascriptSerializationUtil
-                .getPackageFragmentProviders(moduleDescriptor, metadata.getBody());
-        CompositePackageFragmentProvider compositePackageFragmentProvider = new CompositePackageFragmentProvider(providers);
+        PackageFragmentProvider provider =
+                KotlinJavascriptSerializationUtil.createPackageFragmentProvider(moduleDescriptor, metadata.getBody());

-        moduleDescriptor.initialize(compositePackageFragmentProvider);
+        moduleDescriptor.initialize(
+                provider != null ? provider : new CompositePackageFragmentProvider(Collections.<PackageFragmentProvider>emptyList())
+        );
        moduleDescriptor.addDependencyOnModule(KotlinBuiltIns.getInstance().getBuiltInsModule());

        return moduleDescriptor;
    }

    private static void setDependencies(ModuleDescriptorImpl module, List<ModuleDescriptorImpl> modules) {
-        for(ModuleDescriptorImpl moduleItem : modules) {
+        for (ModuleDescriptorImpl moduleItem : modules) {
            module.addDependencyOnModule(moduleItem);
        }
    }
@@ -18,7 +18,7 @@ package org.jetbrains.kotlin.serialization.js

 import com.google.protobuf.ByteString
 import org.jetbrains.kotlin.builtins.BuiltInsSerializationUtil
-import org.jetbrains.kotlin.builtins.BuiltinsPackageFragment
+import org.jetbrains.kotlin.builtins.createBuiltInPackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
@@ -44,29 +44,26 @@ public object KotlinJavascriptSerializationUtil {
    private val PACKAGE_FILE_SUFFIX = "/.kotlin_package"

    platformStatic
-    public fun getPackageFragmentProviders(moduleDescriptor: ModuleDescriptor, metadata: ByteArray): List<PackageFragmentProvider> {
+    public fun createPackageFragmentProvider(moduleDescriptor: ModuleDescriptor, metadata: ByteArray): PackageFragmentProvider? {
        val gzipInputStream = GZIPInputStream(ByteArrayInputStream(metadata))
        val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
        gzipInputStream.close()

        val contentMap: MutableMap<String, ByteArray> = hashMapOf()
-        for(index in 0..content.getEntryCount()-1) {
+        for (index in content.getEntryCount().indices) {
            val entry = content.getEntry(index)
            contentMap[entry.getPath()] = entry.getContent().toByteArray()
        }

-        val packages = getPackages(contentMap)
+        val packageFqNames = getPackages(contentMap).map { FqName(it) }.toSet()
+        if (packageFqNames.isEmpty()) return null

-        val load = { path: String -> if (!contentMap.containsKey(path)) null else ByteArrayInputStream(contentMap.get(path)) }
-
-        val providers = arrayListOf<PackageFragmentProvider>()
-        for (packageName in packages) {
-            val fqName = FqName(packageName)
-            val packageFragment = BuiltinsPackageFragment(fqName, LockBasedStorageManager(), moduleDescriptor, FlexibleTypeCapabilitiesDeserializer.Dynamic, load)
-            providers.add(packageFragment.provider)
+        return createBuiltInPackageFragmentProvider(
+                LockBasedStorageManager(), moduleDescriptor, packageFqNames, FlexibleTypeCapabilitiesDeserializer.Dynamic
+        ) {
+            path ->
+            if (!contentMap.containsKey(path)) null else ByteArrayInputStream(contentMap.get(path))
        }
-
-        return providers
    }

    public fun contentMapToByteArray(contentMap: Map<String, ByteArray>): ByteArray {