JS: eliminate direct usage of builtins serialization for js

This commit is contained in:
Michael Nedzelsky
2015-04-30 04:13:43 +03:00
parent 6dc7c4bc29
commit c420e0f010
6 changed files with 440 additions and 45 deletions
@@ -0,0 +1,74 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.serialization.js
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
import org.jetbrains.kotlin.resolve.constants.CompileTimeConstant
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.types.JetType
class KotlinJavascriptAnnotationAndConstantLoader(
module: ModuleDescriptor
) : AnnotationAndConstantLoader<AnnotationDescriptor, CompileTimeConstant<*>> {
private val deserializer = AnnotationDeserializer(module)
override fun loadClassAnnotations(
classProto: ProtoBuf.Class,
nameResolver: NameResolver
): List<AnnotationDescriptor> {
val annotations = classProto.getExtension(JsProtoBuf.classAnnotation).orEmpty()
return annotations.map { proto -> deserializer.deserializeAnnotation(proto, nameResolver) }
}
override fun loadCallableAnnotations(
container: ProtoContainer,
proto: ProtoBuf.Callable,
nameResolver: NameResolver,
kind: AnnotatedCallableKind
): List<AnnotationDescriptor> {
val annotations = proto.getExtension(JsProtoBuf.callableAnnotation).orEmpty()
return annotations.map { proto -> deserializer.deserializeAnnotation(proto, nameResolver) }
}
override fun loadValueParameterAnnotations(
container: ProtoContainer,
callable: ProtoBuf.Callable,
nameResolver: NameResolver,
kind: AnnotatedCallableKind,
proto: ProtoBuf.Callable.ValueParameter
): List<AnnotationDescriptor> {
val annotations = proto.getExtension(JsProtoBuf.parameterAnnotation).orEmpty()
return annotations.map { proto -> deserializer.deserializeAnnotation(proto, nameResolver) }
}
override fun loadTypeAnnotations(proto: ProtoBuf.Type, nameResolver: NameResolver): List<AnnotationDescriptor> {
// TODO: support type annotations for js descriptors
return listOf()
}
override fun loadPropertyConstant(
container: ProtoContainer,
proto: ProtoBuf.Callable,
nameResolver: NameResolver,
expectedType: JetType
): CompileTimeConstant<*>? {
val value = proto.getExtension(JsProtoBuf.compileTimeValue)
return deserializer.resolveValue(expectedType, value, nameResolver)
}
}
@@ -0,0 +1,39 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.serialization.js
import org.jetbrains.kotlin.serialization.deserialization.DeserializedPackageFragment
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.storage.StorageManager
import java.io.InputStream
public class KotlinJavascriptPackageFragment(
fqName: FqName,
storageManager: StorageManager,
module: ModuleDescriptor,
loadResource: (path: String) -> InputStream?
) : DeserializedPackageFragment(fqName, storageManager, module, KotlinJavascriptSerializedResourcePaths, loadResource) {
protected override fun loadClassNames(fqName: FqName, packageProto: ProtoBuf.Package): Collection<Name> {
val classesStream = loadResourceSure(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName))
val classesProto = JsProtoBuf.Classes.parseFrom(classesStream, serializedResourcePaths.EXTENSION_REGISTRY)
return classesProto.getClassNameList()?.map { id -> nameResolver.getName(id) } ?: listOf()
}
}
@@ -17,12 +17,7 @@
package org.jetbrains.kotlin.serialization.js
import com.google.protobuf.ByteString
import org.jetbrains.kotlin.builtins.BuiltInsSerializationUtil
import org.jetbrains.kotlin.builtins.createBuiltInPackageFragmentProvider
import org.jetbrains.kotlin.descriptors.ClassDescriptor
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.DescriptorUtils
import org.jetbrains.kotlin.resolve.descriptorUtil.classId
@@ -30,7 +25,7 @@ import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
import org.jetbrains.kotlin.serialization.DescriptorSerializer
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.SerializationUtil
import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension
import org.jetbrains.kotlin.serialization.StringTable
import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeCapabilitiesDeserializer
import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
@@ -41,28 +36,51 @@ import java.util.zip.GZIPOutputStream
import kotlin.platform.platformStatic
public object KotlinJavascriptSerializationUtil {
private val PACKAGE_FILE_EXT = ".kotlin_package"
public val CLASS_METADATA_FILE_EXTENSION: String = "kjsm"
private val PACKAGE_DEFAULT_BYTES = run {
val stream = ByteArrayOutputStream()
ProtoBuf.Package.getDefaultInstance().writeTo(stream)
stream.toByteArray()
}
private val CLASSES_IN_PACKAGE_DEFAULT_BYTES = run {
val stream = ByteArrayOutputStream()
JsProtoBuf.Classes.getDefaultInstance().writeTo(stream)
stream.toByteArray()
}
private val STRING_TABLE_DEFAULT_BYTES = run {
val nameStream = ByteArrayOutputStream()
val strings = StringTable(KotlinJavascriptSerializerExtension)
SerializationUtil.serializeStringTable(nameStream, strings.serializeSimpleNames(), strings.serializeQualifiedNames())
nameStream.toByteArray()
}
platformStatic
public fun createPackageFragmentProvider(moduleDescriptor: ModuleDescriptor, metadata: ByteArray, storageManager: StorageManager): PackageFragmentProvider? {
val gzipInputStream = GZIPInputStream(ByteArrayInputStream(metadata))
val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
gzipInputStream.close()
val contentMap: MutableMap<String, ByteArray> = hashMapOf()
for (index in content.getEntryCount().indices) {
val entry = content.getEntry(index)
contentMap[entry.getPath()] = entry.getContent().toByteArray()
}
val contentMap = metadata.toContentMap()
val packageFqNames = getPackages(contentMap).map { FqName(it) }.toSet()
if (packageFqNames.isEmpty()) return null
return createBuiltInPackageFragmentProvider(
return createKotlinJavascriptPackageFragmentProvider(
storageManager, moduleDescriptor, packageFqNames, FlexibleTypeCapabilitiesDeserializer.Dynamic
) {
path ->
if (!contentMap.containsKey(path)) null else ByteArrayInputStream(contentMap.get(path))
if (!contentMap.containsKey(path)) {
when {
KotlinJavascriptSerializedResourcePaths.isPackageMetadataFile(path) ->
ByteArrayInputStream(PACKAGE_DEFAULT_BYTES)
KotlinJavascriptSerializedResourcePaths.isStringTableFile(path) ->
ByteArrayInputStream(STRING_TABLE_DEFAULT_BYTES)
KotlinJavascriptSerializedResourcePaths.isClassesInPackageFile(path) ->
ByteArrayInputStream(CLASSES_IN_PACKAGE_DEFAULT_BYTES)
else ->
null
}
}
else ByteArrayInputStream(contentMap.get(path))
}
}
@@ -81,27 +99,15 @@ public object KotlinJavascriptSerializationUtil {
return byteStream.toByteArray()
}
public fun metadataAsString(moduleName: String, moduleDescriptor: ModuleDescriptor): String {
val contentMap = hashMapOf<String, ByteArray>()
DescriptorUtils.getPackagesFqNames(moduleDescriptor).forEach {
fqName ->
serializePackage(moduleDescriptor, fqName) {
fileName, stream ->
contentMap[fileName] = stream.toByteArray()
}
}
val content = KotlinJavascriptSerializationUtil.contentMapToByteArray(contentMap)
return KotlinJavascriptMetadataUtils.formatMetadataAsString(moduleName, content)
}
public fun metadataAsString(moduleName: String, moduleDescriptor: ModuleDescriptor): String =
KotlinJavascriptMetadataUtils.formatMetadataAsString(moduleName, moduleDescriptor.toBinaryMetadata())
fun serializePackage(module: ModuleDescriptor, fqName: FqName, writeFun: (String, ByteArrayOutputStream) -> Unit) {
val packageView = module.getPackage(fqName) ?: error("No package resolved in $module")
val skip: (DeclarationDescriptor) -> Boolean = { DescriptorUtils.getContainingModule(it) != module }
val serializer = DescriptorSerializer.createTopLevel(BuiltInsSerializerExtension)
val serializer = DescriptorSerializer.createTopLevel(KotlinJavascriptSerializerExtension)
val classifierDescriptors = DescriptorSerializer.sort(packageView.getMemberScope().getDescriptors(DescriptorKindFilter.CLASSIFIERS))
@@ -116,23 +122,95 @@ public object KotlinJavascriptSerializationUtil {
val packageStream = ByteArrayOutputStream()
val fragments = module.getPackageFragmentProvider().getPackageFragments(fqName)
val packageProto = serializer.packageProto(fragments, skip).build() ?: error("Package fragments not serialized: $fragments")
packageProto.writeTo(packageStream)
writeFun(BuiltInsSerializationUtil.getPackageFilePath(fqName), packageStream)
if (packageProto.getMemberCount() > 0) {
packageProto.writeTo(packageStream)
writeFun(KotlinJavascriptSerializedResourcePaths.getPackageFilePath(fqName), packageStream)
}
val nameStream = ByteArrayOutputStream()
val strings = serializer.getStringTable()
SerializationUtil.serializeStringTable(nameStream, strings.serializeSimpleNames(), strings.serializeQualifiedNames())
writeFun(BuiltInsSerializationUtil.getStringTableFilePath(fqName), nameStream)
serializeClassNamesInPackage(fqName, fragments, strings, skip, writeFun)
val simpleNames = strings.serializeSimpleNames()
val qualifiedNames = strings.serializeQualifiedNames()
if (simpleNames.getStringCount() > 0 || qualifiedNames.getQualifiedNameCount() > 0) {
SerializationUtil.serializeStringTable(nameStream, simpleNames, qualifiedNames)
writeFun(KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(fqName), nameStream)
}
}
fun getFileName(classDescriptor: ClassDescriptor): String {
return BuiltInsSerializationUtil.getClassMetadataPath(classDescriptor.classId)
private fun serializeClassNamesInPackage(
fqName: FqName,
packageFragments: Collection<PackageFragmentDescriptor>,
stringTable: StringTable,
skip: (DeclarationDescriptor) -> Boolean,
writeFun: (String, ByteArrayOutputStream) -> Unit
) {
val classes = packageFragments.flatMap {
it.getMemberScope().getDescriptors(DescriptorKindFilter.CLASSIFIERS).filterIsInstance<ClassDescriptor>()
}.filter { !skip(it) }
val builder = JsProtoBuf.Classes.newBuilder()
for (descriptor in DescriptorSerializer.sort(classes)) {
builder.addClassName(stringTable.getSimpleNameIndex(descriptor.getName()))
}
val classesProto = builder.build()
if (classesProto.getClassNameCount() > 0) {
val stream = ByteArrayOutputStream()
classesProto.writeTo(stream)
writeFun(KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(fqName), stream)
}
}
private fun getPackageName(filePath: String): String {
return filePath.substringBeforeLast('/').replace('/', '.')
private fun getFileName(classDescriptor: ClassDescriptor): String {
return KotlinJavascriptSerializedResourcePaths.getClassMetadataPath(classDescriptor.classId)
}
private fun getPackages(contentMap: Map<String, ByteArray>): List<String> =
contentMap.keySet().filter { it.endsWith(PACKAGE_FILE_EXT) }.map { getPackageName(it) }
}
private fun ByteArray.toContentMap(): Map<String, ByteArray> {
val gzipInputStream = GZIPInputStream(ByteArrayInputStream(this))
val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
gzipInputStream.close()
val contentMap: MutableMap<String, ByteArray> = hashMapOf()
content.getEntryList().forEach { entry -> contentMap[entry.getPath()] = entry.getContent().toByteArray() }
return contentMap
}
private fun ModuleDescriptor.toContentMap(): Map<String, ByteArray> {
val contentMap = hashMapOf<String, ByteArray>()
DescriptorUtils.getPackagesFqNames(this).forEach {
serializePackage(this, it) {
fileName, stream -> contentMap[fileName] = stream.toByteArray()
}
}
return contentMap
}
private fun getPackages(contentMap: Map<String, ByteArray>): Set<String> {
val keys = contentMap.keySet().map { (if (it.startsWith('/')) it else "/" + it).substringBeforeLast('/') }.toSet()
val result = hashSetOf<String>()
fun addNames(name: String) {
result.add(name)
if (name != "") {
addNames(name.substringBeforeLast('/'))
}
}
keys.forEach { addNames(it) }
return result.map { it.substringAfter('/').replace('/', '.') }.toSet()
}
private fun ModuleDescriptor.toBinaryMetadata(): ByteArray =
KotlinJavascriptSerializationUtil.contentMapToByteArray(this.toContentMap())
}
@@ -0,0 +1,86 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.serialization.js
import com.google.protobuf.ExtensionRegistryLite
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.serialization.SerializedResourcePaths
public object KotlinJavascriptSerializedResourcePaths : SerializedResourcePaths() {
public override val EXTENSION_REGISTRY: ExtensionRegistryLite
init {
EXTENSION_REGISTRY = ExtensionRegistryLite.newInstance()
JsProtoBuf.registerAllExtensions(EXTENSION_REGISTRY)
}
private val CLASSES_FILE_EXTENSION = "kotlin_classes"
private val STRING_TABLE_FILE_EXTENSION = "kotlin_string_table"
private val PACKAGE_CLASS_NAME_SUFFIX: String = "Package"
private val DEFAULT_PACKAGE_CLASS_NAME: String = "_Default" + PACKAGE_CLASS_NAME_SUFFIX
public fun getClassesInPackageFilePath(fqName: FqName): String =
fqName.toPath().withSepIfNotEmpty() + shortName(fqName) + "." + CLASSES_FILE_EXTENSION
public override fun getClassMetadataPath(classId: ClassId): String {
return classId.getPackageFqName().toPath().withSepIfNotEmpty() + classId.getRelativeClassName().asString() +
"." + KotlinJavascriptSerializationUtil.CLASS_METADATA_FILE_EXTENSION
}
public override fun getPackageFilePath(fqName: FqName): String =
getPackageClassFqName(fqName).toPath() + "." + KotlinJavascriptSerializationUtil.CLASS_METADATA_FILE_EXTENSION
public override fun getStringTableFilePath(fqName: FqName): String =
fqName.toPath().withSepIfNotEmpty() + shortName(fqName) + "." + STRING_TABLE_FILE_EXTENSION
public fun isPackageMetadataFile(fileName: String): Boolean =
KotlinJavascriptSerializedResourcePaths.getPackageFilePath(getPackageFqName(fileName)) == fileName
public fun isStringTableFile(fileName: String): Boolean =
KotlinJavascriptSerializedResourcePaths.getStringTableFilePath(getPackageFqName(fileName)) == fileName
public fun isClassesInPackageFile(fileName: String): Boolean =
KotlinJavascriptSerializedResourcePaths.getClassesInPackageFilePath(getPackageFqName(fileName)) == fileName
private fun getPackageFqName(fileName: String): FqName = FqName(getPackageName(fileName))
private fun getPackageName(filePath: String): String =
if (filePath.indexOf('/') >= 0) filePath.substringBeforeLast('/').replace('/', '.') else ""
private fun FqName.toPath() = this.asString().replace('.', '/')
private fun String.withSepIfNotEmpty() = if (this.isEmpty()) this else this + "/"
private fun shortName(fqName: FqName): String =
if (fqName.isRoot()) "default-package" else fqName.shortName().asString()
private fun getPackageClassFqName(packageFQN: FqName): FqName {
return packageFQN.child(Name.identifier(getPackageClassName(packageFQN)))
}
private fun getPackageClassName(packageFQN: FqName): String {
return if (packageFQN.isRoot()) DEFAULT_PACKAGE_CLASS_NAME else capitalizeNonEmptyString(packageFQN.shortName().asString()) + PACKAGE_CLASS_NAME_SUFFIX
}
private fun capitalizeNonEmptyString(s: String): String {
return if (Character.isUpperCase(s.charAt(0))) s else Character.toUpperCase(s.charAt(0)) + s.substring(1)
}
}
@@ -0,0 +1,57 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.serialization.js
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.resolve.constants.NullValue
import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
import org.jetbrains.kotlin.serialization.*
public object KotlinJavascriptSerializerExtension : SerializerExtension() {
override fun serializeClass(descriptor: ClassDescriptor, proto: ProtoBuf.Class.Builder, stringTable: StringTable) {
for (annotation in descriptor.getAnnotations()) {
proto.addExtension(JsProtoBuf.classAnnotation, AnnotationSerializer.serializeAnnotation(annotation, stringTable))
}
}
override fun serializeCallable(
callable: CallableMemberDescriptor,
proto: ProtoBuf.Callable.Builder,
stringTable: StringTable
) {
for (annotation in callable.getAnnotations()) {
proto.addExtension(JsProtoBuf.callableAnnotation, AnnotationSerializer.serializeAnnotation(annotation, stringTable))
}
val propertyDescriptor = callable as? PropertyDescriptor ?: return
val compileTimeConstant = propertyDescriptor.getCompileTimeInitializer()
if (compileTimeConstant != null && compileTimeConstant !is NullValue) {
val type = compileTimeConstant.getType(propertyDescriptor.builtIns)
proto.setExtension(JsProtoBuf.compileTimeValue, AnnotationSerializer.valueProto(compileTimeConstant, type, stringTable).build())
}
}
override fun serializeValueParameter(
descriptor: ValueParameterDescriptor,
proto: ProtoBuf.Callable.ValueParameter.Builder,
stringTable: StringTable
) {
for (annotation in descriptor.getAnnotations()) {
proto.addExtension(JsProtoBuf.parameterAnnotation, AnnotationSerializer.serializeAnnotation(annotation, stringTable))
}
}
}
@@ -0,0 +1,61 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.serialization.js
import org.jetbrains.kotlin.serialization.deserialization.ResourceLoadingClassDataFinder
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents
import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeCapabilitiesDeserializer
import org.jetbrains.kotlin.serialization.deserialization.LocalClassResolverImpl
import org.jetbrains.kotlin.storage.StorageManager
import java.io.InputStream
public fun createKotlinJavascriptPackageFragmentProvider(
storageManager: StorageManager,
module: ModuleDescriptor,
packageFqNames: Set<FqName>,
flexibleTypeCapabilitiesDeserializer: FlexibleTypeCapabilitiesDeserializer,
loadResource: (String) -> InputStream?
): PackageFragmentProvider {
val packageFragments = packageFqNames.map { fqName ->
KotlinJavascriptPackageFragment(fqName, storageManager, module, loadResource)
}
val provider = PackageFragmentProviderImpl(packageFragments)
val localClassResolver = LocalClassResolverImpl()
val components = DeserializationComponents(
storageManager,
module,
ResourceLoadingClassDataFinder(provider, KotlinJavascriptSerializedResourcePaths, loadResource),
KotlinJavascriptAnnotationAndConstantLoader(module),
provider,
localClassResolver,
flexibleTypeCapabilitiesDeserializer
)
localClassResolver.setDeserializationComponents(components)
for (packageFragment in packageFragments) {
packageFragment.setDeserializationComponents(components)
}
return provider
}