[JS IR BE] Extract metadata deserializer to keep binary compatibility

* add UniqId extension to metadata protobuf
This commit is contained in:
Roman Artemev
2019-01-25 19:56:20 +03:00
committed by romanart
parent 3718edb6bc
commit 5384f23f30
16 changed files with 5158 additions and 3 deletions
@@ -6,21 +6,40 @@
package org.jetbrains.kotlin.ir.backend.js
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.vfs.VfsUtilCore
import org.jetbrains.kotlin.backend.common.CompilerPhaseManager
import org.jetbrains.kotlin.backend.common.output.OutputFile
import org.jetbrains.kotlin.backend.common.output.OutputFileCollection
import org.jetbrains.kotlin.backend.common.output.SimpleOutputBinaryFile
import org.jetbrains.kotlin.backend.common.output.SimpleOutputFileCollection
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata.*
import org.jetbrains.kotlin.ir.backend.js.lower.coroutines.CoroutineIntrinsicLowering
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.util.SymbolTable
import org.jetbrains.kotlin.ir.util.deepCopyWithSymbols
import org.jetbrains.kotlin.js.analyze.TopDownAnalyzerFacadeForJS
import org.jetbrains.kotlin.js.resolve.JsPlatform
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.utils.JsMetadataVersion
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
import java.io.File
import org.jetbrains.kotlin.utils.DFS
enum class ModuleType {
@@ -40,6 +59,13 @@ class CompiledModule(
get() = moduleFragment!!.descriptor as ModuleDescriptorImpl
}
fun OutputFileCollection.writeAll(outputDir: File) {
for (file in asList()) {
val output = File(outputDir, file.relativePath)
FileUtil.writeToFile(output, file.asByteArray())
}
}
fun compile(
project: Project,
files: List<KtFile>,
@@ -91,6 +117,75 @@ fun compile(
// val serializedData = serializer.serializeModule(analysisResult.moduleDescriptor, serializedIr)
// buildLibrary(serializedData)
//
// val declarationTable = DeclarationTable(moduleFragment.irBuiltins, DescriptorTable())
// val serializedIr = IrModuleSerializer(context, declarationTable/*, onlyForInlines = false*/).serializedIrModule(moduleFragment)
val serializer = JsKlibMetadataSerializationUtil
// val serializedData = serializer.serializeModule(analysisResult.moduleDescriptor, serializedIr)
val moduleName = configuration.get(CommonConfigurationKeys.MODULE_NAME) as String
val metadataVersion = configuration.get(CommonConfigurationKeys.METADATA_VERSION) as? JsKlibMetadataVersion
?: JsKlibMetadataVersion.INSTANCE
val moduleDescription =
JsKlibMetadataModuleDescriptor(moduleName, dependencies.map { it.name.asString() }, moduleFragment.descriptor)
var index = 0L
val serializedData = serializer.serializeMetadata(
psi2IrContext.bindingContext,
moduleDescription,
configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)!!,
metadataVersion
) {
// val index = declarationTable.descriptorTable.get(it)
// index?.let { newDescriptorUniqId(it) }
JsKlibMetadataProtoBuf.DescriptorUniqId.newBuilder().setIndex(index++).build()
}
// buildLibrary(serializedData)
//
val stdKlibDir = File("js/js.translator/testData/out/klibs/runtime/").also {
it.deleteRecursively()
it.mkdirs()
}
//
// val moduleFile = File(stdKlibDir, "module.kji")
// moduleFile.writeBytes(serializedIr.module)
//
// for ((id, data) in serializedIr.declarations) {
// val file = File(stdKlibDir, "${id.index}${if (id.isLocal) "L" else "G"}.kjd")
// file.writeBytes(data)
// }
//
//
// val debugFile = File(stdKlibDir, "debug.txt")
//
// for ((id, data) in serializedIr.debugIndex) {
// debugFile.appendText(id.toString())
// debugFile.appendText(" --- ")
// debugFile.appendText(data)
// debugFile.appendText("\n")
// }
val metadata = File(stdKlibDir, "${moduleDescription.name}${JsKlibMetadataSerializationUtil.CLASS_METADATA_FILE_EXTENSION}").also {
it.writeBytes(serializedData.asByteArray())
}
val storageManager = LockBasedStorageManager("JsConfig")
// // CREATE NEW MODULE DESCRIPTOR HERE AND DESERIALIZE IT
val lookupTracker = configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER, LookupTracker.DO_NOTHING)
val parts = serializer.readModuleAsProto(metadata.readBytes())
val md = ModuleDescriptorImpl(
Name.special("<$moduleName>"), storageManager, JsPlatform.builtIns
)
val provider = createJsKlibMetadataPackageFragmentProvider(
storageManager, md, parts.header, parts.body, metadataVersion,
CompilerDeserializationConfiguration(configuration.languageVersionSettings),
lookupTracker
)
md.initialize(provider)
md.setDependencies(listOf(md, md.builtIns.builtInsModule))
TODO("Implemenet IrSerialization")
} else {
@@ -0,0 +1,31 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeDeserializer
import org.jetbrains.kotlin.types.ErrorUtils
import org.jetbrains.kotlin.types.KotlinType
import org.jetbrains.kotlin.types.SimpleType
import org.jetbrains.kotlin.types.checker.StrictEqualityTypeChecker
import org.jetbrains.kotlin.types.createDynamicType
import org.jetbrains.kotlin.types.typeUtil.builtIns
object DynamicTypeDeserializer : FlexibleTypeDeserializer {
const val id = "kotlin.DynamicType"
override fun create(proto: ProtoBuf.Type, flexibleId: String, lowerBound: SimpleType, upperBound: SimpleType): KotlinType {
if (flexibleId != id) return ErrorUtils.createErrorType("Unexpected id: $flexibleId. ($lowerBound..$upperBound)")
return if (StrictEqualityTypeChecker.strictEqualTypes(lowerBound, lowerBound.builtIns.nothingType) &&
StrictEqualityTypeChecker.strictEqualTypes(upperBound, upperBound.builtIns.nullableAnyType)
) {
createDynamicType(lowerBound.builtIns)
} else {
ErrorUtils.createErrorType("Illegal type range for dynamic type: $lowerBound..$upperBound")
}
}
}
@@ -0,0 +1,17 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.psi.KtFile
sealed class JsKlibFileMetadata
data class KotlinPsiFileMetadata(val ktFile: KtFile) : JsKlibFileMetadata()
data class KotlinDeserializedFileMetadata(
val packageFragment: JsKlibMetadataPackageFragment,
val fileId: Int
) : JsKlibFileMetadata()
@@ -0,0 +1,16 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
class JsKlibMetadataFileRegistry {
private val fileIdsImpl = mutableMapOf<JsKlibFileMetadata, Int>()
fun lookup(file: JsKlibFileMetadata) = fileIdsImpl.getOrPut(file) { fileIdsImpl.size }
val fileIds: Map<JsKlibFileMetadata, Int>
get() = fileIdsImpl
}
@@ -0,0 +1,86 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.contracts.ContractDeserializerImpl
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.deserialization.PlatformDependentDeclarationFilter
import org.jetbrains.kotlin.descriptors.impl.EmptyPackageFragmentDescriptor
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.parentOrNull
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.storage.StorageManager
fun createJsKlibMetadataPackageFragmentProvider(
storageManager: StorageManager,
module: ModuleDescriptor,
header: JsKlibMetadataProtoBuf.Header,
packageFragmentProtos: List<ProtoBuf.PackageFragment>,
metadataVersion: JsKlibMetadataVersion,
configuration: DeserializationConfiguration,
lookupTracker: LookupTracker
): PackageFragmentProvider {
val packageFragments: MutableList<PackageFragmentDescriptor> = packageFragmentProtos.mapNotNullTo(mutableListOf()) { proto ->
proto.fqName?.let { fqName ->
JsKlibMetadataPackageFragment(fqName, storageManager, module, proto, header, metadataVersion, configuration)
}
}
// Generate empty PackageFragmentDescriptor instances for packages that aren't mentioned in compilation units directly.
// For example, if there's `package foo.bar` directive, we'll get only PackageFragmentDescriptor for `foo.bar`, but
// none for `foo`. Various descriptor/scope code relies on presence of such package fragments, and currently we
// don't know if it's possible to fix this.
// TODO: think about fixing issues in descriptors/scopes
val packageFqNames = packageFragmentProtos.mapNotNullTo(mutableSetOf()) { it.fqName }
for (packageFqName in packageFqNames.mapNotNull { it.parentOrNull() }) {
var ancestorFqName = packageFqName
while (!ancestorFqName.isRoot && packageFqNames.add(ancestorFqName)) {
packageFragments += EmptyPackageFragmentDescriptor(module, ancestorFqName)
ancestorFqName = ancestorFqName.parent()
}
}
val provider = PackageFragmentProviderImpl(packageFragments)
val notFoundClasses = NotFoundClasses(storageManager, module)
val components = DeserializationComponents(
storageManager,
module,
configuration,
DeserializedClassDataFinder(provider),
AnnotationAndConstantLoaderImpl(module, notFoundClasses, JsKlibMetadataSerializerProtocol),
provider,
LocalClassifierTypeSettings.Default,
ErrorReporter.DO_NOTHING,
lookupTracker,
DynamicTypeDeserializer,
emptyList(),
notFoundClasses,
ContractDeserializerImpl(configuration, storageManager),
platformDependentDeclarationFilter = PlatformDependentDeclarationFilter.NoPlatformDependent,
extensionRegistryLite = JsKlibMetadataSerializerProtocol.extensionRegistry
)
for (packageFragment in packageFragments.filterIsInstance<JsKlibMetadataPackageFragment>()) {
packageFragment.initialize(components)
}
return provider
}
private val ProtoBuf.PackageFragment.fqName: FqName?
get() {
val nameResolver = NameResolverImpl(strings, qualifiedNames)
return when {
hasPackage() -> FqName(nameResolver.getPackageFqName(`package`.getExtension(JsKlibMetadataProtoBuf.packageFqName)))
class_Count > 0 -> nameResolver.getClassId(class_OrBuilderList.first().fqName).packageFqName
else -> null
}
}
@@ -0,0 +1,12 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
class JsKlibMetadataModuleDescriptor<out T>(
val name: String,
val imported: List<String>,
val data: T
)
@@ -0,0 +1,75 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.SourceFile
import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.storage.getValue
class JsKlibMetadataPackageFragment(
fqName: FqName,
storageManager: StorageManager,
module: ModuleDescriptor,
proto: ProtoBuf.PackageFragment,
header: JsKlibMetadataProtoBuf.Header,
metadataVersion: JsKlibMetadataVersion,
configuration: DeserializationConfiguration
) : DeserializedPackageFragmentImpl(
fqName, storageManager, module, proto, metadataVersion, JsContainerSource(fqName, header, configuration)
) {
val fileMap: Map<Int, FileHolder> =
proto.getExtension(JsKlibMetadataProtoBuf.packageFragmentFiles).fileList.withIndex().associate { (index, file) ->
(if (file.hasId()) file.id else index) to FileHolder(file.annotationList)
}
private lateinit var annotationDeserializer: AnnotationDeserializer
override fun initialize(components: DeserializationComponents) {
super.initialize(components)
this.annotationDeserializer = AnnotationDeserializer(components.moduleDescriptor, components.notFoundClasses)
}
inner class FileHolder(private val annotationsProto: List<ProtoBuf.Annotation>) {
val annotations: List<AnnotationDescriptor> by storageManager.createLazyValue {
annotationsProto.map { annotationDeserializer.deserializeAnnotation(it, nameResolver) }
}
}
class JsContainerSource(
private val fqName: FqName,
header: JsKlibMetadataProtoBuf.Header,
configuration: DeserializationConfiguration
) : DeserializedContainerSource {
val annotations: List<ClassId> =
if (header.annotationCount == 0) emptyList()
else NameResolverImpl(header.strings, header.qualifiedNames).let { nameResolver ->
// TODO: read arguments of module annotations
header.annotationList.map { annotation -> nameResolver.getClassId(annotation.id) }
}
// TODO
override fun getContainingFile(): SourceFile = SourceFile.NO_SOURCE_FILE
// This is null because we look for incompatible libraries in dependencies in the beginning of the compilation anyway,
// and refuse to compile against them completely
override val incompatibility: IncompatibleVersionErrorData<*>?
get() = null
override val isPreReleaseInvisible: Boolean =
configuration.reportErrorsOnPreReleaseDependencies && (header.flags and 1) != 0
override val presentableString: String
get() = "Package '$fqName'"
}
}
@@ -0,0 +1,251 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.config.AnalysisFlags
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.incremental.components.NoLookupLocation
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.checkers.ExpectedActualDeclarationChecker
import org.jetbrains.kotlin.resolve.descriptorUtil.filterOutSourceAnnotations
import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
import org.jetbrains.kotlin.resolve.descriptorUtil.module
import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
import org.jetbrains.kotlin.resolve.scopes.MemberScope
import org.jetbrains.kotlin.serialization.AnnotationSerializer
import org.jetbrains.kotlin.serialization.DescriptorSerializer
import org.jetbrains.kotlin.serialization.StringTableImpl
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPropertyDescriptor
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
internal object JsKlibMetadataSerializationUtil {
const val CLASS_METADATA_FILE_EXTENSION: String = "klm"
fun serializeMetadata(
bindingContext: BindingContext,
jsDescriptor: JsKlibMetadataModuleDescriptor<ModuleDescriptor>,
languageVersionSettings: LanguageVersionSettings,
metadataVersion: JsKlibMetadataVersion,
declarationTableHandler: ((DeclarationDescriptor) -> JsKlibMetadataProtoBuf.DescriptorUniqId?)
): SerializedMetadata {
val serializedFragments = HashMap<FqName, ProtoBuf.PackageFragment>()
val module = jsDescriptor.data
for (fqName in getPackagesFqNames(module).sortedBy { it.asString() }) {
val fragment = serializeDescriptors(
bindingContext, module,
module.getPackage(fqName).memberScope.getContributedDescriptors(),
fqName, languageVersionSettings, metadataVersion, declarationTableHandler
)
if (!fragment.isEmpty()) {
serializedFragments[fqName] = fragment
}
}
return SerializedMetadata(serializedFragments, jsDescriptor, languageVersionSettings)
}
class SerializedMetadata(
private val serializedFragments: Map<FqName, ProtoBuf.PackageFragment>,
private val jsDescriptor: JsKlibMetadataModuleDescriptor<ModuleDescriptor>,
private val languageVersionSettings: LanguageVersionSettings
) {
fun asByteArray(): ByteArray =
ByteArrayOutputStream().apply {
GZIPOutputStream(this).use { stream ->
serializeHeader(
jsDescriptor.data,
packageFqName = null,
languageVersionSettings = languageVersionSettings
).writeDelimitedTo(stream)
asLibrary().writeTo(stream)
}
}.toByteArray()
private fun asLibrary(): JsKlibMetadataProtoBuf.Library {
jsDescriptor.imported
val builder = JsKlibMetadataProtoBuf.Library.newBuilder()
jsDescriptor.imported.forEach { builder.addImportedModule(it) }
for ((_, fragment) in serializedFragments.entries.sortedBy { (fqName, _) -> fqName.asString() }) {
builder.addPackageFragment(fragment)
}
return builder.build()
}
}
fun serializeDescriptors(
bindingContext: BindingContext,
module: ModuleDescriptor,
scope: Collection<DeclarationDescriptor>,
fqName: FqName,
languageVersionSettings: LanguageVersionSettings,
metadataVersion: BinaryVersion,
declarationTableHandler: ((DeclarationDescriptor) -> JsKlibMetadataProtoBuf.DescriptorUniqId?)
): ProtoBuf.PackageFragment {
val builder = ProtoBuf.PackageFragment.newBuilder()
val skip = fun(descriptor: DeclarationDescriptor): Boolean {
// TODO: ModuleDescriptor should be able to return the package only with the contents of that module, without dependencies
if (descriptor.module != module) return true
if (descriptor is MemberDescriptor && descriptor.isExpect) {
return !(descriptor is ClassDescriptor && ExpectedActualDeclarationChecker.shouldGenerateExpectClass(descriptor))
}
return false
}
val fileRegistry = JsKlibMetadataFileRegistry()
val extension = JsKlibMetadataSerializerExtension(fileRegistry, languageVersionSettings, metadataVersion, declarationTableHandler)
val classDescriptors = scope.filterIsInstance<ClassDescriptor>().sortedBy { it.fqNameSafe.asString() }
fun serializeClasses(descriptors: Collection<DeclarationDescriptor>, parentSerializer: DescriptorSerializer) {
for (descriptor in descriptors) {
if (descriptor !is ClassDescriptor || skip(descriptor)) continue
val serializer = DescriptorSerializer.create(descriptor, extension, parentSerializer)
serializeClasses(descriptor.unsubstitutedInnerClassesScope.getContributedDescriptors(), serializer)
val classProto = serializer.classProto(descriptor).build() ?: error("Class not serialized: $descriptor")
builder.addClass_(classProto)
}
}
val serializer = DescriptorSerializer.createTopLevel(extension)
serializeClasses(classDescriptors, serializer)
val stringTable = extension.stringTable
val members = scope.filterNot(skip)
builder.`package` = serializer.packagePartProto(fqName, members).build()
builder.setExtension(
JsKlibMetadataProtoBuf.packageFragmentFiles,
serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(stringTable))
)
val (strings, qualifiedNames) = stringTable.buildProto()
builder.strings = strings
builder.qualifiedNames = qualifiedNames
return builder.build()
}
private fun serializeFiles(
fileRegistry: JsKlibMetadataFileRegistry,
bindingContext: BindingContext,
serializer: AnnotationSerializer
): JsKlibMetadataProtoBuf.Files {
val filesProto = JsKlibMetadataProtoBuf.Files.newBuilder()
for ((file, id) in fileRegistry.fileIds.entries.sortedBy { it.value }) {
val fileProto = JsKlibMetadataProtoBuf.File.newBuilder()
if (id != filesProto.fileCount) {
fileProto.id = id
}
val annotations = when (file) {
is KotlinPsiFileMetadata -> file.ktFile.annotationEntries.map { bindingContext[BindingContext.ANNOTATION, it]!! }
is KotlinDeserializedFileMetadata -> file.packageFragment.fileMap[file.fileId]!!.annotations
}
for (annotation in annotations.filterOutSourceAnnotations()) {
fileProto.addAnnotation(serializer.serializeAnnotation(annotation))
}
filesProto.addFile(fileProto)
}
return filesProto.build()
}
private fun ProtoBuf.PackageFragment.isEmpty(): Boolean =
class_Count == 0 && `package`.let { it.functionCount == 0 && it.propertyCount == 0 && it.typeAliasCount == 0 }
fun serializeHeader(
module: ModuleDescriptor, packageFqName: FqName?, languageVersionSettings: LanguageVersionSettings
): JsKlibMetadataProtoBuf.Header {
val header = JsKlibMetadataProtoBuf.Header.newBuilder()
if (packageFqName != null) {
header.packageFqName = packageFqName.asString()
}
if (languageVersionSettings.isPreRelease()) {
header.flags = 1
}
val experimentalAnnotationFqNames = languageVersionSettings.getFlag(AnalysisFlags.experimental)
if (experimentalAnnotationFqNames.isNotEmpty()) {
val stringTable = StringTableImpl()
for (fqName in experimentalAnnotationFqNames) {
val descriptor = module.resolveClassByFqName(FqName(fqName), NoLookupLocation.FOR_ALREADY_TRACKED) ?: continue
header.addAnnotation(ProtoBuf.Annotation.newBuilder().apply {
id = stringTable.getFqNameIndex(descriptor)
})
}
val (strings, qualifiedNames) = stringTable.buildProto()
header.strings = strings
header.qualifiedNames = qualifiedNames
}
// TODO: write JS code binary version
return header.build()
}
private fun getPackagesFqNames(module: ModuleDescriptor): Set<FqName> {
return mutableSetOf<FqName>().apply {
getSubPackagesFqNames(module.getPackage(FqName.ROOT), this)
add(FqName.ROOT)
}
}
private fun getSubPackagesFqNames(packageView: PackageViewDescriptor, result: MutableSet<FqName>) {
val fqName = packageView.fqName
if (!fqName.isRoot) {
result.add(fqName)
}
for (descriptor in packageView.memberScope.getContributedDescriptors(DescriptorKindFilter.PACKAGES, MemberScope.ALL_NAME_FILTER)) {
if (descriptor is PackageViewDescriptor) {
getSubPackagesFqNames(descriptor, result)
}
}
}
@JvmStatic
fun readModuleAsProto(metadata: ByteArray): JsKlibMetadataParts {
val (header, content) = GZIPInputStream(ByteArrayInputStream(metadata)).use { stream ->
JsKlibMetadataProtoBuf.Header.parseDelimitedFrom(stream, JsKlibMetadataSerializerProtocol.extensionRegistry) to
JsKlibMetadataProtoBuf.Library.parseFrom(stream, JsKlibMetadataSerializerProtocol.extensionRegistry)
}
return JsKlibMetadataParts(header, content.packageFragmentList, content.importedModuleList)
}
}
data class JsKlibMetadataParts(
val header: JsKlibMetadataProtoBuf.Header,
val body: List<ProtoBuf.PackageFragment>,
val importedModules: List<String>
)
internal fun DeclarationDescriptor.extractFileId(): Int? = when (this) {
is DeserializedClassDescriptor -> classProto.getExtension(JsKlibMetadataProtoBuf.classContainingFileId)
is DeserializedSimpleFunctionDescriptor -> proto.getExtension(JsKlibMetadataProtoBuf.functionContainingFileId)
is DeserializedPropertyDescriptor -> proto.getExtension(JsKlibMetadataProtoBuf.propertyContainingFileId)
else -> null
}
@@ -0,0 +1,118 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.config.LanguageFeature
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.serialization.MutableVersionRequirementTable
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.resolve.DescriptorUtils
import org.jetbrains.kotlin.resolve.source.PsiSourceFile
import org.jetbrains.kotlin.serialization.DescriptorSerializer
import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase
import org.jetbrains.kotlin.types.FlexibleType
class JsKlibMetadataSerializerExtension(
private val fileRegistry: JsKlibMetadataFileRegistry,
private val languageVersionSettings: LanguageVersionSettings,
override val metadataVersion: BinaryVersion,
val declarationTableHandler: ((DeclarationDescriptor) -> JsKlibMetadataProtoBuf.DescriptorUniqId?)
) : KotlinSerializerExtensionBase(JsKlibMetadataSerializerProtocol) {
override val stringTable = JsKlibMetadataStringTable()
override fun serializeFlexibleType(flexibleType: FlexibleType, lowerProto: ProtoBuf.Type.Builder, upperProto: ProtoBuf.Type.Builder) {
lowerProto.flexibleTypeCapabilitiesId = stringTable.getStringIndex(DynamicTypeDeserializer.id)
}
private fun uniqId(descriptor: DeclarationDescriptor): JsKlibMetadataProtoBuf.DescriptorUniqId? {
// val index = declarationTable.descriptorTable.get(descriptor)
// return index?.let { newDescriptorUniqId(it) }
return declarationTableHandler(descriptor)
}
override fun serializeTypeParameter(typeParameter: TypeParameterDescriptor, proto: ProtoBuf.TypeParameter.Builder) {
uniqId(typeParameter)?.let { proto.setExtension(JsKlibMetadataProtoBuf.typeParamUniqId, it) }
super.serializeTypeParameter(typeParameter, proto)
}
override fun serializeValueParameter(descriptor: ValueParameterDescriptor, proto: ProtoBuf.ValueParameter.Builder) {
uniqId(descriptor)?.let { proto.setExtension(JsKlibMetadataProtoBuf.valueParamUniqId, it) }
super.serializeValueParameter(descriptor, proto)
}
override fun serializeEnumEntry(descriptor: ClassDescriptor, proto: ProtoBuf.EnumEntry.Builder) {
uniqId(descriptor)?.let { proto.setExtension(JsKlibMetadataProtoBuf.enumEntryUniqId, it) }
super.serializeEnumEntry(descriptor, proto)
}
override fun serializeConstructor(descriptor: ConstructorDescriptor, proto: ProtoBuf.Constructor.Builder,
childSerializer: DescriptorSerializer
) {
uniqId(descriptor)?.let { proto.setExtension(JsKlibMetadataProtoBuf.constructorUniqId, it) }
super.serializeConstructor(descriptor, proto, childSerializer)
}
override fun serializeClass(
descriptor: ClassDescriptor,
proto: ProtoBuf.Class.Builder,
versionRequirementTable: MutableVersionRequirementTable,
childSerializer: DescriptorSerializer
) {
uniqId(descriptor)?.let { proto.setExtension(JsKlibMetadataProtoBuf.classUniqId, it) }
val id = getFileId(descriptor)
if (id != null) {
proto.setExtension(JsKlibMetadataProtoBuf.classContainingFileId, id)
}
super.serializeClass(descriptor, proto, versionRequirementTable, childSerializer)
}
override fun serializeProperty(
descriptor: PropertyDescriptor,
proto: ProtoBuf.Property.Builder,
versionRequirementTable: MutableVersionRequirementTable,
childSerializer: DescriptorSerializer
) {
uniqId(descriptor)?.let { proto.setExtension(JsKlibMetadataProtoBuf.propertyUniqId, it) }
val id = getFileId(descriptor)
if (id != null) {
proto.setExtension(JsKlibMetadataProtoBuf.propertyContainingFileId, id)
}
super.serializeProperty(descriptor, proto, versionRequirementTable, childSerializer)
}
override fun serializeFunction(descriptor: FunctionDescriptor,
proto: ProtoBuf.Function.Builder,
childSerializer: DescriptorSerializer
) {
uniqId(descriptor)?.let { proto.setExtension(JsKlibMetadataProtoBuf.functionUniqId, it) }
val id = getFileId(descriptor)
if (id != null) {
proto.setExtension(JsKlibMetadataProtoBuf.functionContainingFileId, id)
}
super.serializeFunction(descriptor, proto, childSerializer)
}
private fun getFileId(descriptor: DeclarationDescriptor): Int? {
if (!DescriptorUtils.isTopLevelDeclaration(descriptor) || descriptor !is DeclarationDescriptorWithSource) return null
val fileId = descriptor.extractFileId()
if (fileId != null) {
(descriptor.containingDeclaration as? JsKlibMetadataPackageFragment)?.let { packageFragment ->
return fileRegistry.lookup(KotlinDeserializedFileMetadata(packageFragment, fileId))
}
}
val file = descriptor.source.containingFile as? PsiSourceFile ?: return null
val psiFile = file.psiFile
return (psiFile as? KtFile)?.let { fileRegistry.lookup(KotlinPsiFileMetadata(it)) }
}
override fun releaseCoroutines() = languageVersionSettings.supportsFeature(LanguageFeature.ReleaseCoroutines)
}
@@ -0,0 +1,35 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
object JsKlibMetadataSerializerProtocol : SerializerExtensionProtocol(
ExtensionRegistryLite.newInstance().apply(JsKlibMetadataProtoBuf::registerAllExtensions),
JsKlibMetadataProtoBuf.packageFqName,
JsKlibMetadataProtoBuf.constructorAnnotation,
JsKlibMetadataProtoBuf.classAnnotation,
JsKlibMetadataProtoBuf.functionAnnotation,
JsKlibMetadataProtoBuf.propertyAnnotation,
JsKlibMetadataProtoBuf.propertyGetterAnnotation,
JsKlibMetadataProtoBuf.propertySetterAnnotation,
JsKlibMetadataProtoBuf.enumEntryAnnotation,
JsKlibMetadataProtoBuf.compileTimeValue,
JsKlibMetadataProtoBuf.parameterAnnotation,
JsKlibMetadataProtoBuf.typeAnnotation,
JsKlibMetadataProtoBuf.typeParameterAnnotation
) {
fun getKjsmFilePath(packageFqName: FqName): String {
val shortName = if (packageFqName.isRoot) Name.identifier("root-package") else packageFqName.shortName()
return packageFqName.child(shortName).asString().replace('.', '/') +
"." +
JsKlibMetadataSerializationUtil.CLASS_METADATA_FILE_EXTENSION
}
}
@@ -0,0 +1,34 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor
import org.jetbrains.kotlin.descriptors.ClassifierDescriptorWithTypeParameters
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.resolve.DescriptorUtils
import org.jetbrains.kotlin.resolve.descriptorUtil.classId
import org.jetbrains.kotlin.resolve.descriptorUtil.getAllSuperClassifiers
import org.jetbrains.kotlin.serialization.StringTableImpl
class JsKlibMetadataStringTable : StringTableImpl() {
override fun getLocalClassIdReplacement(descriptor: ClassifierDescriptorWithTypeParameters): ClassId? {
return if (descriptor.containingDeclaration is CallableMemberDescriptor) {
val superClassifiers = descriptor.getAllSuperClassifiers()
.mapNotNull { it as ClassifierDescriptorWithTypeParameters }
.filter { it != descriptor }
.toList()
if (superClassifiers.size == 1) {
superClassifiers[0].classId
} else {
val superClass = superClassifiers.find { !DescriptorUtils.isInterface(it) }
superClass?.classId ?: ClassId.topLevel(KotlinBuiltIns.FQ_NAMES.any.toSafe())
}
} else {
super.getLocalClassIdReplacement(descriptor)
}
}
}
@@ -0,0 +1,39 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import java.io.DataInputStream
import java.io.InputStream
class JsKlibMetadataVersion(vararg numbers: Int) : BinaryVersion(*numbers) {
override fun isCompatible(): Boolean =
this.isCompatibleTo(INSTANCE)
fun toInteger() = (patch shl 16) + (minOf(minor, 255) shl 8) + minOf(major, 255)
companion object {
@JvmField
val INSTANCE = JsKlibMetadataVersion(1, 2, 6)
@JvmField
val INVALID_VERSION = JsKlibMetadataVersion()
fun fromInteger(version: Int): JsKlibMetadataVersion =
JsKlibMetadataVersion(version and 255, (version shr 8) and 255, version shr 16)
fun readFrom(stream: InputStream): JsKlibMetadataVersion {
val dataInput = DataInputStream(stream)
val size = dataInput.readInt()
// We assume here that the version will always have 3 components. This is needed to prevent reading an unpredictable amount
// of integers from old .kjsm files (pre-1.1) because they did not have the version in the beginning
if (size != INSTANCE.toArray().size) return INVALID_VERSION
return JsKlibMetadataVersion(*(1..size).map { dataInput.readInt() }.toIntArray())
}
}
}
@@ -0,0 +1,122 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata;
import "core/metadata/src/metadata.proto";
option java_outer_classname = "JsKlibMetadataProtoBuf";
option optimize_for = LITE_RUNTIME;
message Header {
/*
preRelease
*/
optional int32 flags = 1;
// (patch << 16) + (minor << 8) + major
optional int32 js_code_binary_version = 2 [default = 1];
optional string package_fq_name = 3;
optional org.jetbrains.kotlin.metadata.StringTable strings = 4;
optional org.jetbrains.kotlin.metadata.QualifiedNameTable qualified_names = 5;
// Annotations on the whole module
repeated org.jetbrains.kotlin.metadata.Annotation annotation = 6;
}
message File {
// If absent, id is the index of the file in the Files.file list
optional int32 id = 1;
repeated org.jetbrains.kotlin.metadata.Annotation annotation = 2;
}
message Files {
repeated File file = 1;
}
message DescriptorUniqId {
required int64 index = 1;
}
extend org.jetbrains.kotlin.metadata.Package {
optional int32 package_fq_name = 131;
}
extend org.jetbrains.kotlin.metadata.Class {
repeated org.jetbrains.kotlin.metadata.Annotation class_annotation = 130;
optional int32 class_containing_file_id = 135;
optional DescriptorUniqId class_uniq_id = 136;
}
extend org.jetbrains.kotlin.metadata.Constructor {
repeated org.jetbrains.kotlin.metadata.Annotation constructor_annotation = 130;
optional DescriptorUniqId constructor_uniq_id = 131;
}
extend org.jetbrains.kotlin.metadata.Function {
repeated org.jetbrains.kotlin.metadata.Annotation function_annotation = 130;
optional int32 function_containing_file_id = 135;
optional DescriptorUniqId function_uniq_id = 136;
}
extend org.jetbrains.kotlin.metadata.Property {
repeated org.jetbrains.kotlin.metadata.Annotation property_annotation = 130;
repeated org.jetbrains.kotlin.metadata.Annotation property_getter_annotation = 132;
repeated org.jetbrains.kotlin.metadata.Annotation property_setter_annotation = 133;
optional org.jetbrains.kotlin.metadata.Annotation.Argument.Value compile_time_value = 131;
optional int32 property_containing_file_id = 135;
optional DescriptorUniqId property_uniq_id = 136;
}
extend org.jetbrains.kotlin.metadata.EnumEntry {
repeated org.jetbrains.kotlin.metadata.Annotation enum_entry_annotation = 130;
optional DescriptorUniqId enum_entry_uniq_id = 131;
}
extend org.jetbrains.kotlin.metadata.ValueParameter {
repeated org.jetbrains.kotlin.metadata.Annotation parameter_annotation = 130;
optional DescriptorUniqId value_param_uniq_id = 131;
}
extend org.jetbrains.kotlin.metadata.Type {
repeated org.jetbrains.kotlin.metadata.Annotation type_annotation = 130;
}
extend org.jetbrains.kotlin.metadata.TypeParameter {
repeated org.jetbrains.kotlin.metadata.Annotation type_parameter_annotation = 130;
optional DescriptorUniqId type_param_uniq_id = 131;
}
extend org.jetbrains.kotlin.metadata.PackageFragment {
optional Files package_fragment_files = 130;
}
message Classes {
// id in StringTable
repeated int32 class_name = 1 [packed = true];
}
message Library {
repeated org.jetbrains.kotlin.metadata.PackageFragment package_fragment = 2;
repeated string imported_module = 3;
}
@@ -57,7 +57,8 @@ val PROTO_PATHS: List<ProtoPath> = listOf(
ProtoPath("konan/library-reader/src/konan.proto"),
ProtoPath("core/metadata.jvm/src/jvm_metadata.proto"),
ProtoPath("core/metadata.jvm/src/jvm_module.proto"),
ProtoPath("build-common/src/java_descriptors.proto")
ProtoPath("build-common/src/java_descriptors.proto"),
ProtoPath("compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/metadata/js.proto")
)
private val EXT_OPTIONS_PROTO_PATH = ProtoPath("core/metadata/src/ext_options.proto")
@@ -13,7 +13,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.ir.backend.js.compile
import org.jetbrains.kotlin.js.test.runtimeSources
import org.jetbrains.kotlin.js.test.JsIrTestRuntime
import org.jetbrains.kotlin.psi.KtFile
import java.io.File
@@ -53,7 +53,7 @@ fun main() {
val result = compile(
environment.project,
runtimeSources.map(::createPsiFile),
JsIrTestRuntime.FULL.sources.map(::createPsiFile),
buildConfiguration(environment),
null,
true,