Add BinaryVersion to DeserializationContext

This will be useful to implement version-dependent deserialization,
which is needed for gradual fixes of issues in metadata

 #KT-25120 In Progress
This commit is contained in:
Alexander Udalov
2018-07-11 19:14:21 +02:00
parent 1e9694f1db
commit 4122021090
40 changed files with 201 additions and 93 deletions
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.incremental.js
import org.jetbrains.kotlin.incremental.IncrementalJsCache
import org.jetbrains.kotlin.utils.JsMetadataVersion
import java.io.File
class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) : IncrementalDataProvider {
@@ -14,4 +15,7 @@ class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) :
override val compiledPackageParts: Map<File, TranslationResultValue>
get() = cache.nonDirtyPackageParts()
}
override val metadataVersion: IntArray
get() = JsMetadataVersion.INSTANCE.toArray() // TODO: store and load correct metadata version
}
@@ -135,6 +135,8 @@ open class CompilerCallbackServicesFacadeServer(
override fun incrementalDataProvider_getHeaderMetadata(): ByteArray = incrementalDataProvider!!.headerMetadata
override fun incrementalDataProvider_getMetadataVersion(): IntArray = incrementalDataProvider!!.metadataVersion
override fun incrementalDataProvider_getCompiledPackageParts() =
incrementalDataProvider!!.compiledPackageParts.entries.map {
CompiledPackagePart(it.key.path, it.value.metadata, it.value.binaryAst)
@@ -119,6 +119,9 @@ interface CompilerCallbackServicesFacade : Remote {
@Throws(RemoteException::class)
fun incrementalDataProvider_getCompiledPackageParts(): Collection<CompiledPackagePart>
@Throws(RemoteException::class)
fun incrementalDataProvider_getMetadataVersion(): IntArray
}
class CompiledPackagePart(
@@ -27,4 +27,9 @@ class RemoteIncrementalDataProvider(val facade: CompilerCallbackServicesFacade,
}
result
}
override val metadataVersion: IntArray
get() = rpcProfiler.withMeasure(this) {
facade.incrementalDataProvider_getMetadataVersion()
}
}
@@ -24,6 +24,7 @@ import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder
import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.name.ClassId
@@ -88,12 +89,16 @@ class IncrementalPackageFragmentProvider(
val jvmBinaryClass =
kotlinClassFinder.findKotlinClass(ClassId.topLevel(partName.fqNameForTopLevelClassMaybeWithDollars))
val metadataVersion =
jvmBinaryClass?.classHeader?.metadataVersion
?: JvmMetadataVersion.INSTANCE
DeserializedPackageMemberScope(
this, packageProto, nameResolver,
JvmPackagePartSource(
this, packageProto, nameResolver, metadataVersion,
JvmPackagePartSource(
partName, facadeName, packageProto, nameResolver, knownJvmBinaryClass = jvmBinaryClass
),
deserializationComponents, classNames = { emptyList() }
deserializationComponents, classNames = { emptyList() }
)
}
}
@@ -38,6 +38,7 @@ import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.test.KotlinTestUtils
import org.jetbrains.kotlin.test.TestCaseWithTmpdir
import org.jetbrains.kotlin.test.util.RecursiveDescriptorComparator
import org.jetbrains.kotlin.utils.JsMetadataVersion
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
import org.jetbrains.kotlin.utils.sure
import java.io.File
@@ -92,12 +93,12 @@ class KotlinJavascriptSerializerTest : TestCaseWithTmpdir() {
private fun deserialize(metaFile: File): ModuleDescriptorImpl {
val module = KotlinTestUtils.createEmptyModule("<${KotlinTestUtils.TEST_MODULE_NAME}>", JsPlatform.builtIns)
val metadata = KotlinJavascriptMetadataUtils.loadMetadata(metaFile)
assert(metadata.size == 1)
val metadata = KotlinJavascriptMetadataUtils.loadMetadata(metaFile).single()
val (header, packageFragmentProtos) = readModuleAsProto(metadata.single().body)
val (header, packageFragmentProtos) = readModuleAsProto(metadata.body, metadata.version)
val provider = createKotlinJavascriptPackageFragmentProvider(
LockBasedStorageManager(), module, header, packageFragmentProtos, DeserializationConfiguration.Default, LookupTracker.DO_NOTHING
LockBasedStorageManager(), module, header, packageFragmentProtos, metadata.version,
DeserializationConfiguration.Default, LookupTracker.DO_NOTHING
).sure { "No package fragment provider was created" }
module.initialize(provider)
@@ -51,6 +51,6 @@ class JvmBuiltInsPackageFragmentProvider(
override fun findPackage(fqName: FqName): DeserializedPackageFragment? =
finder.findBuiltInsData(fqName)?.let { inputStream ->
BuiltInsPackageFragmentImpl(fqName, storageManager, moduleDescriptor, inputStream)
BuiltInsPackageFragmentImpl.create(fqName, storageManager, moduleDescriptor, inputStream)
}
}
@@ -53,7 +53,7 @@ class DeserializedDescriptorResolver {
JvmProtoBufUtil.readClassDataFrom(data, strings)
} ?: return null
val source = KotlinJvmBinarySourceElement(kotlinClass, kotlinClass.incompatibility, kotlinClass.isPreReleaseInvisible)
return ClassData(nameResolver, classProto, source)
return ClassData(nameResolver, classProto, kotlinClass.classHeader.metadataVersion, source)
}
fun createKotlinPackagePartScope(descriptor: PackageFragmentDescriptor, kotlinClass: KotlinJvmBinaryClass): MemberScope? {
@@ -65,7 +65,9 @@ class DeserializedDescriptorResolver {
val source = JvmPackagePartSource(
kotlinClass, packageProto, nameResolver, kotlinClass.incompatibility, kotlinClass.isPreReleaseInvisible
)
return DeserializedPackageMemberScope(descriptor, packageProto, nameResolver, source, components) {
return DeserializedPackageMemberScope(
descriptor, packageProto, nameResolver, kotlinClass.classHeader.metadataVersion, source, components
) {
// All classes are included into Java scope
emptyList()
}
@@ -7,10 +7,12 @@ package org.jetbrains.kotlin.serialization.deserialization
import org.jetbrains.kotlin.descriptors.SourceElement
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
data class ClassData(
val nameResolver: NameResolver,
val classProto: ProtoBuf.Class,
val metadataVersion: BinaryVersion,
val sourceElement: SourceElement
)
@@ -39,7 +39,7 @@ class ClassDeserializer(private val components: DeserializationComponents) {
}
if (classId in BLACK_LIST) return null
val (nameResolver, classProto, sourceElement) = key.classData
val (nameResolver, classProto, metadataVersion, sourceElement) = key.classData
?: components.classDataFinder.findClassData(classId)
?: return null
@@ -60,11 +60,12 @@ class ClassDeserializer(private val components: DeserializationComponents) {
fragment, nameResolver,
TypeTable(classProto.typeTable),
VersionRequirementTable.create(classProto.versionRequirementTable),
metadataVersion,
containerSource = null
)
}
return DeserializedClassDescriptor(outerContext, classProto, nameResolver, sourceElement)
return DeserializedClassDescriptor(outerContext, classProto, nameResolver, metadataVersion, sourceElement)
}
private class ClassKey(val classId: ClassId, val classData: ClassData?) {
@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.serialization.deserialization
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.SourceElement
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.scopes.MemberScope
@@ -31,11 +32,13 @@ abstract class DeserializedPackageFragmentImpl(
storageManager: StorageManager,
module: ModuleDescriptor,
proto: ProtoBuf.PackageFragment,
private val metadataVersion: BinaryVersion,
private val containerSource: DeserializedContainerSource?
) : DeserializedPackageFragment(fqName, storageManager, module) {
protected val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames)
override val classDataFinder = ProtoBasedClassDataFinder(proto, nameResolver) { containerSource ?: SourceElement.NO_SOURCE }
override val classDataFinder =
ProtoBasedClassDataFinder(proto, nameResolver, metadataVersion) { containerSource ?: SourceElement.NO_SOURCE }
// Temporary storage: until `initialize` is called
private var _proto: ProtoBuf.PackageFragment? = proto
@@ -45,7 +48,7 @@ abstract class DeserializedPackageFragmentImpl(
val proto = _proto ?: error("Repeated call to DeserializedPackageFragmentImpl::initialize")
_proto = null
_memberScope = DeserializedPackageMemberScope(
this, proto.`package`, nameResolver, containerSource, components,
this, proto.`package`, nameResolver, metadataVersion, containerSource, components,
classNames = {
classDataFinder.allClassIds.filter { classId ->
!classId.isNestedClass && classId !in ClassDeserializer.BLACK_LIST
@@ -36,6 +36,7 @@ import org.jetbrains.kotlin.types.KotlinType
class MemberDeserializer(private val c: DeserializationContext) {
private val annotationDeserializer = AnnotationDeserializer(c.components.moduleDescriptor, c.components.notFoundClasses)
fun loadProperty(proto: ProtoBuf.Property): PropertyDescriptor {
val flags = if (proto.hasFlags()) proto.flags else loadOldFlags(proto.oldFlags)
@@ -78,11 +78,11 @@ class MetadataPackageFragment(
override val classDataFinder = ClassDataFinder { classId ->
val topLevelClassId = generateSequence(classId, ClassId::getOuterClassId).last()
val stream = finder.findMetadata(topLevelClassId) ?: return@ClassDataFinder null
val (message, nameResolver) = readProto(stream)
val (message, nameResolver, version) = readProto(stream)
message.class_List.firstOrNull { classProto ->
nameResolver.getClassId(classProto.fqName) == classId
}?.let { classProto ->
ClassData(nameResolver, classProto, SourceElement.NO_SOURCE)
ClassData(nameResolver, classProto, version, SourceElement.NO_SOURCE)
}
}
@@ -101,10 +101,11 @@ class MetadataPackageFragment(
val scopes = arrayListOf<DeserializedPackageMemberScope>()
for (partName in packageParts) {
val stream = finder.findMetadata(ClassId(fqName, Name.identifier(partName))) ?: continue
val (proto, nameResolver) = readProto(stream)
val (proto, nameResolver, version) = readProto(stream)
scopes.add(DeserializedPackageMemberScope(
this, proto.`package`, nameResolver, containerSource = null, components = components, classNames = { emptyList() }
this, proto.`package`, nameResolver, version, containerSource = null, components = components,
classNames = { emptyList() }
))
}
@@ -112,6 +113,7 @@ class MetadataPackageFragment(
scopes.add(object : DeserializedPackageMemberScope(
this, ProtoBuf.Package.getDefaultInstance(),
NameResolverImpl(ProtoBuf.StringTable.getDefaultInstance(), ProtoBuf.QualifiedNameTable.getDefaultInstance()),
BuiltInsBinaryVersion.INSTANCE, // Exact version does not matter here
containerSource = null, components = components, classNames = { emptyList() }
) {
override fun hasClass(name: Name): Boolean = hasTopLevelClass(name)
@@ -129,7 +131,7 @@ class MetadataPackageFragment(
return true
}
private fun readProto(stream: InputStream): Pair<ProtoBuf.PackageFragment, NameResolverImpl> {
private fun readProto(stream: InputStream): Triple<ProtoBuf.PackageFragment, NameResolverImpl, BuiltInsBinaryVersion> {
val version = BuiltInsBinaryVersion.readFrom(stream)
if (!version.isCompatible()) {
@@ -143,7 +145,7 @@ class MetadataPackageFragment(
val message = ProtoBuf.PackageFragment.parseFrom(stream, BuiltInSerializerProtocol.extensionRegistry)
val nameResolver = NameResolverImpl(message.strings, message.qualifiedNames)
return Pair(message, nameResolver)
return Triple(message, nameResolver, version)
}
companion object {
@@ -18,12 +18,14 @@ package org.jetbrains.kotlin.serialization.deserialization
import org.jetbrains.kotlin.descriptors.SourceElement
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
import org.jetbrains.kotlin.name.ClassId
class ProtoBasedClassDataFinder(
proto: ProtoBuf.PackageFragment,
private val nameResolver: NameResolver,
private val metadataVersion: BinaryVersion,
private val classSource: (ClassId) -> SourceElement = { SourceElement.NO_SOURCE }
) : ClassDataFinder {
private val classIdToProto =
@@ -35,6 +37,6 @@ class ProtoBasedClassDataFinder(
override fun findClassData(classId: ClassId): ClassData? {
val classProto = classIdToProto[classId] ?: return null
return ClassData(nameResolver, classProto, classSource(classId))
return ClassData(nameResolver, classProto, metadataVersion, classSource(classId))
}
}
@@ -53,7 +53,7 @@ class BuiltInsLoaderImpl : BuiltInsLoader {
val packageFragments = packageFqNames.map { fqName ->
val resourcePath = BuiltInSerializerProtocol.getBuiltInsFilePath(fqName)
val inputStream = loadResource(resourcePath) ?: throw IllegalStateException("Resource not found in classpath: $resourcePath")
BuiltInsPackageFragmentImpl(fqName, storageManager, module, inputStream)
BuiltInsPackageFragmentImpl.create(fqName, storageManager, module, inputStream)
}
val provider = PackageFragmentProviderImpl(packageFragments)
@@ -14,22 +14,40 @@ import org.jetbrains.kotlin.serialization.deserialization.DeserializedPackageFra
import org.jetbrains.kotlin.storage.StorageManager
import java.io.InputStream
class BuiltInsPackageFragmentImpl(
class BuiltInsPackageFragmentImpl private constructor(
fqName: FqName,
storageManager: StorageManager,
module: ModuleDescriptor,
inputStream: InputStream
) : BuiltInsPackageFragment, DeserializedPackageFragmentImpl(fqName, storageManager, module, inputStream.use { stream ->
val version = BuiltInsBinaryVersion.readFrom(stream)
proto: ProtoBuf.PackageFragment,
metadataVersion: BuiltInsBinaryVersion
) : BuiltInsPackageFragment, DeserializedPackageFragmentImpl(
fqName, storageManager, module, proto, metadataVersion, containerSource = null
) {
companion object {
fun create(
fqName: FqName,
storageManager: StorageManager,
module: ModuleDescriptor,
inputStream: InputStream
): BuiltInsPackageFragmentImpl {
lateinit var version: BuiltInsBinaryVersion
if (!version.isCompatible()) {
// TODO: report a proper diagnostic
throw UnsupportedOperationException(
"Kotlin built-in definition format version is not supported: " +
"expected ${BuiltInsBinaryVersion.INSTANCE}, actual $version. " +
"Please update Kotlin"
)
val proto = inputStream.use { stream ->
version = BuiltInsBinaryVersion.readFrom(stream)
if (!version.isCompatible()) {
// TODO: report a proper diagnostic
throw UnsupportedOperationException(
"Kotlin built-in definition format version is not supported: " +
"expected ${BuiltInsBinaryVersion.INSTANCE}, actual $version. " +
"Please update Kotlin"
)
}
ProtoBuf.PackageFragment.parseFrom(stream, BuiltInSerializerProtocol.extensionRegistry)
}
return BuiltInsPackageFragmentImpl(fqName, storageManager, module, proto, version)
}
}
ProtoBuf.PackageFragment.parseFrom(stream, BuiltInSerializerProtocol.extensionRegistry)
}, containerSource = null)
}
@@ -24,6 +24,7 @@ import org.jetbrains.kotlin.descriptors.deserialization.ClassDescriptorFactory
import org.jetbrains.kotlin.descriptors.deserialization.PlatformDependentDeclarationFilter
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
import org.jetbrains.kotlin.metadata.deserialization.TypeTable
import org.jetbrains.kotlin.metadata.deserialization.VersionRequirementTable
@@ -60,10 +61,11 @@ class DeserializationComponents(
nameResolver: NameResolver,
typeTable: TypeTable,
versionRequirementTable: VersionRequirementTable,
metadataVersion: BinaryVersion,
containerSource: DeserializedContainerSource?
): DeserializationContext =
DeserializationContext(
this, nameResolver, descriptor, typeTable, versionRequirementTable, containerSource,
this, nameResolver, descriptor, typeTable, versionRequirementTable, metadataVersion, containerSource,
parentTypeDeserializer = null, typeParameters = listOf()
)
}
@@ -75,6 +77,7 @@ class DeserializationContext(
val containingDeclaration: DeclarationDescriptor,
val typeTable: TypeTable,
val versionRequirementTable: VersionRequirementTable,
val metadataVersion: BinaryVersion,
val containerSource: DeserializedContainerSource?,
parentTypeDeserializer: TypeDeserializer?,
typeParameters: List<ProtoBuf.TypeParameter>
@@ -92,9 +95,10 @@ class DeserializationContext(
descriptor: DeclarationDescriptor,
typeParameterProtos: List<ProtoBuf.TypeParameter>,
nameResolver: NameResolver = this.nameResolver,
typeTable: TypeTable = this.typeTable
typeTable: TypeTable = this.typeTable,
metadataVersion: BinaryVersion = this.metadataVersion
): DeserializationContext = DeserializationContext(
components, nameResolver, descriptor, typeTable, versionRequirementTable, this.containerSource,
components, nameResolver, descriptor, typeTable, versionRequirementTable, metadataVersion, this.containerSource,
parentTypeDeserializer = this.typeDeserializer, typeParameters = typeParameterProtos
)
}
@@ -33,6 +33,7 @@ class DeserializedClassDescriptor(
outerContext: DeserializationContext,
val classProto: ProtoBuf.Class,
nameResolver: NameResolver,
val metadataVersion: BinaryVersion,
private val sourceElement: SourceElement
) : AbstractClassDescriptor(
outerContext.storageManager,
@@ -44,7 +45,7 @@ class DeserializedClassDescriptor(
private val visibility = ProtoEnumFlags.visibility(Flags.VISIBILITY.get(classProto.flags))
private val kind = ProtoEnumFlags.classKind(Flags.CLASS_KIND.get(classProto.flags))
val c = outerContext.childContext(this, classProto.typeParameterList, nameResolver, TypeTable(classProto.typeTable))
val c = outerContext.childContext(this, classProto.typeParameterList, nameResolver, TypeTable(classProto.typeTable), metadataVersion)
private val staticScope = if (kind == ClassKind.ENUM_CLASS) StaticScopeForKotlinEnum(c.storageManager, this) else MemberScope.Empty
private val typeConstructor = DeserializedClassTypeConstructor()
@@ -23,6 +23,7 @@ import org.jetbrains.kotlin.incremental.components.LookupLocation
import org.jetbrains.kotlin.incremental.components.NoLookupLocation
import org.jetbrains.kotlin.incremental.record
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
import org.jetbrains.kotlin.metadata.deserialization.TypeTable
import org.jetbrains.kotlin.metadata.deserialization.VersionRequirementTable
@@ -35,13 +36,14 @@ open class DeserializedPackageMemberScope(
private val packageDescriptor: PackageFragmentDescriptor,
proto: ProtoBuf.Package,
nameResolver: NameResolver,
metadataVersion: BinaryVersion,
containerSource: DeserializedContainerSource?,
components: DeserializationComponents,
classNames: () -> Collection<Name>
) : DeserializedMemberScope(
components.createContext(
packageDescriptor, nameResolver, TypeTable(proto.typeTable),
VersionRequirementTable.create(proto.versionRequirementTable), containerSource
VersionRequirementTable.create(proto.versionRequirementTable), metadataVersion, containerSource
),
proto.functionList, proto.propertyList, proto.typeAliasList, classNames
) {
@@ -209,7 +209,10 @@ internal class KClassImpl<T : Any>(override val jClass: Class<T>) : KDeclaration
return (descriptor as? DeserializedClassDescriptor)?.let { descriptor ->
descriptor.classProto.getExtensionOrNull(JvmProtoBuf.classLocalVariable, index)?.let { proto ->
deserializeToDescriptor(jClass, proto, descriptor.c.nameResolver, descriptor.c.typeTable, MemberDeserializer::loadProperty)
deserializeToDescriptor(
jClass, proto, descriptor.c.nameResolver, descriptor.c.typeTable, descriptor.metadataVersion,
MemberDeserializer::loadProperty
)
}
}
}
@@ -24,6 +24,7 @@ import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.TypeTable
import org.jetbrains.kotlin.metadata.deserialization.getExtensionOrNull
import org.jetbrains.kotlin.metadata.jvm.JvmProtoBuf
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmNameResolver
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
import org.jetbrains.kotlin.name.Name
@@ -63,12 +64,13 @@ internal class KPackageImpl(
}
}
val metadata: Pair<JvmNameResolver, ProtoBuf.Package>? by ReflectProperties.lazy {
val metadata: Triple<JvmNameResolver, ProtoBuf.Package, JvmMetadataVersion>? by ReflectProperties.lazy {
kotlinClass?.classHeader?.let { header ->
val data = header.data
val strings = header.strings
if (data != null && strings != null) {
JvmProtoBufUtil.readPackageDataFrom(data, strings)
val (nameResolver, proto) = JvmProtoBufUtil.readPackageDataFrom(data, strings)
Triple(nameResolver, proto, header.metadataVersion)
} else null
}
}
@@ -101,9 +103,12 @@ internal class KPackageImpl(
scope.getContributedFunctions(name, NoLookupLocation.FROM_REFLECTION)
override fun getLocalProperty(index: Int): PropertyDescriptor? {
return data().metadata?.let { (nameResolver, packageProto) ->
return data().metadata?.let { (nameResolver, packageProto, metadataVersion) ->
packageProto.getExtensionOrNull(JvmProtoBuf.packageLocalVariable, index)?.let { proto ->
deserializeToDescriptor(jClass, proto, nameResolver, TypeTable(packageProto.typeTable), MemberDeserializer::loadProperty)
deserializeToDescriptor(
jClass, proto, nameResolver, TypeTable(packageProto.typeTable), metadataVersion,
MemberDeserializer::loadProperty
)
}
}
}
@@ -24,10 +24,7 @@ import org.jetbrains.kotlin.load.java.JvmAbi
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
import org.jetbrains.kotlin.metadata.deserialization.TypeTable
import org.jetbrains.kotlin.metadata.deserialization.VersionRequirementTable
import org.jetbrains.kotlin.metadata.deserialization.getExtensionOrNull
import org.jetbrains.kotlin.metadata.deserialization.*
import org.jetbrains.kotlin.metadata.jvm.JvmProtoBuf
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
import org.jetbrains.kotlin.name.FqName
@@ -158,6 +155,7 @@ internal fun <M : MessageLite, D : CallableDescriptor> deserializeToDescriptor(
proto: M,
nameResolver: NameResolver,
typeTable: TypeTable,
metadataVersion: BinaryVersion,
createDescriptor: MemberDeserializer.(M) -> D
): D? {
val moduleData = moduleAnchor.getOrCreateModule()
@@ -169,7 +167,7 @@ internal fun <M : MessageLite, D : CallableDescriptor> deserializeToDescriptor(
}
val context = DeserializationContext(
moduleData.deserialization, nameResolver, moduleData.module, typeTable, VersionRequirementTable.EMPTY,
moduleData.deserialization, nameResolver, moduleData.module, typeTable, VersionRequirementTable.EMPTY, metadataVersion,
containerSource = null, parentTypeDeserializer = null, typeParameters = typeParameters
)
return MemberDeserializer(context).createDescriptor(proto)
@@ -19,6 +19,7 @@
package kotlin.reflect.jvm
import org.jetbrains.kotlin.metadata.deserialization.TypeTable
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
import org.jetbrains.kotlin.serialization.deserialization.MemberDeserializer
import kotlin.reflect.KFunction
@@ -35,9 +36,11 @@ fun <R> Function<R>.reflect(): KFunction<R>? {
val annotation = javaClass.getAnnotation(Metadata::class.java) ?: return null
val data = annotation.d1.takeUnless(Array<String>::isEmpty) ?: return null
val (nameResolver, proto) = JvmProtoBufUtil.readFunctionDataFrom(data, annotation.d2)
val metadataVersion = JvmMetadataVersion(*annotation.mv)
val descriptor = deserializeToDescriptor(javaClass, proto, nameResolver, TypeTable(proto.typeTable), MemberDeserializer::loadFunction)
?: return null
val descriptor = deserializeToDescriptor(
javaClass, proto, nameResolver, TypeTable(proto.typeTable), metadataVersion, MemberDeserializer::loadFunction
) ?: return null
@Suppress("UNCHECKED_CAST")
return KFunctionImpl(EmptyContainerForLocal, descriptor) as KFunction<R>
@@ -73,9 +73,10 @@ object JsAnalyzerFacade : ResolverForModuleFactory() {
.flatMap { KotlinJavascriptMetadataUtils.loadMetadata(it) }
.filter { it.version.isCompatible() }
.map { metadata ->
val (header, packageFragmentProtos) = KotlinJavascriptSerializationUtil.readModuleAsProto(metadata.body)
val (header, packageFragmentProtos) =
KotlinJavascriptSerializationUtil.readModuleAsProto(metadata.body, metadata.version)
createKotlinJavascriptPackageFragmentProvider(
moduleContext.storageManager, moduleDescriptor, header, packageFragmentProtos,
moduleContext.storageManager, moduleDescriptor, header, packageFragmentProtos, metadata.version,
container.get(), LookupTracker.DO_NOTHING
)
}
@@ -44,9 +44,10 @@ class KotlinBuiltInDecompiler : KotlinMetadataDecompiler<BuiltInsBinaryVersion>(
class BuiltInDefinitionFile(
proto: ProtoBuf.PackageFragment,
version: BuiltInsBinaryVersion,
val packageDirectory: VirtualFile,
val isMetadata: Boolean
) : FileWithMetadata.Compatible(proto, BuiltInSerializerProtocol) {
) : FileWithMetadata.Compatible(proto, version, BuiltInSerializerProtocol) {
override val classesToDecompile: List<ProtoBuf.Class>
get() = super.classesToDecompile.let { classes ->
if (isMetadata || !FILTER_OUT_CLASSES_EXISTING_AS_JVM_CLASS_FILES) classes
@@ -73,7 +74,8 @@ class BuiltInDefinitionFile(
}
val proto = ProtoBuf.PackageFragment.parseFrom(stream, BuiltInSerializerProtocol.extensionRegistry)
val result = BuiltInDefinitionFile(proto, file.parent, file.extension == MetadataPackageFragment.METADATA_FILE_EXTENSION)
val result =
BuiltInDefinitionFile(proto, version, file.parent, file.extension == MetadataPackageFragment.METADATA_FILE_EXTENSION)
val packageProto = result.proto.`package`
if (result.classesToDecompile.isEmpty() &&
packageProto.typeAliasCount == 0 && packageProto.functionCount == 0 && packageProto.propertyCount == 0) {
@@ -87,8 +87,9 @@ class DeserializerForClassfileDecompiler(
}
val (nameResolver, packageProto) = JvmProtoBufUtil.readPackageDataFrom(annotationData, strings)
val membersScope = DeserializedPackageMemberScope(
createDummyPackageFragment(header.packageName?.let(::FqName) ?: facadeFqName.parent()), packageProto, nameResolver,
JvmPackagePartSource(binaryClassForPackageClass, packageProto, nameResolver), deserializationComponents
createDummyPackageFragment(header.packageName?.let(::FqName) ?: facadeFqName.parent()),
packageProto, nameResolver, header.metadataVersion,
JvmPackagePartSource(binaryClassForPackageClass, packageProto, nameResolver), deserializationComponents
) { emptyList() }
return membersScope.getContributedDescriptors().toList()
}
@@ -145,6 +146,6 @@ class DirectoryBasedDataFinder(
}
val (nameResolver, classProto) = JvmProtoBufUtil.readClassDataFrom(data, strings)
return ClassData(nameResolver, classProto, KotlinJvmBinarySourceElement(binaryClass))
return ClassData(nameResolver, classProto, classHeader.metadataVersion, KotlinJvmBinarySourceElement(binaryClass))
}
}
@@ -101,7 +101,7 @@ abstract class KotlinMetadataDecompiler<out V : BinaryVersion>(
is FileWithMetadata.Compatible -> {
val packageFqName = file.packageFqName
val resolver = KotlinMetadataDeserializerForDecompiler(
packageFqName, file.proto, file.nameResolver,
packageFqName, file.proto, file.nameResolver, file.version,
targetPlatform, serializerProtocol, flexibleTypeDeserializer
)
val declarations = arrayListOf<DeclarationDescriptor>()
@@ -120,8 +120,9 @@ sealed class FileWithMetadata {
class Incompatible(val version: BinaryVersion) : FileWithMetadata()
open class Compatible(
val proto: ProtoBuf.PackageFragment,
serializerProtocol: SerializerExtensionProtocol
val proto: ProtoBuf.PackageFragment,
val version: BinaryVersion,
serializerProtocol: SerializerExtensionProtocol
) : FileWithMetadata() {
val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames)
val packageFqName = FqName(nameResolver.getPackageFqName(proto.`package`.getExtension(serializerProtocol.packageFqName)))
@@ -26,6 +26,7 @@ import org.jetbrains.kotlin.idea.decompiler.textBuilder.LoggingErrorReporter
import org.jetbrains.kotlin.idea.decompiler.textBuilder.ResolveEverythingToKotlinAnyLocalClassifierResolver
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.TargetPlatform
@@ -37,6 +38,7 @@ class KotlinMetadataDeserializerForDecompiler(
packageFqName: FqName,
private val proto: ProtoBuf.PackageFragment,
private val nameResolver: NameResolver,
private val metadataVersion: BinaryVersion,
override val targetPlatform: TargetPlatform,
serializerProtocol: SerializerExtensionProtocol,
flexibleTypeDeserializer: FlexibleTypeDeserializer
@@ -49,7 +51,8 @@ class KotlinMetadataDeserializerForDecompiler(
val notFoundClasses = NotFoundClasses(storageManager, moduleDescriptor)
deserializationComponents = DeserializationComponents(
storageManager, moduleDescriptor, DeserializationConfiguration.Default, ProtoBasedClassDataFinder(proto, nameResolver),
storageManager, moduleDescriptor, DeserializationConfiguration.Default,
ProtoBasedClassDataFinder(proto, nameResolver, metadataVersion),
AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, serializerProtocol), packageFragmentProvider,
ResolveEverythingToKotlinAnyLocalClassifierResolver(builtIns), LoggingErrorReporter(LOG),
LookupTracker.DO_NOTHING, flexibleTypeDeserializer, emptyList(), notFoundClasses,
@@ -64,8 +67,8 @@ class KotlinMetadataDeserializerForDecompiler(
}
val membersScope = DeserializedPackageMemberScope(
createDummyPackageFragment(facadeFqName), proto.`package`, nameResolver, containerSource = null,
components = deserializationComponents
createDummyPackageFragment(facadeFqName), proto.`package`, nameResolver, metadataVersion, containerSource = null,
components = deserializationComponents
) { emptyList() }
return membersScope.getContributedDescriptors().toList()
@@ -51,7 +51,7 @@ open class KotlinMetadataStubBuilder(
val packageFqName = file.packageFqName
val nameResolver = file.nameResolver
val components = ClsStubBuilderComponents(
ProtoBasedClassDataFinder(file.proto, nameResolver),
ProtoBasedClassDataFinder(file.proto, nameResolver, file.version),
AnnotationLoaderForStubBuilderImpl(serializerProtocol),
virtualFile
)
@@ -43,6 +43,6 @@ class KotlinJavaScriptMetaFileDecompiler : KotlinMetadataDecompiler<JsMetadataVe
JsProtoBuf.Header.parseDelimitedFrom(stream)
val proto = ProtoBuf.PackageFragment.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
return FileWithMetadata.Compatible(proto, JsSerializerProtocol)
return FileWithMetadata.Compatible(proto, version, JsSerializerProtocol)
}
}
@@ -247,7 +247,7 @@ private class ClassClsStubBuilder(
}
private fun createNestedClassStub(classBody: StubElement<out PsiElement>, nestedClassId: ClassId) {
val (nameResolver, classProto, sourceElement) =
val (nameResolver, classProto, _, sourceElement) =
c.components.classDataFinder.findClassData(nestedClassId)
?: c.components.virtualFileForDebug.let { rootFile ->
LOG.error(
@@ -40,9 +40,10 @@ import org.jetbrains.kotlin.incremental.testingUtils.TouchPolicy
import org.jetbrains.kotlin.incremental.testingUtils.copyTestSources
import org.jetbrains.kotlin.incremental.testingUtils.getModificationsToPerform
import org.jetbrains.kotlin.incremental.utils.TestMessageCollector
import org.jetbrains.kotlin.jps.incremental.runJSCompiler
import org.jetbrains.kotlin.jps.incremental.createTestingCompilerEnvironment
import org.jetbrains.kotlin.jps.incremental.runJSCompiler
import org.jetbrains.kotlin.test.KotlinTestUtils
import org.jetbrains.kotlin.utils.JsMetadataVersion
import java.io.*
import java.util.*
@@ -119,7 +120,10 @@ abstract class AbstractJsLookupTrackerTest : AbstractLookupTrackerTest() {
override fun Services.Builder.registerAdditionalServices() {
if (header != null) {
register(IncrementalDataProvider::class.java, IncrementalDataProviderImpl(header!!, packageParts!!))
register(
IncrementalDataProvider::class.java,
IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray())
)
}
register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl())
@@ -22,11 +22,15 @@ import java.io.File
interface IncrementalDataProvider {
/** gets header metadata (serialized [JsProtoBuf.Header]) from previous compilation */
val headerMetadata: ByteArray
/** gets non-dirty package parts data from previous compilation */
val compiledPackageParts: Map<File, TranslationResultValue>
val metadataVersion: IntArray
}
class IncrementalDataProviderImpl(
override val headerMetadata: ByteArray,
override val compiledPackageParts: Map<File, TranslationResultValue>
) : IncrementalDataProvider
override val compiledPackageParts: Map<File, TranslationResultValue>,
override val metadataVersion: IntArray
) : IncrementalDataProvider
@@ -28,6 +28,7 @@ import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProvid
import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil
import org.jetbrains.kotlin.serialization.js.ModuleKind
import org.jetbrains.kotlin.serialization.js.PackagesWithHeaderMetadata
import org.jetbrains.kotlin.utils.JsMetadataVersion
object TopDownAnalyzerFacadeForJS {
@JvmStatic
@@ -85,8 +86,12 @@ object TopDownAnalyzerFacadeForJS {
val lookupTracker = configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER) ?: LookupTracker.DO_NOTHING
val expectActualTracker = configuration.get(CommonConfigurationKeys.EXPECT_ACTUAL_TRACKER) ?: ExpectActualTracker.DoNothing
val languageVersionSettings = configuration.languageVersionSettings
val packageFragment = configuration[JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER]?.let {
val metadata = PackagesWithHeaderMetadata(it.headerMetadata, it.compiledPackageParts.values.map { it.metadata })
val packageFragment = configuration[JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER]?.let { incrementalData ->
val metadata = PackagesWithHeaderMetadata(
incrementalData.headerMetadata,
incrementalData.compiledPackageParts.values.map { it.metadata },
JsMetadataVersion(*incrementalData.metadataVersion)
)
KotlinJavascriptSerializationUtil.readDescriptors(
metadata, moduleContext.storageManager, moduleContext.module,
CompilerDeserializationConfiguration(languageVersionSettings), lookupTracker
@@ -250,8 +250,9 @@ public class JsConfig {
Name.special("<" + cached.getName() + ">"), storageManager, JsPlatform.INSTANCE.getBuiltIns()
);
KotlinJavaScriptLibraryParts parts = cached.getData();
PackageFragmentProvider provider = KotlinJavascriptPackageFragmentProviderKt.createKotlinJavascriptPackageFragmentProvider(
storageManager, moduleDescriptor, cached.getData().getHeader(), cached.getData().getBody(),
storageManager, moduleDescriptor, parts.getHeader(), parts.getBody(), parts.getMetadataVersion(),
new CompilerDeserializationConfiguration(languageVersionSettings),
LookupTracker.DO_NOTHING.INSTANCE
);
@@ -309,9 +310,9 @@ public class JsConfig {
);
LookupTracker lookupTracker = configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER, LookupTracker.DO_NOTHING.INSTANCE);
KotlinJavaScriptLibraryParts parts = KotlinJavascriptSerializationUtil.readModuleAsProto(m.getBody());
KotlinJavaScriptLibraryParts parts = KotlinJavascriptSerializationUtil.readModuleAsProto(m.getBody(), m.getVersion());
PackageFragmentProvider provider = KotlinJavascriptPackageFragmentProviderKt.createKotlinJavascriptPackageFragmentProvider(
storageManager, moduleDescriptor, parts.getHeader(), parts.getBody(),
storageManager, moduleDescriptor, parts.getHeader(), parts.getBody(), m.getVersion(),
new CompilerDeserializationConfiguration(languageVersionSettings),
lookupTracker
);
@@ -31,6 +31,7 @@ import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.storage.getValue
import org.jetbrains.kotlin.utils.JsMetadataVersion
class KotlinJavascriptPackageFragment(
fqName: FqName,
@@ -38,8 +39,11 @@ class KotlinJavascriptPackageFragment(
module: ModuleDescriptor,
proto: ProtoBuf.PackageFragment,
header: JsProtoBuf.Header,
metadataVersion: JsMetadataVersion,
configuration: DeserializationConfiguration
) : DeserializedPackageFragmentImpl(fqName, storageManager, module, proto, JsContainerSource(fqName, header, configuration)) {
) : DeserializedPackageFragmentImpl(
fqName, storageManager, module, proto, metadataVersion, JsContainerSource(fqName, header, configuration)
) {
val fileMap: Map<Int, FileHolder> =
proto.getExtension(JsProtoBuf.packageFragmentFiles).fileList.withIndex().associate { (index, file) ->
(if (file.hasId()) file.id else index) to FileHolder(file.annotationList)
@@ -62,7 +62,9 @@ object KotlinJavascriptSerializationUtil {
ProtoBuf.PackageFragment.parseFrom(it, JsSerializerProtocol.extensionRegistry)
}
val headerProto = JsProtoBuf.Header.parseFrom(CodedInputStream.newInstance(metadata.header), JsSerializerProtocol.extensionRegistry)
return createKotlinJavascriptPackageFragmentProvider(storageManager, module, headerProto, scopeProto, configuration, lookupTracker)
return createKotlinJavascriptPackageFragmentProvider(
storageManager, module, headerProto, scopeProto, metadata.metadataVersion, configuration, lookupTracker
)
}
fun serializeMetadata(
@@ -302,7 +304,7 @@ object KotlinJavascriptSerializationUtil {
}
@JvmStatic
fun readModuleAsProto(metadata: ByteArray): KotlinJavaScriptLibraryParts {
fun readModuleAsProto(metadata: ByteArray, metadataVersion: JsMetadataVersion): KotlinJavaScriptLibraryParts {
val (header, content) = GZIPInputStream(ByteArrayInputStream(metadata)).use { stream ->
JsProtoBuf.Header.parseDelimitedFrom(stream, JsSerializerProtocol.extensionRegistry) to
JsProtoBuf.Library.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
@@ -315,7 +317,7 @@ object KotlinJavascriptSerializationUtil {
JsProtoBuf.Library.Kind.UMD -> ModuleKind.UMD
}
return KotlinJavaScriptLibraryParts(header, content.packageFragmentList, moduleKind, content.importedModuleList)
return KotlinJavaScriptLibraryParts(header, content.packageFragmentList, moduleKind, content.importedModuleList, metadataVersion)
}
}
@@ -323,7 +325,8 @@ data class KotlinJavaScriptLibraryParts(
val header: JsProtoBuf.Header,
val body: List<ProtoBuf.PackageFragment>,
val kind: ModuleKind,
val importedModules: List<String>
val importedModules: List<String>,
val metadataVersion: JsMetadataVersion
)
internal fun DeclarationDescriptor.extractFileId(): Int? = when (this) {
@@ -16,4 +16,10 @@
package org.jetbrains.kotlin.serialization.js
class PackagesWithHeaderMetadata(val header: ByteArray, val packages: List<ByteArray>)
import org.jetbrains.kotlin.utils.JsMetadataVersion
class PackagesWithHeaderMetadata(
val header: ByteArray,
val packages: List<ByteArray>,
val metadataVersion: JsMetadataVersion
)
@@ -28,18 +28,20 @@ import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.parentOrNull
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.utils.JsMetadataVersion
fun createKotlinJavascriptPackageFragmentProvider(
storageManager: StorageManager,
module: ModuleDescriptor,
header: JsProtoBuf.Header,
packageFragmentProtos: List<ProtoBuf.PackageFragment>,
metadataVersion: JsMetadataVersion,
configuration: DeserializationConfiguration,
lookupTracker: LookupTracker
): PackageFragmentProvider {
val packageFragments: MutableList<PackageFragmentDescriptor> = packageFragmentProtos.mapNotNullTo(mutableListOf()) { proto ->
proto.fqName?.let { fqName ->
KotlinJavascriptPackageFragment(fqName, storageManager, module, proto, header, configuration)
KotlinJavascriptPackageFragment(fqName, storageManager, module, proto, header, metadataVersion, configuration)
}
}
@@ -59,6 +59,7 @@ import org.jetbrains.kotlin.test.KotlinTestUtils.TestFileFactory
import org.jetbrains.kotlin.test.KotlinTestWithEnvironment
import org.jetbrains.kotlin.test.TargetBackend
import org.jetbrains.kotlin.utils.DFS
import org.jetbrains.kotlin.utils.JsMetadataVersion
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadata
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
import java.io.*
@@ -409,10 +410,11 @@ abstract class BasicBoxTest(
return String(out.toByteArray(), Charset.forName("UTF-8"))
}
private fun metadataAsString(metadata: String): String {
val containers = mutableListOf<KotlinJavascriptMetadata>()
KotlinJavascriptMetadataUtils.parseMetadata(metadata, containers)
val metadataParts = KotlinJavascriptSerializationUtil.readModuleAsProto(containers.single().body).body
private fun metadataAsString(metadataText: String): String {
val metadata = mutableListOf<KotlinJavascriptMetadata>().apply {
KotlinJavascriptMetadataUtils.parseMetadata(metadataText, this)
}.single()
val metadataParts = KotlinJavascriptSerializationUtil.readModuleAsProto(metadata.body, metadata.version).body
return metadataParts.joinToString("-----\n") {
val binary = it.toByteArray()
DebugProtoBuf.PackageFragment.parseFrom(binary, JsSerializerProtocol.extensionRegistry).toString()
@@ -576,8 +578,10 @@ abstract class BasicBoxTest(
if (hasFilesToRecompile) {
val header = incrementalData?.header
if (header != null) {
configuration.put(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER,
IncrementalDataProviderImpl(header, incrementalData.translatedFiles))
configuration.put(
JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER,
IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray())
)
}
configuration.put(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER, IncrementalResultsConsumerImpl())
@@ -743,7 +747,7 @@ abstract class BasicBoxTest(
companion object {
val METADATA_CACHE = (JsConfig.JS_STDLIB + JsConfig.JS_KOTLIN_TEST).flatMap { path ->
KotlinJavascriptMetadataUtils.loadMetadata(path).map { metadata ->
val parts = KotlinJavascriptSerializationUtil.readModuleAsProto(metadata.body)
val parts = KotlinJavascriptSerializationUtil.readModuleAsProto(metadata.body, metadata.version)
JsModuleDescriptor(metadata.moduleName, parts.kind, parts.importedModules, parts)
}
}