move Kotlin/Native metadata deserialization and KLIB reader to a separate module (#1908)

1. move Kotlin/Native metadata deserialization and KLIB reader to a separate module
2. decouple KLIB reader from ModuleDescriptor creation
This commit is contained in:
Dmitriy Dolovov
2018-08-22 14:01:20 +03:00
committed by GitHub
parent 0f00b39fda
commit f0cfbb9918
47 changed files with 610 additions and 720 deletions
@@ -1 +0,0 @@
rootProject.name = "konan.metadata.serializer"
@@ -19,7 +19,7 @@ repositories {
sourceSets {
main.kotlin {
srcDir 'src/main/kotlin'
srcDir 'src'
}
}
@@ -0,0 +1 @@
rootProject.name = "konan.serializer"
@@ -0,0 +1,44 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
// This scheme describes the Kotlin/Native Library (klib) layout.
interface KonanLibraryLayout {
val libDir: File
val target: KonanTarget?
// This is a default implementation. Can't make it an assignment.
get() = null
val manifestFile
get() = File(libDir, "manifest")
val resourcesDir
get() = File(libDir, "resources")
val targetsDir
get() = File(libDir, "targets")
val targetDir
get() = File(targetsDir, target!!.visibleName)
val kotlinDir
get() = File(targetDir, "kotlin")
val nativeDir
get() = File(targetDir, "native")
val includedDir
get() = File(targetDir, "included")
val linkdataDir
get() = File(libDir, "linkdata")
val moduleHeaderFile
get() = File(linkdataDir, "module")
val dataFlowGraphFile
get() = File(linkdataDir, "module_data_flow_graph")
fun packageFile(packageName: String)
= File(linkdataDir, if (packageName == "") "root_package.knm" else "package_$packageName.knm")
}
interface KonanLibrary: KonanLibraryLayout {
val libraryName: String
}
@@ -0,0 +1,28 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.properties.Properties
interface KonanLibraryReader {
val libraryFile: File
val libraryName: String
val manifestProperties: Properties
val uniqueName: String
val linkerOpts: List<String>
val unresolvedDependencies: List<String>
val bitcodePaths: List<String>
val includedPaths: List<String>
val dataFlowGraph: ByteArray?
val moduleHeaderData: ByteArray
fun packageMetadata(fqName: String): ByteArray
val isDefaultLibrary: Boolean get() = false
// FIXME: ddol: to be refactored into some global resolution context
val isNeededForLink: Boolean get() = true
val resolvedDependencies: MutableList<KonanLibraryReader>
fun markPackageAccessed(fqName: String)
}
@@ -0,0 +1,6 @@
package org.jetbrains.kotlin.konan.library
interface MetadataReader {
fun loadSerializedModule(library: KonanLibrary): ByteArray
fun loadSerializedPackageFragment(library: KonanLibrary, fqName: String): ByteArray
}
@@ -0,0 +1,12 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.MetadataReader
object DefaultMetadataReaderImpl : MetadataReader {
override fun loadSerializedModule(library: KonanLibrary): ByteArray = library.moduleHeaderFile.readBytes()
override fun loadSerializedPackageFragment(library: KonanLibrary, fqName: String): ByteArray =
library.packageFile(fqName).readBytes()
}
@@ -0,0 +1,85 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
import org.jetbrains.kotlin.konan.file.*
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
class ZippedKonanLibrary(val klibFile: File, override val target: KonanTarget? = null): KonanLibrary {
init {
check(klibFile.exists) { "Could not find $klibFile." }
check(klibFile.isFile) { "Expected $klibFile to be a regular file." }
val extension = klibFile.extension
check(extension.isEmpty() || extension == KLIB_FILE_EXTENSION) { "Unexpected file extension: $extension" }
}
override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
override val libDir by lazy { klibFile.asZipRoot }
fun unpackTo(newDir: File) {
if (newDir.exists) {
if (newDir.isDirectory)
newDir.deleteRecursively()
else
newDir.delete()
}
libDir.recursiveCopyTo(newDir)
check(newDir.exists) { "Could not unpack $klibFile as $newDir." }
}
}
// This class automatically extracts pieces of
// the library on first access. Use it if you need
// to pass extracted files to an external tool.
// Otherwise, stick to ZippedKonanLibrary.
private class FileExtractor(zippedLibrary: KonanLibrary): KonanLibrary by zippedLibrary {
override val manifestFile: File by lazy { extract(super.manifestFile) }
override val resourcesDir: File by lazy { extractDir(super.resourcesDir) }
override val includedDir: File by lazy { extractDir(super.includedDir) }
override val kotlinDir: File by lazy { extractDir(super.kotlinDir) }
override val nativeDir: File by lazy { extractDir(super.nativeDir) }
override val linkdataDir: File by lazy { extractDir(super.linkdataDir) }
fun extract(file: File): File {
val temporary = createTempFile(file.name)
file.copyTo(temporary)
temporary.deleteOnExit()
return temporary
}
fun extractDir(directory: File): File {
val temporary = createTempDir(directory.name)
directory.recursiveCopyTo(temporary)
temporary.deleteOnExitRecursively()
return temporary
}
}
class UnzippedKonanLibrary(override val libDir: File, override val target: KonanTarget? = null): KonanLibrary {
override val libraryName = libDir.path
val targetList: List<String> by lazy { targetsDir.listFiles.map { it.name } }
}
fun KonanLibrary(klib: File, target: KonanTarget? = null) =
if (klib.isFile) ZippedKonanLibrary(klib, target) else UnzippedKonanLibrary(klib, target)
internal val KonanLibrary.realFiles
get() = when (this) {
is ZippedKonanLibrary -> FileExtractor(this)
// Unpacked library just provides its own files.
is UnzippedKonanLibrary -> this
else -> error("Provide an extractor for your container.")
}
@@ -0,0 +1,83 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.MetadataReader
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.properties.propertyString
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
import org.jetbrains.kotlin.konan.util.substitute
import org.jetbrains.kotlin.serialization.konan.emptyPackages
class LibraryReaderImpl(
override val libraryFile: File,
val currentAbiVersion: Int,
val target: KonanTarget? = null,
override val isDefaultLibrary: Boolean = false,
private val metadataReader: MetadataReader = DefaultMetadataReaderImpl
) : KonanLibraryReader {
// For the zipped libraries inPlace gives files from zip file system
// whereas realFiles extracts them to /tmp.
// For unzipped libraries inPlace and realFiles are the same
// providing files in the library directory.
private val inPlace = KonanLibrary(libraryFile, target)
private val realFiles = inPlace.realFiles
override val manifestProperties: Properties by lazy {
val properties = inPlace.manifestFile.loadProperties()
if (target != null) substitute(properties, defaultTargetSubstitutions(target))
properties
}
val abiVersion: String
get() {
val manifestAbiVersion = manifestProperties.getProperty("abi_version")
check("$currentAbiVersion" == manifestAbiVersion) {
"ABI version mismatch. Compiler expects: $currentAbiVersion, the library is $manifestAbiVersion"
}
return manifestAbiVersion
}
val targetList = inPlace.targetsDir.listFiles.map { it.name }
override val dataFlowGraph by lazy { inPlace.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } }
override val libraryName
get() = inPlace.libraryName
override val uniqueName
get() = manifestProperties.propertyString("unique_name")!!
override val bitcodePaths: List<String>
get() = (realFiles.kotlinDir.listFilesOrEmpty + realFiles.nativeDir.listFilesOrEmpty).map { it.absolutePath }
override val includedPaths: List<String>
get() = realFiles.includedDir.listFilesOrEmpty.map { it.absolutePath }
override val linkerOpts: List<String>
get() = manifestProperties.propertyList("linkerOpts", target!!.visibleName)
override val unresolvedDependencies: List<String>
get() = manifestProperties.propertyList("depends")
override val resolvedDependencies = mutableListOf<KonanLibraryReader>()
override val moduleHeaderData: ByteArray by lazy { metadataReader.loadSerializedModule(inPlace) }
override var isNeededForLink: Boolean = false
private set
private val emptyPackages by lazy { emptyPackages(moduleHeaderData) }
override fun markPackageAccessed(fqName: String) {
if (!isNeededForLink // fast path
&& !emptyPackages.contains(fqName)) {
isNeededForLink = true
}
}
override fun packageMetadata(fqName: String) = metadataReader.loadSerializedPackageFragment(inPlace, fqName)
}
@@ -0,0 +1,29 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.descriptors.SourceElement
import org.jetbrains.kotlin.metadata.deserialization.NameResolver
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.serialization.deserialization.ClassData
import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
import org.jetbrains.kotlin.serialization.deserialization.getClassId
class KonanClassDataFinder(
private val fragment: KonanProtoBuf.LinkDataPackageFragment,
private val nameResolver: NameResolver
) : ClassDataFinder {
override fun findClassData(classId: ClassId): ClassData? {
val proto = fragment.classes
val nameList = proto.classNameList
val index = nameList.indexOfFirst { nameResolver.getClassId(it) == classId }
if (index == -1)
error("Could not find serialized class $classId")
val foundClass = proto.getClasses(index) ?: error("Could not find data for serialized class $classId")
/* TODO: binary version supposed to be read from protobuf. */
return ClassData(nameResolver, foundClass, KonanMetadataVersion.INSTANCE, SourceElement.NO_SOURCE)
}
}
@@ -0,0 +1,11 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
fun parsePackageFragment(packageData: ByteArray): KonanProtoBuf.LinkDataPackageFragment =
KonanProtoBuf.LinkDataPackageFragment.parseFrom(packageData, KonanSerializerProtocol.extensionRegistry)
fun parseModuleHeader(libraryData: ByteArray): KonanProtoBuf.LinkDataLibrary =
KonanProtoBuf.LinkDataLibrary.parseFrom(libraryData, KonanSerializerProtocol.extensionRegistry)
fun emptyPackages(libraryData: ByteArray) = parseModuleHeader(libraryData).emptyPackageList
@@ -0,0 +1,16 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
class KonanMetadataVersion(vararg numbers: Int) : BinaryVersion(*numbers) {
override fun isCompatible(): Boolean = this.major == 1 && this.minor == 0
companion object {
@JvmField
val INSTANCE = KonanMetadataVersion(1, 0, 0)
@JvmField
val INVALID_VERSION = KonanMetadataVersion()
}
}
@@ -0,0 +1,139 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
import org.jetbrains.kotlin.descriptors.konan.createKonanModuleDescriptor
import org.jetbrains.kotlin.descriptors.konan.interop.InteropFqNames
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager
// FIXME: ddol: this is a temporary solution, to be refactored into some global resolution context
interface KonanModuleDescriptorFactory {
fun createModuleDescriptor(
libraryReader: KonanLibraryReader,
specifics: LanguageVersionSettings,
storageManager: StorageManager = LockBasedStorageManager()
): ModuleDescriptor
}
object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
override fun createModuleDescriptor(
libraryReader: KonanLibraryReader,
specifics: LanguageVersionSettings,
storageManager: StorageManager
): ModuleDescriptorImpl {
val libraryProto = parseModuleHeader(libraryReader.moduleHeaderData)
val moduleName = libraryProto.moduleName
val moduleDescriptor = createKonanModuleDescriptor(
Name.special(moduleName),
storageManager,
origin = DeserializedKonanModuleOrigin(libraryReader)
)
val deserializationConfiguration = CompilerDeserializationConfiguration(specifics)
val provider = createPackageFragmentProvider(
libraryReader,
libraryProto.packageFragmentNameList,
storageManager,
moduleDescriptor,
deserializationConfiguration)
moduleDescriptor.initialize(provider)
return moduleDescriptor
}
private fun createPackageFragmentProvider(
libraryReader: KonanLibraryReader,
fragmentNames: List<String>,
storageManager: StorageManager,
moduleDescriptor: ModuleDescriptor,
configuration: DeserializationConfiguration
): PackageFragmentProvider {
val deserializedPackageFragments = fragmentNames.map{
KonanPackageFragment(it, libraryReader, storageManager, moduleDescriptor)
}
val syntheticPackageFragments = getSyntheticPackageFragments(
libraryReader,
moduleDescriptor,
deserializedPackageFragments)
val provider = PackageFragmentProviderImpl(deserializedPackageFragments + syntheticPackageFragments)
val notFoundClasses = NotFoundClasses(storageManager, moduleDescriptor)
val annotationAndConstantLoader = AnnotationAndConstantLoaderImpl(
moduleDescriptor,
notFoundClasses,
KonanSerializerProtocol)
val components = DeserializationComponents(
storageManager,
moduleDescriptor,
configuration,
DeserializedClassDataFinder(provider),
annotationAndConstantLoader,
provider,
LocalClassifierTypeSettings.Default,
ErrorReporter.DO_NOTHING,
LookupTracker.DO_NOTHING,
NullFlexibleTypeDeserializer,
emptyList(),
notFoundClasses,
ContractDeserializer.DEFAULT,
extensionRegistryLite = KonanSerializerProtocol.extensionRegistry)
for (packageFragment in deserializedPackageFragments) {
packageFragment.initialize(components)
}
return provider
}
private fun getSyntheticPackageFragments(
libraryReader: KonanLibraryReader,
moduleDescriptor: ModuleDescriptor,
konanPackageFragments: List<KonanPackageFragment>
): List<PackageFragmentDescriptor> {
if (libraryReader.manifestProperties.getProperty("interop") != "true") return emptyList()
val packageFqName = libraryReader.manifestProperties.getProperty("package")?.let { FqName(it) }
?: error("Inconsistent manifest: interop library ${libraryReader.libraryName} should have `package` specified")
val exportForwardDeclarations = libraryReader.manifestProperties.getProperty("exportForwardDeclarations")
.split(' ')
.map { it.trim() }
.filter { it.isNotEmpty() }
.map { FqName(it) }
val interopPackageFragments = konanPackageFragments.filter { it.fqName == packageFqName }
val result = mutableListOf<PackageFragmentDescriptor>()
// Allow references to forwarding declarations to be resolved into classifiers declared in this library:
listOf(InteropFqNames.cNamesStructs, InteropFqNames.objCNamesClasses, InteropFqNames.objCNamesProtocols).mapTo(result) { fqName ->
ClassifierAliasingPackageFragmentDescriptor(interopPackageFragments, moduleDescriptor, fqName)
}
// TODO: use separate namespaces for structs, enums, Objective-C protocols etc.
result.add(ExportedForwardDeclarationsPackageFragmentDescriptor(moduleDescriptor, packageFqName, exportForwardDeclarations))
return result
}
}
@@ -0,0 +1,80 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents
import org.jetbrains.kotlin.serialization.deserialization.DeserializedPackageFragment
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
import org.jetbrains.kotlin.serialization.deserialization.getClassId
import org.jetbrains.kotlin.serialization.deserialization.getName
import org.jetbrains.kotlin.storage.StorageManager
class KonanPackageFragment(
val fqNameString: String,
val reader: KonanLibraryReader,
storageManager: StorageManager,
module: ModuleDescriptor
) : DeserializedPackageFragment(FqName(fqNameString), storageManager, module) {
lateinit var components: DeserializationComponents
override fun initialize(components: DeserializationComponents) {
this.components = components
}
// The proto field is lazy so that we can load only needed
// packages from the library.
private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy {
parsePackageFragment(reader.packageMetadata(fqNameString))
}
val proto: KonanProtoBuf.LinkDataPackageFragment
get() = protoForNames.also { reader.markPackageAccessed(fqNameString) }
private val nameResolver by lazy {
NameResolverImpl(protoForNames.stringTable, protoForNames.nameTable)
}
override val classDataFinder by lazy {
KonanClassDataFinder(proto, nameResolver)
}
private val memberScope_ by lazy {
/* TODO: we fake proto binary versioning for now. */
DeserializedPackageMemberScope(
this,
proto.getPackage(),
nameResolver,
KonanMetadataVersion.INSTANCE,
/* containerSource = */ null,
components) { loadClassNames() }
}
override fun getMemberScope(): DeserializedPackageMemberScope = memberScope_
private val classifierNames: Set<Name> by lazy {
val result = mutableSetOf<Name>()
result.addAll(loadClassNames())
protoForNames.getPackage().typeAliasList.mapTo(result) { nameResolver.getName(it.name) }
result
}
fun hasTopLevelClassifier(name: Name): Boolean = name in classifierNames
private fun loadClassNames(): Collection<Name> {
val classNameList = protoForNames.classes.classNameList
val names = classNameList.mapNotNull {
val classId = nameResolver.getClassId(it)
val shortName = classId.shortClassName
if (!classId.isNestedClass) shortName else null
}
return names
}
}
@@ -0,0 +1,19 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
object KonanSerializerProtocol : SerializerExtensionProtocol(
ExtensionRegistryLite.newInstance().apply { KonanProtoBuf.registerAllExtensions(this) },
KonanProtoBuf.packageFqName,
KonanProtoBuf.constructorAnnotation,
KonanProtoBuf.classAnnotation,
KonanProtoBuf.functionAnnotation,
KonanProtoBuf.propertyAnnotation,
KonanProtoBuf.enumEntryAnnotation,
KonanProtoBuf.compileTimeValue,
KonanProtoBuf.parameterAnnotation,
KonanProtoBuf.typeAnnotation,
KonanProtoBuf.typeParameterAnnotation
)
@@ -0,0 +1,14 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeDeserializer
import org.jetbrains.kotlin.types.SimpleType
object NullFlexibleTypeDeserializer : FlexibleTypeDeserializer {
override fun create(
proto: ProtoBuf.Type,
flexibleId: String,
lowerBound: SimpleType,
upperBound: SimpleType) = error("Illegal use of flexible type deserializer.")
}
@@ -0,0 +1,82 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.descriptors.ClassifierDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
import org.jetbrains.kotlin.incremental.components.LookupLocation
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.resolve.scopes.MemberScope
import org.jetbrains.kotlin.resolve.scopes.MemberScopeImpl
import org.jetbrains.kotlin.utils.Printer
import org.jetbrains.kotlin.utils.addToStdlib.firstNotNullResult
/**
* The package fragment to export forward declarations from interop package namespace, i.e.
* redirect "$pkg.$name" to e.g. "cnames.structs.$name".
*/
class ExportedForwardDeclarationsPackageFragmentDescriptor(
module: ModuleDescriptor,
fqName: FqName,
declarations: List<FqName>
) : PackageFragmentDescriptorImpl(module, fqName) {
private val memberScope = object : MemberScopeImpl() {
private val nameToFqName = declarations.map { it.shortName() to it }.toMap()
override fun getContributedClassifier(name: Name, location: LookupLocation): ClassifierDescriptor? {
val declFqName = nameToFqName[name] ?: return null
val packageView = module.getPackage(declFqName.parent())
return packageView.memberScope.getContributedClassifier(name, location)
}
override fun printScopeStructure(p: Printer) {
p.println(this::class.java.simpleName, " {")
p.pushIndent()
p.println("declarations = $declarations")
p.popIndent()
p.println("}")
}
}
override fun getMemberScope() = memberScope
}
/**
* The package fragment that redirects all requests for classifier lookup to its targets.
*/
class ClassifierAliasingPackageFragmentDescriptor(
targets: List<KonanPackageFragment>,
module: ModuleDescriptor,
fqName: FqName
) : PackageFragmentDescriptorImpl(module, fqName) {
private val memberScope = object : MemberScopeImpl() {
override fun getContributedClassifier(name: Name, location: LookupLocation) =
targets.firstNotNullResult {
if (it.hasTopLevelClassifier(name)) {
it.getMemberScope().getContributedClassifier(name, location)
} else {
null
}
}
override fun printScopeStructure(p: Printer) {
p.println(this::class.java.simpleName, " {")
p.pushIndent()
p.println("targets = $targets")
p.popIndent()
p.println("}")
}
}
override fun getMemberScope(): MemberScope = memberScope
}