KLIB reader: separate library reading from dependency resolving (#1925)

This commit is contained in:
Dmitriy Dolovov
2018-08-24 10:29:13 +03:00
committed by GitHub
parent 59eae5e04b
commit 076f6a7c99
22 changed files with 421 additions and 316 deletions
@@ -4,6 +4,7 @@ import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.storage.StorageManager
@@ -32,6 +33,6 @@ fun createKonanModuleDescriptor(
capabilities = mapOf(KonanModuleOrigin.CAPABILITY to origin)
)
private val STDLIB_MODULE_NAME = Name.special("<stdlib>")
private val STDLIB_MODULE_NAME = Name.special("<$KONAN_STDLIB_NAME>")
fun ModuleDescriptor.isKonanStdlib() = name == STDLIB_MODULE_NAME
@@ -11,8 +11,8 @@ sealed class KonanModuleOrigin {
sealed class CompiledKonanModuleOrigin: KonanModuleOrigin()
// FIXME: ddol: replace `Any` by `KonanLibraryReader` when ready
class DeserializedKonanModuleOrigin(val reader: Any) : CompiledKonanModuleOrigin()
// FIXME(ddol): replace `Any` by `KonanLibrary` when ready
class DeserializedKonanModuleOrigin(val library: Any) : CompiledKonanModuleOrigin()
object CurrentKonanModuleOrigin: CompiledKonanModuleOrigin()
@@ -8,7 +8,7 @@ package org.jetbrains.kotlin.metadata.konan;
// in the same namespace of renamed org.jetbrains.kotlin.protobuf packages.
// In case we merge to the main Kotlin workspace the scheme will be simplified.
// FIXME: ddol: fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
// FIXME(ddol): fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
import "metadata.proto1";
option java_outer_classname = "KonanProtoBuf";
@@ -14,11 +14,17 @@ const val KLIB_PROPERTY_PACKAGE = "package"
const val KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS = "exportForwardDeclarations"
const val KLIB_PROPERTY_INCLUDED_HEADERS = "includedHeaders"
/**
* An abstraction for getting access to the information stored inside of Kotlin/Native library.
*/
interface KonanLibrary {
val libraryName: String
val libraryFile: File
// whether this library is default (provided by Kotlin/Native distribution)
val isDefault: Boolean
// properties:
val manifestProperties: Properties
val abiVersion: String
@@ -33,12 +39,6 @@ interface KonanLibrary {
val dataFlowGraph: ByteArray?
val moduleHeaderData: ByteArray
fun packageMetadata(fqName: String): ByteArray
// FIXME: ddol: to be refactored into some global resolution context
val isDefaultLibrary: Boolean get() = false
val isNeededForLink: Boolean get() = true
val resolvedDependencies: MutableList<KonanLibrary>
fun markPackageAccessed(fqName: String)
}
val KonanLibrary.uniqueName
@@ -55,10 +55,11 @@ val KonanLibrary.packageFqName
val KonanLibrary.exportForwardDeclarations
get() = manifestProperties.getProperty(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS)
.split(' ')
.split(' ').asSequence()
.map { it.trim() }
.filter { it.isNotEmpty() }
.map { FqName(it) }
.toList()
val KonanLibrary.includedHeaders
get() = manifestProperties.getProperty(KLIB_PROPERTY_INCLUDED_HEADERS).split(' ')
@@ -3,9 +3,6 @@ package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
// This scheme describes the Kotlin/Native Library (klib) layout.
interface KonanLibraryLayout {
@@ -5,6 +5,8 @@ import org.jetbrains.kotlin.konan.library.impl.DefaultMetadataReaderImpl
import org.jetbrains.kotlin.konan.library.impl.KonanLibraryImpl
import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryChecks
import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryRoot
import org.jetbrains.kotlin.konan.library.resolver.KonanLibraryResolver
import org.jetbrains.kotlin.konan.library.resolver.impl.KonanLibraryResolverImpl
import org.jetbrains.kotlin.konan.target.KonanTarget
fun File.unpackZippedKonanLibraryTo(newDir: File) {
@@ -23,10 +25,13 @@ fun File.unpackZippedKonanLibraryTo(newDir: File) {
check(newDir.exists) { "Could not unpack $this as $newDir." }
}
fun createKonanLibraryReader(
fun createKonanLibrary(
libraryFile: File,
currentAbiVersion: Int,
target: KonanTarget? = null,
isDefaultLibrary: Boolean = false,
isDefault: Boolean = false,
metadataReader: MetadataReader = DefaultMetadataReaderImpl
): KonanLibrary = KonanLibraryImpl(libraryFile, currentAbiVersion, target, isDefaultLibrary, metadataReader)
): KonanLibrary = KonanLibraryImpl(libraryFile, currentAbiVersion, target, isDefault, metadataReader)
fun SearchPathResolverWithTarget.libraryResolver(abiVersion: Int): KonanLibraryResolver =
KonanLibraryResolverImpl(this, abiVersion)
@@ -11,13 +11,12 @@ import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
import org.jetbrains.kotlin.konan.util.substitute
import org.jetbrains.kotlin.serialization.konan.emptyPackages
internal class KonanLibraryImpl(
override val libraryFile: File,
private val currentAbiVersion: Int,
internal val target: KonanTarget?,
override val isDefaultLibrary: Boolean,
override val isDefault: Boolean,
private val metadataReader: MetadataReader
) : KonanLibrary {
@@ -63,17 +62,5 @@ internal class KonanLibraryImpl(
override fun packageMetadata(fqName: String) = metadataReader.loadSerializedPackageFragment(inPlace, fqName)
override var isNeededForLink: Boolean = false
private set
override val resolvedDependencies = mutableListOf<KonanLibrary>()
override fun markPackageAccessed(fqName: String) {
if (!isNeededForLink // fast path
&& !emptyPackages.contains(fqName)) {
isNeededForLink = true
}
}
private val emptyPackages by lazy { emptyPackages(moduleHeaderData) }
override fun toString() = "$libraryName[default=$isDefault]"
}
@@ -10,7 +10,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
private class ZippedKonanLibraryLayout(val klibFile: File, override val target: KonanTarget? = null): KonanLibraryLayout {
private class ZippedKonanLibraryLayout(val klibFile: File, override val target: KonanTarget?): KonanLibraryLayout {
init { zippedKonanLibraryChecks(klibFile) }
@@ -29,7 +29,7 @@ internal fun zippedKonanLibraryChecks(klibFile: File) {
internal fun zippedKonanLibraryRoot(klibFile: File) = klibFile.asZipRoot
private class UnzippedKonanLibraryLayout(override val libDir: File, override val target: KonanTarget? = null): KonanLibraryLayout {
private class UnzippedKonanLibraryLayout(override val libDir: File, override val target: KonanTarget?): KonanLibraryLayout {
override val libraryName = libDir.path
}
@@ -0,0 +1,59 @@
package org.jetbrains.kotlin.konan.library.resolver
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.SearchPathResolverWithTarget
typealias DuplicatedLibraryLogger = (String) -> Unit
interface KonanLibraryResolver {
val searchPathResolver: SearchPathResolverWithTarget
val abiVersion: Int
/**
* Given the list of Kotlin/Native library names, ABI version and other parameters
* resolves libraries and evaluates dependencies between them.
*/
fun resolveWithDependencies(
libraryNames: List<String>,
noStdLib: Boolean = false,
noDefaultLibs: Boolean = false,
logger: DuplicatedLibraryLogger? = null
): KonanLibraryResolveResult
}
interface KonanLibraryResolveResult {
fun filterRoots(predicate: (KonanResolvedLibrary) -> Boolean): KonanLibraryResolveResult
fun getFullList(order: LibraryOrder? = null): List<KonanLibrary>
fun forEach(action: (KonanLibrary, PackageAccessedHandler) -> Unit)
}
typealias LibraryOrder = (Iterable<KonanResolvedLibrary>) -> List<KonanResolvedLibrary>
val TopologicalLibraryOrder: LibraryOrder = { input ->
val sorted = mutableListOf<KonanResolvedLibrary>()
val visited = mutableSetOf<KonanResolvedLibrary>()
val tempMarks = mutableSetOf<KonanResolvedLibrary>()
fun visit(node: KonanResolvedLibrary, result: MutableList<KonanResolvedLibrary>) {
if (visited.contains(node)) return
if (tempMarks.contains(node)) error("Cyclic dependency in library graph.")
tempMarks.add(node)
node.resolvedDependencies.forEach {
visit(it, result)
}
visited.add(node)
result += node
}
input.forEach next@{
if (visited.contains(it)) return@next
visit(it, sorted)
}
sorted
}
@@ -0,0 +1,27 @@
package org.jetbrains.kotlin.konan.library.resolver
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.name.FqName
interface PackageAccessedHandler {
fun markPackageAccessed(fqName: FqName)
}
/**
* A [KonanLibrary] wrapper that is used for resolving library's dependencies.
*/
interface KonanResolvedLibrary: PackageAccessedHandler {
// the library itself
val library: KonanLibrary
// dependencies on other libraries
val resolvedDependencies: List<KonanResolvedLibrary>
// if it's needed to linker
val isNeededForLink: Boolean
// is provided by the distribution
val isDefault: Boolean
}
@@ -0,0 +1,137 @@
package org.jetbrains.kotlin.konan.library.resolver.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.SearchPathResolverWithTarget
import org.jetbrains.kotlin.konan.library.createKonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.*
import org.jetbrains.kotlin.konan.library.unresolvedDependencies
internal class KonanLibraryResolverImpl(
override val searchPathResolver: SearchPathResolverWithTarget,
override val abiVersion: Int
): KonanLibraryResolver {
override fun resolveWithDependencies(
libraryNames: List<String>,
noStdLib: Boolean,
noDefaultLibs: Boolean,
logger: DuplicatedLibraryLogger?
) = findLibraries(libraryNames, noStdLib, noDefaultLibs)
.leaveDistinct(logger)
.resolveDependencies()
/**
* Returns the list of libraries based on [libraryNames], [noStdLib] and [noDefaultLibs] criteria.
*
* This method does not return any libraries that might be available via transitive dependencies
* from the original library set (root set).
*/
private fun findLibraries(
libraryNames: List<String>,
noStdLib: Boolean,
noDefaultLibs: Boolean
): List<KonanLibrary> {
val userProvidedLibraries = libraryNames.asSequence()
.map { searchPathResolver.resolve(it) }
.map { createKonanLibrary(it, abiVersion, searchPathResolver.target) }
.toList()
val defaultLibraries = searchPathResolver.defaultLinks(noStdLib, noDefaultLibs).map {
createKonanLibrary(it, abiVersion, searchPathResolver.target, isDefault = true)
}
// Make sure the user provided ones appear first, so that
// they have precedence over defaults when duplicates are eliminated.
return userProvidedLibraries + defaultLibraries
}
/**
* Leaves only distinct libraries (by absolute path), warns on duplicated paths.
*/
private fun List<KonanLibrary>.leaveDistinct(logger: DuplicatedLibraryLogger?) =
this.groupBy { it.libraryFile.absolutePath }.let { groupedByAbsolutePath ->
warnOnLibraryDuplicates(groupedByAbsolutePath.filter { it.value.size > 1 }.keys, logger)
groupedByAbsolutePath.map { it.value.first() }
}
private fun warnOnLibraryDuplicates(duplicatedPaths: Iterable<String>, logger: DuplicatedLibraryLogger? ) {
if (logger == null) return
duplicatedPaths.forEach { logger("library included more than once: $it") }
}
/**
* Given the list of root libraries does the following:
*
* 1. Evaluates other libraries that are available via transitive dependencies.
* 2. Wraps each [KonanLibrary] into a [KonanResolvedLibrary] with information about dependencies on other libraries.
* 3. Creates resulting [KonanLibraryResolveResult] object.
*/
private fun List<KonanLibrary>.resolveDependencies(): KonanLibraryResolveResult {
val rootLibraries = this.map { KonanResolvedLibraryImpl(it) }
// as far as the list of root libraries is known from the very beginning, the result can be
// constructed from the very beginning as well
val result = KonanLibraryResolverResultImpl(rootLibraries)
val cache = mutableMapOf<File, KonanResolvedLibrary>()
cache.putAll(rootLibraries.map { it.library.libraryFile.absoluteFile to it })
var newDependencies = rootLibraries
do {
newDependencies = newDependencies.map { library: KonanResolvedLibraryImpl ->
library.library.unresolvedDependencies.asSequence()
.map { searchPathResolver.resolve(it).absoluteFile }
.mapNotNull {
if (it in cache) {
library.addDependency(cache[it]!!)
null
} else {
val newLibrary = KonanResolvedLibraryImpl(createKonanLibrary(it, abiVersion, searchPathResolver.target))
cache[it] = newLibrary
library.addDependency(newLibrary)
newLibrary
}
}
.toList()
}.flatten()
} while (newDependencies.isNotEmpty())
return result
}
}
internal class KonanLibraryResolverResultImpl(
private val roots: List<KonanResolvedLibrary>
): KonanLibraryResolveResult {
private val all: List<KonanResolvedLibrary> by lazy {
val result = mutableSetOf<KonanResolvedLibrary>().also { it.addAll(roots) }
var newDependencies = result.toList()
do {
newDependencies = newDependencies
.map { it -> it.resolvedDependencies }.flatten()
.filter { it !in result }
result.addAll(newDependencies)
} while (newDependencies.isNotEmpty())
result.toList()
}
override fun filterRoots(predicate: (KonanResolvedLibrary) -> Boolean) =
KonanLibraryResolverResultImpl(roots.filter(predicate))
override fun getFullList(order: LibraryOrder?) = (order?.invoke(all) ?: all).asPlain()
override fun forEach(action: (KonanLibrary, PackageAccessedHandler) -> Unit) {
all.forEach { action(it.library, it) }
}
private fun List<KonanResolvedLibrary>.asPlain() = map { it.library }
override fun toString() = "roots=$roots, all=$all"
}
@@ -0,0 +1,34 @@
package org.jetbrains.kotlin.konan.library.resolver.impl
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.KonanResolvedLibrary
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.konan.emptyPackages
internal class KonanResolvedLibraryImpl(
override val library: KonanLibrary
): KonanResolvedLibrary {
private val _resolvedDependencies = mutableListOf<KonanResolvedLibrary>()
private val _emptyPackages by lazy { emptyPackages(library.moduleHeaderData) }
override val resolvedDependencies: List<KonanResolvedLibrary>
get() = _resolvedDependencies
internal fun addDependency(resolvedLibrary: KonanResolvedLibrary) = _resolvedDependencies.add(resolvedLibrary)
override var isNeededForLink: Boolean = false
private set
override val isDefault: Boolean
get() = library.isDefault
override fun markPackageAccessed(fqName: FqName) {
if (!isNeededForLink // fast path
&& !_emptyPackages.contains(fqName.asString())) {
isNeededForLink = true
}
}
override fun toString() = "library=$library, dependsOn=${_resolvedDependencies.joinToString { it.library.toString() }}"
}
@@ -11,18 +11,21 @@ import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.exportForwardDeclarations
import org.jetbrains.kotlin.konan.library.isInterop
import org.jetbrains.kotlin.konan.library.packageFqName
import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager
// FIXME: ddol: this is a temporary solution, to be refactored into some global resolution context
// FIXME(ddol): this is a temporary solution, to be refactored into some global resolution context
interface KonanModuleDescriptorFactory {
fun createModuleDescriptor(
libraryReader: KonanLibrary,
library: KonanLibrary,
specifics: LanguageVersionSettings,
packageAccessedHandler: PackageAccessedHandler? = null,
storageManager: StorageManager = LockBasedStorageManager()
): ModuleDescriptor
}
@@ -30,24 +33,26 @@ interface KonanModuleDescriptorFactory {
object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
override fun createModuleDescriptor(
libraryReader: KonanLibrary,
library: KonanLibrary,
specifics: LanguageVersionSettings,
packageAccessedHandler: PackageAccessedHandler?,
storageManager: StorageManager
): ModuleDescriptorImpl {
val libraryProto = parseModuleHeader(libraryReader.moduleHeaderData)
val libraryProto = parseModuleHeader(library.moduleHeaderData)
val moduleName = libraryProto.moduleName
val moduleDescriptor = createKonanModuleDescriptor(
Name.special(moduleName),
storageManager,
origin = DeserializedKonanModuleOrigin(libraryReader)
origin = DeserializedKonanModuleOrigin(library)
)
val deserializationConfiguration = CompilerDeserializationConfiguration(specifics)
val provider = createPackageFragmentProvider(
libraryReader,
library,
packageAccessedHandler,
libraryProto.packageFragmentNameList,
storageManager,
moduleDescriptor,
@@ -59,7 +64,8 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
}
private fun createPackageFragmentProvider(
libraryReader: KonanLibrary,
library: KonanLibrary,
packageAccessedHandler: PackageAccessedHandler?,
fragmentNames: List<String>,
storageManager: StorageManager,
moduleDescriptor: ModuleDescriptor,
@@ -67,11 +73,11 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
): PackageFragmentProvider {
val deserializedPackageFragments = fragmentNames.map{
KonanPackageFragment(it, libraryReader, storageManager, moduleDescriptor)
KonanPackageFragment(FqName(it), library, packageAccessedHandler, storageManager, moduleDescriptor)
}
val syntheticPackageFragments = getSyntheticPackageFragments(
libraryReader,
library,
moduleDescriptor,
deserializedPackageFragments)
@@ -108,17 +114,17 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
}
private fun getSyntheticPackageFragments(
libraryReader: KonanLibrary,
library: KonanLibrary,
moduleDescriptor: ModuleDescriptor,
konanPackageFragments: List<KonanPackageFragment>
): List<PackageFragmentDescriptor> {
if (!libraryReader.isInterop) return emptyList()
if (!library.isInterop) return emptyList()
val packageFqName = libraryReader.packageFqName
?: error("Inconsistent manifest: interop library ${libraryReader.libraryName} should have `package` specified")
val packageFqName = library.packageFqName
?: error("Inconsistent manifest: interop library ${library.libraryName} should have `package` specified")
val exportForwardDeclarations = libraryReader.exportForwardDeclarations
val exportForwardDeclarations = library.exportForwardDeclarations
val interopPackageFragments = konanPackageFragments.filter { it.fqName == packageFqName }
@@ -2,6 +2,7 @@ package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
import org.jetbrains.kotlin.name.FqName
@@ -14,11 +15,12 @@ import org.jetbrains.kotlin.serialization.deserialization.getName
import org.jetbrains.kotlin.storage.StorageManager
class KonanPackageFragment(
val fqNameString: String,
val reader: KonanLibrary,
fqName: FqName,
private val library: KonanLibrary,
private val packageAccessedHandler: PackageAccessedHandler?,
storageManager: StorageManager,
module: ModuleDescriptor
) : DeserializedPackageFragment(FqName(fqNameString), storageManager, module) {
) : DeserializedPackageFragment(fqName, storageManager, module) {
lateinit var components: DeserializationComponents
@@ -29,11 +31,11 @@ class KonanPackageFragment(
// The proto field is lazy so that we can load only needed
// packages from the library.
private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy {
parsePackageFragment(reader.packageMetadata(fqNameString))
parsePackageFragment(library.packageMetadata(fqName.asString()))
}
val proto: KonanProtoBuf.LinkDataPackageFragment
get() = protoForNames.also { reader.markPackageAccessed(fqNameString) }
get() = protoForNames.also { packageAccessedHandler?.markPackageAccessed(fqName) }
private val nameResolver by lazy {
NameResolverImpl(protoForNames.stringTable, protoForNames.nameTable)
@@ -43,7 +45,7 @@ class KonanPackageFragment(
KonanClassDataFinder(proto, nameResolver)
}
private val memberScope_ by lazy {
private val _memberScope by lazy {
/* TODO: we fake proto binary versioning for now. */
DeserializedPackageMemberScope(
this,
@@ -54,7 +56,7 @@ class KonanPackageFragment(
components) { loadClassNames() }
}
override fun getMemberScope(): DeserializedPackageMemberScope = memberScope_
override fun getMemberScope(): DeserializedPackageMemberScope = _memberScope
private val classifierNames: Set<Name> by lazy {
val result = mutableSetOf<Name>()