KLIB reader: separate library reading from dependency resolving (#1925)

This commit is contained in:
Dmitriy Dolovov
2018-08-24 10:29:13 +03:00
committed by GitHub
parent 59eae5e04b
commit 076f6a7c99
22 changed files with 421 additions and 316 deletions
@@ -17,9 +17,6 @@
package org.jetbrains.kotlin.backend.konan
import com.intellij.openapi.project.Project
import org.jetbrains.kotlin.backend.konan.library.resolveImmediateLibraries
import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
import org.jetbrains.kotlin.backend.konan.library.withResolvedDependencies
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
@@ -34,6 +31,7 @@ import org.jetbrains.kotlin.konan.TempFiles
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.defaultResolver
import org.jetbrains.kotlin.konan.library.libraryResolver
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.profile
import org.jetbrains.kotlin.serialization.konan.DefaultKonanModuleDescriptorFactory
@@ -91,29 +89,25 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
get() = configuration.getList(KonanConfigKeys.LIBRARY_FILES)
private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES)
private val resolver = defaultResolver(repositories, target, distribution)
private val resolver = defaultResolver(repositories, target, distribution).libraryResolver(currentAbiVersion)
internal val immediateLibraries: List<KonanLibrary> by lazy {
val result = resolver.resolveImmediateLibraries(
internal val resolvedLibraries by lazy {
resolver.resolveWithDependencies(
libraryNames,
target,
currentAbiVersion,
configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS),
{ msg -> configuration.report(STRONG_WARNING, msg) })
resolver.resolveLibrariesRecursive(result, target, currentAbiVersion)
result
noStdLib = configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
noDefaultLibs = configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS) ) { msg ->
configuration.report(STRONG_WARNING, msg) }
}
fun librariesWithDependencies(moduleDescriptor: ModuleDescriptor?): List<KonanLibrary> {
if (moduleDescriptor == null) error("purgeUnneeded() only works correctly after resolve is over, and we have successfully marked package files as needed or not needed.")
return immediateLibraries.purgeUnneeded(this).withResolvedDependencies()
return resolvedLibraries.filterRoots { (!it.isDefault && !this.purgeUserLibs) || it.isNeededForLink }.getFullList()
}
private val loadedDescriptors = loadLibMetadata()
internal lateinit var friends:Set<ModuleDescriptorImpl>
internal lateinit var friends: Set<ModuleDescriptorImpl>
internal val defaultNativeLibraries =
if (produce == CompilerOutputKind.PROGRAM)
@@ -128,24 +122,25 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
fun loadLibMetadata(): List<ModuleDescriptorImpl> {
val allMetadata = mutableListOf<ModuleDescriptorImpl>()
val specifics = configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)!!
val friendLibsSet = configuration.get(KonanConfigKeys.FRIEND_MODULES)?.map { File(it) }?.toSet()
val libraries = immediateLibraries.withResolvedDependencies()
val friendLibsSet = configuration.get(KonanConfigKeys.FRIEND_MODULES)?.map{File(it)}?.toSet()
val allMetadata = mutableListOf<ModuleDescriptorImpl>()
val friends = mutableListOf<ModuleDescriptorImpl>()
for (klib in libraries) {
profile("Loading ${klib.libraryName}") {
resolvedLibraries.forEach { library, packageAccessedHandler ->
profile("Loading ${library.libraryName}") {
// MutableModuleContext needs ModuleDescriptorImpl, rather than ModuleDescriptor.
val moduleDescriptor = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(klib, specifics)
val moduleDescriptor = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(library, specifics, packageAccessedHandler)
allMetadata.add(moduleDescriptor)
friendLibsSet?.apply {
if (contains(klib.libraryFile))
friends.add(moduleDescriptor)
if (contains(library.libraryFile)) friends.add(moduleDescriptor)
}
}
}
this.friends = friends.toSet()
return allMetadata
}
@@ -176,6 +171,3 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
fun CompilerConfiguration.report(priority: CompilerMessageSeverity, message: String)
= this.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(priority, message)
private fun <T: KonanLibrary> List<T>.purgeUnneeded(config: KonanConfig): List<T> =
this.filter{ (!it.isDefaultLibrary && !config.purgeUserLibs) || it.isNeededForLink }
@@ -19,6 +19,8 @@ package org.jetbrains.kotlin.backend.konan.library
import llvm.LLVMModuleRef
import org.jetbrains.kotlin.konan.library.KonanLibrary
const val KLIB_CURRENT_ABI_VERSION = 1
interface KonanLibraryWriter {
fun addLinkData(linkData: LinkData)
fun addNativeBitcode(library: String)
@@ -1,127 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.SearchPathResolver
import org.jetbrains.kotlin.konan.library.createKonanLibraryReader
import org.jetbrains.kotlin.konan.library.unresolvedDependencies
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KONAN_CURRENT_ABI_VERSION = 1
/**
* Returns the list of [KonanLibrary]s given the list of user provided [libraryNames] along with
* other parameters: [target], [abiVersion], [noStdLib], [noDefaultLibs].
*/
fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
target: KonanTarget,
abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
noStdLib: Boolean = false,
noDefaultLibs: Boolean = false,
logger: ((String) -> Unit)?): List<KonanLibrary> {
val userProvidedLibraries = libraryNames
.map { resolve(it) }
.map{ createKonanLibraryReader(it, abiVersion, target) }
val defaultLibraries = defaultLinks(noStdLib = noStdLib, noDefaultLibs = noDefaultLibs).map {
createKonanLibraryReader(it, abiVersion, target, isDefaultLibrary = true)
}
// Make sure the user provided ones appear first, so that
// they have precedence over defaults when duplicates are eliminated.
val resolvedLibraries = userProvidedLibraries + defaultLibraries
warnOnLibraryDuplicates(resolvedLibraries.map { it.libraryFile }, logger)
return resolvedLibraries.distinctBy { it.libraryFile.absolutePath }
}
private fun warnOnLibraryDuplicates(resolvedLibraries: List<File>, logger: ((String) -> Unit)? ) {
if (logger == null) return
val duplicates = resolvedLibraries.groupBy { it.absolutePath } .values.filter { it.size > 1 }
duplicates.forEach {
logger("library included more than once: ${it.first().absolutePath}")
}
}
/**
* For each of the given [immediateLibraries] fills in `resolvedDependencies` field with the
* [KonanLibrary]s the library !!directly!! depends on.
*/
fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanLibrary>,
target: KonanTarget,
abiVersion: Int) {
val cache = mutableMapOf<File, KonanLibrary>()
cache.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it })
var newDependencies = cache.values.toList()
do {
newDependencies = newDependencies.map { library: KonanLibrary ->
library.unresolvedDependencies
.map { resolve(it).absoluteFile }
.mapNotNull {
if (it in cache) {
library.resolvedDependencies.add(cache[it]!!)
null
} else {
val reader = createKonanLibraryReader(it, abiVersion, target)
cache[it] = reader
library.resolvedDependencies.add(reader)
reader
}
}
} .flatten()
} while (newDependencies.isNotEmpty())
}
/**
* For the given list of [KonanLibrary]s returns the list of [KonanLibrary]s
* that includes the same libraries plus all their (transitive) dependencies.
*/
fun List<KonanLibrary>.withResolvedDependencies(): List<KonanLibrary> {
val result = mutableSetOf<KonanLibrary>()
result.addAll(this)
var newDependencies = result.toList()
do {
newDependencies = newDependencies
.map { it -> it.resolvedDependencies }.flatten()
.filter { it !in result }
result.addAll(newDependencies)
} while (newDependencies.isNotEmpty())
return result.toList()
}
fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List<String>,
target: KonanTarget,
abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
noStdLib: Boolean = false,
noDefaultLibs: Boolean = false): List<KonanLibrary> {
val immediateLibraries = resolveImmediateLibraries(
libraryNames = libraryNames,
target = target,
abiVersion = abiVersion,
noStdLib = noStdLib,
noDefaultLibs = noDefaultLibs,
logger = null
)
resolveLibrariesRecursive(immediateLibraries, target, abiVersion)
return immediateLibraries.withResolvedDependencies()
}
@@ -25,14 +25,14 @@ import org.jetbrains.kotlin.backend.konan.Context
import org.jetbrains.kotlin.backend.konan.descriptors.findPackage
import org.jetbrains.kotlin.backend.konan.hash.GlobalHash
import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.withResolvedDependencies
import org.jetbrains.kotlin.descriptors.konan.CompiledKonanModuleOrigin
import org.jetbrains.kotlin.descriptors.konan.CurrentKonanModuleOrigin
import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
import org.jetbrains.kotlin.ir.declarations.IrField
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.TopologicalLibraryOrder
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
@@ -330,47 +330,23 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
private val allLibraries = context.librariesWithDependencies.toSet()
override fun add(origin: CompiledKonanModuleOrigin) {
val reader = when (origin) {
val library = when (origin) {
CurrentKonanModuleOrigin -> return
is DeserializedKonanModuleOrigin -> origin.reader as KonanLibrary
is DeserializedKonanModuleOrigin -> origin.library as KonanLibrary
}
if (reader !in allLibraries) {
error("$reader (${reader.libraryName}) is used but not requested")
if (library !in allLibraries) {
error("Library (${library.libraryName}) is used but not requested.\nRequested libraries: ${allLibraries.joinToString { it.libraryName }}")
}
usedLibraries.add(reader)
usedLibraries.add(library)
}
}
val librariesToLink: List<KonanLibrary> by lazy {
context.config.immediateLibraries
.filter { (!it.isDefaultLibrary && !context.config.purgeUserLibs) || it in usedLibraries }
.withResolvedDependencies()
.topoSort()
}
private fun List<KonanLibrary>.topoSort(): List<KonanLibrary> {
var sorted = mutableListOf<KonanLibrary>()
val visited = mutableSetOf<KonanLibrary>()
val tempMarks = mutableSetOf<KonanLibrary>()
fun visit(node: KonanLibrary, result: MutableList<KonanLibrary>) {
if (visited.contains(node)) return
if (tempMarks.contains(node)) error("Cyclic dependency in library graph.")
tempMarks.add(node)
node.resolvedDependencies.forEach {
visit(it, result)
}
visited.add(node)
result += node
}
this.forEach next@{
if (visited.contains(it)) return@next
visit(it, sorted)
}
return sorted
val librariesToLink: List<KonanLibrary> by lazy {
context.config.resolvedLibraries
.filterRoots { (!it.isDefault && !context.config.purgeUserLibs) || it.library in usedLibraries }
.getFullList(TopologicalLibraryOrder)
}
val librariesForLibraryManifest: List<KonanLibrary> get() {
@@ -2,7 +2,7 @@
syntax = "proto2";
package org.jetbrains.kotlin.metadata;
// FIXME: ddol: fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
// FIXME(ddol): fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
import "org/jetbrains/kotlin/backend/konan/serialization/metadata.proto1";
option java_outer_classname = "KonanIr";
option optimize_for = LITE_RUNTIME;
@@ -4,6 +4,7 @@ import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.storage.StorageManager
@@ -32,6 +33,6 @@ fun createKonanModuleDescriptor(
capabilities = mapOf(KonanModuleOrigin.CAPABILITY to origin)
)
private val STDLIB_MODULE_NAME = Name.special("<stdlib>")
private val STDLIB_MODULE_NAME = Name.special("<$KONAN_STDLIB_NAME>")
fun ModuleDescriptor.isKonanStdlib() = name == STDLIB_MODULE_NAME
@@ -11,8 +11,8 @@ sealed class KonanModuleOrigin {
sealed class CompiledKonanModuleOrigin: KonanModuleOrigin()
// FIXME: ddol: replace `Any` by `KonanLibraryReader` when ready
class DeserializedKonanModuleOrigin(val reader: Any) : CompiledKonanModuleOrigin()
// FIXME(ddol): replace `Any` by `KonanLibrary` when ready
class DeserializedKonanModuleOrigin(val library: Any) : CompiledKonanModuleOrigin()
object CurrentKonanModuleOrigin: CompiledKonanModuleOrigin()
@@ -8,7 +8,7 @@ package org.jetbrains.kotlin.metadata.konan;
// in the same namespace of renamed org.jetbrains.kotlin.protobuf packages.
// In case we merge to the main Kotlin workspace the scheme will be simplified.
// FIXME: ddol: fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
// FIXME(ddol): fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
import "metadata.proto1";
option java_outer_classname = "KonanProtoBuf";
@@ -14,11 +14,17 @@ const val KLIB_PROPERTY_PACKAGE = "package"
const val KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS = "exportForwardDeclarations"
const val KLIB_PROPERTY_INCLUDED_HEADERS = "includedHeaders"
/**
* An abstraction for getting access to the information stored inside of Kotlin/Native library.
*/
interface KonanLibrary {
val libraryName: String
val libraryFile: File
// whether this library is default (provided by Kotlin/Native distribution)
val isDefault: Boolean
// properties:
val manifestProperties: Properties
val abiVersion: String
@@ -33,12 +39,6 @@ interface KonanLibrary {
val dataFlowGraph: ByteArray?
val moduleHeaderData: ByteArray
fun packageMetadata(fqName: String): ByteArray
// FIXME: ddol: to be refactored into some global resolution context
val isDefaultLibrary: Boolean get() = false
val isNeededForLink: Boolean get() = true
val resolvedDependencies: MutableList<KonanLibrary>
fun markPackageAccessed(fqName: String)
}
val KonanLibrary.uniqueName
@@ -55,10 +55,11 @@ val KonanLibrary.packageFqName
val KonanLibrary.exportForwardDeclarations
get() = manifestProperties.getProperty(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS)
.split(' ')
.split(' ').asSequence()
.map { it.trim() }
.filter { it.isNotEmpty() }
.map { FqName(it) }
.toList()
val KonanLibrary.includedHeaders
get() = manifestProperties.getProperty(KLIB_PROPERTY_INCLUDED_HEADERS).split(' ')
@@ -3,9 +3,6 @@ package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
// This scheme describes the Kotlin/Native Library (klib) layout.
interface KonanLibraryLayout {
@@ -5,6 +5,8 @@ import org.jetbrains.kotlin.konan.library.impl.DefaultMetadataReaderImpl
import org.jetbrains.kotlin.konan.library.impl.KonanLibraryImpl
import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryChecks
import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryRoot
import org.jetbrains.kotlin.konan.library.resolver.KonanLibraryResolver
import org.jetbrains.kotlin.konan.library.resolver.impl.KonanLibraryResolverImpl
import org.jetbrains.kotlin.konan.target.KonanTarget
fun File.unpackZippedKonanLibraryTo(newDir: File) {
@@ -23,10 +25,13 @@ fun File.unpackZippedKonanLibraryTo(newDir: File) {
check(newDir.exists) { "Could not unpack $this as $newDir." }
}
fun createKonanLibraryReader(
fun createKonanLibrary(
libraryFile: File,
currentAbiVersion: Int,
target: KonanTarget? = null,
isDefaultLibrary: Boolean = false,
isDefault: Boolean = false,
metadataReader: MetadataReader = DefaultMetadataReaderImpl
): KonanLibrary = KonanLibraryImpl(libraryFile, currentAbiVersion, target, isDefaultLibrary, metadataReader)
): KonanLibrary = KonanLibraryImpl(libraryFile, currentAbiVersion, target, isDefault, metadataReader)
fun SearchPathResolverWithTarget.libraryResolver(abiVersion: Int): KonanLibraryResolver =
KonanLibraryResolverImpl(this, abiVersion)
@@ -11,13 +11,12 @@ import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
import org.jetbrains.kotlin.konan.util.substitute
import org.jetbrains.kotlin.serialization.konan.emptyPackages
internal class KonanLibraryImpl(
override val libraryFile: File,
private val currentAbiVersion: Int,
internal val target: KonanTarget?,
override val isDefaultLibrary: Boolean,
override val isDefault: Boolean,
private val metadataReader: MetadataReader
) : KonanLibrary {
@@ -63,17 +62,5 @@ internal class KonanLibraryImpl(
override fun packageMetadata(fqName: String) = metadataReader.loadSerializedPackageFragment(inPlace, fqName)
override var isNeededForLink: Boolean = false
private set
override val resolvedDependencies = mutableListOf<KonanLibrary>()
override fun markPackageAccessed(fqName: String) {
if (!isNeededForLink // fast path
&& !emptyPackages.contains(fqName)) {
isNeededForLink = true
}
}
private val emptyPackages by lazy { emptyPackages(moduleHeaderData) }
override fun toString() = "$libraryName[default=$isDefault]"
}
@@ -10,7 +10,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
private class ZippedKonanLibraryLayout(val klibFile: File, override val target: KonanTarget? = null): KonanLibraryLayout {
private class ZippedKonanLibraryLayout(val klibFile: File, override val target: KonanTarget?): KonanLibraryLayout {
init { zippedKonanLibraryChecks(klibFile) }
@@ -29,7 +29,7 @@ internal fun zippedKonanLibraryChecks(klibFile: File) {
internal fun zippedKonanLibraryRoot(klibFile: File) = klibFile.asZipRoot
private class UnzippedKonanLibraryLayout(override val libDir: File, override val target: KonanTarget? = null): KonanLibraryLayout {
private class UnzippedKonanLibraryLayout(override val libDir: File, override val target: KonanTarget?): KonanLibraryLayout {
override val libraryName = libDir.path
}
@@ -0,0 +1,59 @@
package org.jetbrains.kotlin.konan.library.resolver
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.SearchPathResolverWithTarget
typealias DuplicatedLibraryLogger = (String) -> Unit
interface KonanLibraryResolver {
val searchPathResolver: SearchPathResolverWithTarget
val abiVersion: Int
/**
* Given the list of Kotlin/Native library names, ABI version and other parameters
* resolves libraries and evaluates dependencies between them.
*/
fun resolveWithDependencies(
libraryNames: List<String>,
noStdLib: Boolean = false,
noDefaultLibs: Boolean = false,
logger: DuplicatedLibraryLogger? = null
): KonanLibraryResolveResult
}
interface KonanLibraryResolveResult {
fun filterRoots(predicate: (KonanResolvedLibrary) -> Boolean): KonanLibraryResolveResult
fun getFullList(order: LibraryOrder? = null): List<KonanLibrary>
fun forEach(action: (KonanLibrary, PackageAccessedHandler) -> Unit)
}
typealias LibraryOrder = (Iterable<KonanResolvedLibrary>) -> List<KonanResolvedLibrary>
val TopologicalLibraryOrder: LibraryOrder = { input ->
val sorted = mutableListOf<KonanResolvedLibrary>()
val visited = mutableSetOf<KonanResolvedLibrary>()
val tempMarks = mutableSetOf<KonanResolvedLibrary>()
fun visit(node: KonanResolvedLibrary, result: MutableList<KonanResolvedLibrary>) {
if (visited.contains(node)) return
if (tempMarks.contains(node)) error("Cyclic dependency in library graph.")
tempMarks.add(node)
node.resolvedDependencies.forEach {
visit(it, result)
}
visited.add(node)
result += node
}
input.forEach next@{
if (visited.contains(it)) return@next
visit(it, sorted)
}
sorted
}
@@ -0,0 +1,27 @@
package org.jetbrains.kotlin.konan.library.resolver
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.name.FqName
interface PackageAccessedHandler {
fun markPackageAccessed(fqName: FqName)
}
/**
* A [KonanLibrary] wrapper that is used for resolving library's dependencies.
*/
interface KonanResolvedLibrary: PackageAccessedHandler {
// the library itself
val library: KonanLibrary
// dependencies on other libraries
val resolvedDependencies: List<KonanResolvedLibrary>
// if it's needed to linker
val isNeededForLink: Boolean
// is provided by the distribution
val isDefault: Boolean
}
@@ -0,0 +1,137 @@
package org.jetbrains.kotlin.konan.library.resolver.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.SearchPathResolverWithTarget
import org.jetbrains.kotlin.konan.library.createKonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.*
import org.jetbrains.kotlin.konan.library.unresolvedDependencies
internal class KonanLibraryResolverImpl(
override val searchPathResolver: SearchPathResolverWithTarget,
override val abiVersion: Int
): KonanLibraryResolver {
override fun resolveWithDependencies(
libraryNames: List<String>,
noStdLib: Boolean,
noDefaultLibs: Boolean,
logger: DuplicatedLibraryLogger?
) = findLibraries(libraryNames, noStdLib, noDefaultLibs)
.leaveDistinct(logger)
.resolveDependencies()
/**
* Returns the list of libraries based on [libraryNames], [noStdLib] and [noDefaultLibs] criteria.
*
* This method does not return any libraries that might be available via transitive dependencies
* from the original library set (root set).
*/
private fun findLibraries(
libraryNames: List<String>,
noStdLib: Boolean,
noDefaultLibs: Boolean
): List<KonanLibrary> {
val userProvidedLibraries = libraryNames.asSequence()
.map { searchPathResolver.resolve(it) }
.map { createKonanLibrary(it, abiVersion, searchPathResolver.target) }
.toList()
val defaultLibraries = searchPathResolver.defaultLinks(noStdLib, noDefaultLibs).map {
createKonanLibrary(it, abiVersion, searchPathResolver.target, isDefault = true)
}
// Make sure the user provided ones appear first, so that
// they have precedence over defaults when duplicates are eliminated.
return userProvidedLibraries + defaultLibraries
}
/**
* Leaves only distinct libraries (by absolute path), warns on duplicated paths.
*/
private fun List<KonanLibrary>.leaveDistinct(logger: DuplicatedLibraryLogger?) =
this.groupBy { it.libraryFile.absolutePath }.let { groupedByAbsolutePath ->
warnOnLibraryDuplicates(groupedByAbsolutePath.filter { it.value.size > 1 }.keys, logger)
groupedByAbsolutePath.map { it.value.first() }
}
private fun warnOnLibraryDuplicates(duplicatedPaths: Iterable<String>, logger: DuplicatedLibraryLogger? ) {
if (logger == null) return
duplicatedPaths.forEach { logger("library included more than once: $it") }
}
/**
* Given the list of root libraries does the following:
*
* 1. Evaluates other libraries that are available via transitive dependencies.
* 2. Wraps each [KonanLibrary] into a [KonanResolvedLibrary] with information about dependencies on other libraries.
* 3. Creates resulting [KonanLibraryResolveResult] object.
*/
private fun List<KonanLibrary>.resolveDependencies(): KonanLibraryResolveResult {
val rootLibraries = this.map { KonanResolvedLibraryImpl(it) }
// as far as the list of root libraries is known from the very beginning, the result can be
// constructed from the very beginning as well
val result = KonanLibraryResolverResultImpl(rootLibraries)
val cache = mutableMapOf<File, KonanResolvedLibrary>()
cache.putAll(rootLibraries.map { it.library.libraryFile.absoluteFile to it })
var newDependencies = rootLibraries
do {
newDependencies = newDependencies.map { library: KonanResolvedLibraryImpl ->
library.library.unresolvedDependencies.asSequence()
.map { searchPathResolver.resolve(it).absoluteFile }
.mapNotNull {
if (it in cache) {
library.addDependency(cache[it]!!)
null
} else {
val newLibrary = KonanResolvedLibraryImpl(createKonanLibrary(it, abiVersion, searchPathResolver.target))
cache[it] = newLibrary
library.addDependency(newLibrary)
newLibrary
}
}
.toList()
}.flatten()
} while (newDependencies.isNotEmpty())
return result
}
}
internal class KonanLibraryResolverResultImpl(
private val roots: List<KonanResolvedLibrary>
): KonanLibraryResolveResult {
private val all: List<KonanResolvedLibrary> by lazy {
val result = mutableSetOf<KonanResolvedLibrary>().also { it.addAll(roots) }
var newDependencies = result.toList()
do {
newDependencies = newDependencies
.map { it -> it.resolvedDependencies }.flatten()
.filter { it !in result }
result.addAll(newDependencies)
} while (newDependencies.isNotEmpty())
result.toList()
}
override fun filterRoots(predicate: (KonanResolvedLibrary) -> Boolean) =
KonanLibraryResolverResultImpl(roots.filter(predicate))
override fun getFullList(order: LibraryOrder?) = (order?.invoke(all) ?: all).asPlain()
override fun forEach(action: (KonanLibrary, PackageAccessedHandler) -> Unit) {
all.forEach { action(it.library, it) }
}
private fun List<KonanResolvedLibrary>.asPlain() = map { it.library }
override fun toString() = "roots=$roots, all=$all"
}
@@ -0,0 +1,34 @@
package org.jetbrains.kotlin.konan.library.resolver.impl
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.KonanResolvedLibrary
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.konan.emptyPackages
internal class KonanResolvedLibraryImpl(
override val library: KonanLibrary
): KonanResolvedLibrary {
private val _resolvedDependencies = mutableListOf<KonanResolvedLibrary>()
private val _emptyPackages by lazy { emptyPackages(library.moduleHeaderData) }
override val resolvedDependencies: List<KonanResolvedLibrary>
get() = _resolvedDependencies
internal fun addDependency(resolvedLibrary: KonanResolvedLibrary) = _resolvedDependencies.add(resolvedLibrary)
override var isNeededForLink: Boolean = false
private set
override val isDefault: Boolean
get() = library.isDefault
override fun markPackageAccessed(fqName: FqName) {
if (!isNeededForLink // fast path
&& !_emptyPackages.contains(fqName.asString())) {
isNeededForLink = true
}
}
override fun toString() = "library=$library, dependsOn=${_resolvedDependencies.joinToString { it.library.toString() }}"
}
@@ -11,18 +11,21 @@ import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.exportForwardDeclarations
import org.jetbrains.kotlin.konan.library.isInterop
import org.jetbrains.kotlin.konan.library.packageFqName
import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
import org.jetbrains.kotlin.serialization.deserialization.*
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager
// FIXME: ddol: this is a temporary solution, to be refactored into some global resolution context
// FIXME(ddol): this is a temporary solution, to be refactored into some global resolution context
interface KonanModuleDescriptorFactory {
fun createModuleDescriptor(
libraryReader: KonanLibrary,
library: KonanLibrary,
specifics: LanguageVersionSettings,
packageAccessedHandler: PackageAccessedHandler? = null,
storageManager: StorageManager = LockBasedStorageManager()
): ModuleDescriptor
}
@@ -30,24 +33,26 @@ interface KonanModuleDescriptorFactory {
object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
override fun createModuleDescriptor(
libraryReader: KonanLibrary,
library: KonanLibrary,
specifics: LanguageVersionSettings,
packageAccessedHandler: PackageAccessedHandler?,
storageManager: StorageManager
): ModuleDescriptorImpl {
val libraryProto = parseModuleHeader(libraryReader.moduleHeaderData)
val libraryProto = parseModuleHeader(library.moduleHeaderData)
val moduleName = libraryProto.moduleName
val moduleDescriptor = createKonanModuleDescriptor(
Name.special(moduleName),
storageManager,
origin = DeserializedKonanModuleOrigin(libraryReader)
origin = DeserializedKonanModuleOrigin(library)
)
val deserializationConfiguration = CompilerDeserializationConfiguration(specifics)
val provider = createPackageFragmentProvider(
libraryReader,
library,
packageAccessedHandler,
libraryProto.packageFragmentNameList,
storageManager,
moduleDescriptor,
@@ -59,7 +64,8 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
}
private fun createPackageFragmentProvider(
libraryReader: KonanLibrary,
library: KonanLibrary,
packageAccessedHandler: PackageAccessedHandler?,
fragmentNames: List<String>,
storageManager: StorageManager,
moduleDescriptor: ModuleDescriptor,
@@ -67,11 +73,11 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
): PackageFragmentProvider {
val deserializedPackageFragments = fragmentNames.map{
KonanPackageFragment(it, libraryReader, storageManager, moduleDescriptor)
KonanPackageFragment(FqName(it), library, packageAccessedHandler, storageManager, moduleDescriptor)
}
val syntheticPackageFragments = getSyntheticPackageFragments(
libraryReader,
library,
moduleDescriptor,
deserializedPackageFragments)
@@ -108,17 +114,17 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
}
private fun getSyntheticPackageFragments(
libraryReader: KonanLibrary,
library: KonanLibrary,
moduleDescriptor: ModuleDescriptor,
konanPackageFragments: List<KonanPackageFragment>
): List<PackageFragmentDescriptor> {
if (!libraryReader.isInterop) return emptyList()
if (!library.isInterop) return emptyList()
val packageFqName = libraryReader.packageFqName
?: error("Inconsistent manifest: interop library ${libraryReader.libraryName} should have `package` specified")
val packageFqName = library.packageFqName
?: error("Inconsistent manifest: interop library ${library.libraryName} should have `package` specified")
val exportForwardDeclarations = libraryReader.exportForwardDeclarations
val exportForwardDeclarations = library.exportForwardDeclarations
val interopPackageFragments = konanPackageFragments.filter { it.fqName == packageFqName }
@@ -2,6 +2,7 @@ package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
import org.jetbrains.kotlin.name.FqName
@@ -14,11 +15,12 @@ import org.jetbrains.kotlin.serialization.deserialization.getName
import org.jetbrains.kotlin.storage.StorageManager
class KonanPackageFragment(
val fqNameString: String,
val reader: KonanLibrary,
fqName: FqName,
private val library: KonanLibrary,
private val packageAccessedHandler: PackageAccessedHandler?,
storageManager: StorageManager,
module: ModuleDescriptor
) : DeserializedPackageFragment(FqName(fqNameString), storageManager, module) {
) : DeserializedPackageFragment(fqName, storageManager, module) {
lateinit var components: DeserializationComponents
@@ -29,11 +31,11 @@ class KonanPackageFragment(
// The proto field is lazy so that we can load only needed
// packages from the library.
private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy {
parsePackageFragment(reader.packageMetadata(fqNameString))
parsePackageFragment(library.packageMetadata(fqName.asString()))
}
val proto: KonanProtoBuf.LinkDataPackageFragment
get() = protoForNames.also { reader.markPackageAccessed(fqNameString) }
get() = protoForNames.also { packageAccessedHandler?.markPackageAccessed(fqName) }
private val nameResolver by lazy {
NameResolverImpl(protoForNames.stringTable, protoForNames.nameTable)
@@ -43,7 +45,7 @@ class KonanPackageFragment(
KonanClassDataFinder(proto, nameResolver)
}
private val memberScope_ by lazy {
private val _memberScope by lazy {
/* TODO: we fake proto binary versioning for now. */
DeserializedPackageMemberScope(
this,
@@ -54,7 +56,7 @@ class KonanPackageFragment(
components) { loadClassNames() }
}
override fun getMemberScope(): DeserializedPackageMemberScope = memberScope_
override fun getMemberScope(): DeserializedPackageMemberScope = _memberScope
private val classifierNames: Set<Name> by lazy {
val result = mutableSetOf<Name>()
@@ -17,7 +17,7 @@
package org.jetbrains.kotlin.cli.klib
// TODO: Extract `library` package as a shared jar?
import org.jetbrains.kotlin.backend.konan.library.KONAN_CURRENT_ABI_VERSION
import org.jetbrains.kotlin.backend.konan.library.KLIB_CURRENT_ABI_VERSION
import org.jetbrains.kotlin.config.ApiVersion
import org.jetbrains.kotlin.config.LanguageVersion
import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
@@ -25,8 +25,8 @@ import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.descriptors.konan.isKonanStdlib
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION_WITH_DOT
import org.jetbrains.kotlin.konan.library.KonanLibrarySearchPathResolver
import org.jetbrains.kotlin.konan.library.createKonanLibraryReader
import org.jetbrains.kotlin.konan.library.createKonanLibrary
import org.jetbrains.kotlin.konan.library.noTargetResolver
import org.jetbrains.kotlin.konan.library.unpackZippedKonanLibraryTo
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.PlatformManager
@@ -106,7 +106,7 @@ class Library(val name: String, val requestedRepository: String?, val target: St
val repository = requestedRepository?.File() ?: defaultRepository
fun info() {
val library = libraryInRepoOrCurrentDir(repository, name)
val reader = createKonanLibraryReader(library, currentAbiVersion)
val reader = createKonanLibrary(library, currentAbiVersion)
val headerAbiVersion = reader.abiVersion
val moduleName = ModuleDeserializer(reader.moduleHeaderData).moduleName
@@ -144,20 +144,19 @@ class Library(val name: String, val requestedRepository: String?, val target: St
}
fun contents(output: Appendable = out) {
val reader = createKonanLibraryReader(libraryInRepoOrCurrentDir(repository, name), currentAbiVersion)
val library = createKonanLibrary(libraryInRepoOrCurrentDir(repository, name), currentAbiVersion)
val versionSpec = LanguageVersionSettingsImpl(currentLanguageVersion, currentApiVersion)
val module = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(reader, versionSpec)
val module = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(library, versionSpec)
val defaultModules = mutableListOf<ModuleDescriptorImpl>()
if (!module.isKonanStdlib()) {
val resolver = KonanLibrarySearchPathResolver(emptyList(),
target = null,
val resolver = noTargetResolver(
emptyList(),
distributionKlib = Distribution().klib,
localKonanDir = null,
skipCurrentDir = true)
resolver.defaultLinks(false, true)
.mapTo(defaultModules) {
DefaultKonanModuleDescriptorFactory.createModuleDescriptor(
createKonanLibraryReader(it, currentAbiVersion), versionSpec)
createKonanLibrary(it, currentAbiVersion), versionSpec)
}
}
@@ -169,41 +168,17 @@ class Library(val name: String, val requestedRepository: String?, val target: St
}
}
val currentAbiVersion = KONAN_CURRENT_ABI_VERSION
const val currentAbiVersion = KLIB_CURRENT_ABI_VERSION
val currentLanguageVersion = LanguageVersion.LATEST_STABLE
val currentApiVersion = ApiVersion.LATEST_STABLE
fun libraryInRepo(repository: File, name: String): File {
val resolver = KonanLibrarySearchPathResolver(
repositories = listOf(repository.absolutePath),
target = null,
distributionKlib = null,
localKonanDir = null,
skipCurrentDir = true
)
return resolver.resolve(name)
}
fun libraryInRepo(repository: File, name: String) =
noTargetResolver(listOf(repository.absolutePath), skipCurrentDir = true).resolve(name)
fun libraryInCurrentDir(name: String): File {
val resolver = KonanLibrarySearchPathResolver(
repositories = emptyList(),
target = null,
distributionKlib = null,
localKonanDir = null
)
return resolver.resolve(name)
}
fun libraryInRepoOrCurrentDir(repository: File, name: String): File {
val resolver = KonanLibrarySearchPathResolver(
repositories = listOf(repository.absolutePath),
target = null,
distributionKlib = null,
localKonanDir = null
)
return resolver.resolve(name)
}
fun libraryInCurrentDir(name: String) = noTargetResolver(emptyList()).resolve(name)
fun libraryInRepoOrCurrentDir(repository: File, name: String) =
noTargetResolver(listOf(repository.absolutePath)).resolve(name)
fun main(args: Array<String>) {
val command = Command(args)
@@ -6,6 +6,10 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
import org.jetbrains.kotlin.konan.util.suffixIfNot
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
const val KONAN_STDLIB_NAME = "stdlib"
interface SearchPathResolver {
val searchRoots: List<File>
@@ -13,23 +17,40 @@ interface SearchPathResolver {
fun defaultLinks(noStdLib: Boolean, noDefaultLibs: Boolean): List<File>
}
fun defaultResolver(repositories: List<String>, target: KonanTarget): SearchPathResolver =
defaultResolver(repositories, target, Distribution())
interface SearchPathResolverWithTarget: SearchPathResolver {
val target: KonanTarget
}
fun defaultResolver(repositories: List<String>, target: KonanTarget, distribution: Distribution): SearchPathResolver =
KonanLibrarySearchPathResolver(
repositories,
target,
distribution.klib,
distribution.localKonanDir.absolutePath
)
class KonanLibrarySearchPathResolver(
fun defaultResolver(
repositories: List<String>,
target: KonanTarget
): SearchPathResolverWithTarget = defaultResolver(repositories, target, Distribution())
fun defaultResolver(
repositories: List<String>,
target: KonanTarget,
distribution: Distribution,
skipCurrentDir: Boolean = false
): SearchPathResolverWithTarget = KonanLibrarySearchPathResolverWithTarget(
repositories,
target,
distribution.klib,
distribution.localKonanDir.absolutePath,
skipCurrentDir)
fun noTargetResolver(
repositories: List<String>,
distributionKlib: String? = null,
localKonanDir: String? = null,
skipCurrentDir: Boolean = false
): SearchPathResolver = KonanLibrarySearchPathResolver(repositories, distributionKlib, localKonanDir, skipCurrentDir)
internal open class KonanLibrarySearchPathResolver(
repositories: List<String>,
val target: KonanTarget?,
val distributionKlib: String?,
val localKonanDir: String?,
val skipCurrentDir: Boolean = false
val skipCurrentDir: Boolean
) : SearchPathResolver {
val localHead: File?
@@ -38,15 +59,12 @@ class KonanLibrarySearchPathResolver(
val distHead: File?
get() = distributionKlib?.File()?.child("common")
val distPlatformHead: File?
get() = target?.let { distributionKlib?.File()?.child("platform")?.child(target.visibleName) }
open val distPlatformHead: File? = null
val currentDirHead: File?
get() = if (!skipCurrentDir) File.userDir else null
private val repoRoots: List<File> by lazy {
repositories.map { File(it) }
}
private val repoRoots: List<File> by lazy { repositories.map { File(it) } }
// This is the place where we specify the order of library search.
override val searchRoots: List<File> by lazy {
@@ -57,8 +75,8 @@ class KonanLibrarySearchPathResolver(
fun check(file: File): Boolean =
file.exists && (file.isFile || File(file, "manifest").exists)
val noSuffix = File(candidate.path.removeSuffixIfPresent(".klib"))
val withSuffix = File(candidate.path.suffixIfNot(".klib"))
val noSuffix = File(candidate.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT))
val withSuffix = File(candidate.path.suffixIfNot(KLIB_FILE_EXTENSION_WITH_DOT))
return when {
check(withSuffix) -> withSuffix
check(noSuffix) -> noSuffix
@@ -90,12 +108,12 @@ class KonanLibrarySearchPathResolver(
val result = mutableListOf<File>()
if (!noStdLib) {
result.add(resolve("stdlib"))
result.add(resolve(KONAN_STDLIB_NAME))
}
if (!noDefaultLibs) {
val defaultLibs = defaultRoots.flatMap { it.listFiles }
.filterNot { it.name.removeSuffixIfPresent(".klib") == "stdlib" }
.filterNot { it.name.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT) == KONAN_STDLIB_NAME }
.map { File(it.absolutePath) }
result.addAll(defaultLibs)
}
@@ -103,3 +121,15 @@ class KonanLibrarySearchPathResolver(
return result
}
}
internal class KonanLibrarySearchPathResolverWithTarget(
repositories: List<String>,
override val target: KonanTarget,
distributionKlib: String?,
localKonanDir: String?,
skipCurrentDir: Boolean
): KonanLibrarySearchPathResolver(repositories, distributionKlib, localKonanDir, skipCurrentDir), SearchPathResolverWithTarget {
override val distPlatformHead: File?
get() = distributionKlib?.File()?.child("platform")?.child(target.visibleName)
}
@@ -16,17 +16,18 @@
package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
import org.jetbrains.kotlin.backend.konan.library.KLIB_CURRENT_ABI_VERSION
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.defaultResolver
import org.jetbrains.kotlin.konan.library.includedHeaders
import org.jetbrains.kotlin.konan.library.libraryResolver
import org.jetbrains.kotlin.konan.library.packageFqName
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.native.interop.gen.jvm.interop
import org.jetbrains.kotlin.utils.addIfNotNull
private val NODEFAULTLIBS = "-nodefaultlibs"
private val PURGE_USER_LIBS = "--purge_user_libs"
private const val NODEFAULTLIBS = "-nodefaultlibs"
private const val PURGE_USER_LIBS = "--purge_user_libs"
// TODO: this function should eventually be eliminated from 'utilities'.
// The interaction of interop and the compler should be streamlined.
@@ -68,15 +69,15 @@ fun invokeInterop(flavor: String, args: Array<String>): Array<String> {
val manifest = File(buildDir, "manifest.properties")
val target = PlatformManager().targetManager(targetRequest).target
val resolver = defaultResolver(repos, target)
val allLibraries = resolver.resolveLibrariesRecursive(
libraries, target, noStdLib = true, noDefaultLibs = noDefaultLibs
)
val resolver = defaultResolver(repos, target).libraryResolver(KLIB_CURRENT_ABI_VERSION)
val allLibraries = resolver.resolveWithDependencies(
libraries, noStdLib = true, noDefaultLibs = noDefaultLibs
).getFullList()
val importArgs = allLibraries.flatMap {libraryReader ->
val importArgs = allLibraries.flatMap { library ->
// TODO: handle missing properties?
libraryReader.packageFqName?.asString()?.let { packageFqName ->
val headerIds = libraryReader.includedHeaders
library.packageFqName?.asString()?.let { packageFqName ->
val headerIds = library.includedHeaders
val arg = "$packageFqName:${headerIds.joinToString(";")}"
listOf("-import", arg)
} ?: emptyList()