From e245a23c34defabb7be46b01253dfaeea144ce26 Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Mon, 2 Oct 2017 18:38:56 +0300 Subject: [PATCH] Minor improvements in recursive klib dependency support * Support -r option in the cinterop tool. * Move klib resolve into separate methods. * Don't treat directory as a library if it doesn't contain a manifest. * Rework recursive library search + duplicate removal. * Support recursive klib processing in the cinterop tool. --- .../kotlin/backend/konan/KonanConfig.kt | 50 ++++-------- .../konan/library/SearchPathResolver.kt | 78 +++++++++++++++++-- .../jetbrains/kotlin/konan/file/NoJavaUtil.kt | 2 +- .../jetbrains/kotlin/cli/utilities/main.kt | 28 +++++-- 4 files changed, 105 insertions(+), 53 deletions(-) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 0fba27135e9..1475ef498be 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -18,8 +18,7 @@ package org.jetbrains.kotlin.backend.konan import com.intellij.openapi.project.Project import org.jetbrains.kotlin.backend.konan.descriptors.createForwardDeclarationsModule -import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader -import org.jetbrains.kotlin.backend.konan.library.KonanLibrarySearchPathResolver +import org.jetbrains.kotlin.backend.konan.library.* import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl import org.jetbrains.kotlin.backend.konan.util.profile import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent @@ -33,8 +32,6 @@ import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.* -import org.jetbrains.kotlin.konan.target.TargetManager.* -import org.jetbrains.kotlin.konan.target.CompilerOutputKind.* import org.jetbrains.kotlin.konan.util.DependencyProcessor import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.storage.StorageManager @@ -79,40 +76,24 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration get() = configuration.getList(KonanConfigKeys.LIBRARY_FILES) private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES) - private val resolver = KonanLibrarySearchPathResolver(repositories, distribution.klib, distribution.localKonanDir) + private val resolver = defaultResolver(repositories, distribution) val immediateLibraries: List by lazy { - val target = targetManager.target - - val defaultLibraries = resolver.defaultLinks( + resolver.resolveImmediateLibraries( + libraryNames, + targetManager.target, + currentAbiVersion, configuration.getBoolean(KonanConfigKeys.NOSTDLIB), - configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS) - ) .map { LibraryReaderImpl(it, currentAbiVersion, target, isDefaultLink=true) } - - val userProvidedLibraries = libraryNames - .map { resolver.resolve(it) } - .map{ LibraryReaderImpl(it, currentAbiVersion, target) } - - var resolvedLibraries = defaultLibraries + userProvidedLibraries - warnOnLibraryDuplicates(resolvedLibraries.map{ it.libraryFile }) - resolvedLibraries.distinctBy { it.libraryFile.absolutePath } + configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS), + false + ).let { + warnOnLibraryDuplicates(it.map { it.libraryFile }) + it.distinctBy { it.libraryFile.absolutePath } + } } val libraries: List by lazy { - val result = mutableListOf() - result.addAll(immediateLibraries) - do { - val dependencies = result - .map { it.dependencies } .flatten() - .map { resolver.resolve(it) } - .map { LibraryReaderImpl(it, currentAbiVersion, targetManager.target) } - .distinctBy { it.libraryFile.absolutePath } - - val newDependencies = dependencies.deleteMatching(result, { it.libraryFile.absolutePath }) - result.addAll(newDependencies) - } while (newDependencies.size > 0) - - result + resolver.resolveLibrariesRecursive(immediateLibraries, targetManager.target, currentAbiVersion) } private val loadedDescriptors = loadLibMetadata() @@ -170,10 +151,5 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } } -private fun List.deleteMatching(other: List, transform: (T) -> K): List { - val transformed = other.map { transform(it) } - return this.filterNot { transformed.contains( transform(it) ) } -} - fun CompilerConfiguration.report(priority: CompilerMessageSeverity, message: String) = this.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(priority, message) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt index 9b5c18f194d..1a28635cb18 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt @@ -16,9 +16,13 @@ package org.jetbrains.kotlin.backend.konan.library +import org.jetbrains.kotlin.backend.konan.Distribution +import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent import org.jetbrains.kotlin.backend.konan.util.suffixIfNot import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.TargetManager interface SearchPathResolver { val searchRoots: List @@ -26,6 +30,63 @@ interface SearchPathResolver { fun defaultLinks(nostdlib: Boolean, noDefaultLibs: Boolean): List } +fun defaultResolver(repositories: List, targetManager: TargetManager): SearchPathResolver = + defaultResolver(repositories, Distribution(targetManager)) + +fun defaultResolver(repositories: List, distribution: Distribution): SearchPathResolver = + KonanLibrarySearchPathResolver(repositories, distribution.klib, distribution.localKonanDir) + +fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List, + target: KonanTarget, + abiVersion: Int = 1, + noStdLib: Boolean = false, + noDefaultLibs: Boolean = false, + removeDuplicates: Boolean = true): List { + + val defaultLibraries = defaultLinks(noStdLib, noDefaultLibs).map { + LibraryReaderImpl(it, abiVersion, target, isDefaultLink = true) + } + + val userProvidedLibraries = libraryNames + .map { resolve(it) } + .map{ LibraryReaderImpl(it, abiVersion, target) } + + val resolvedLibraries = defaultLibraries + userProvidedLibraries + + return resolvedLibraries.let { + if (removeDuplicates) it.distinctBy { it.libraryFile.absolutePath } else it + } +} + +fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List, + target: KonanTarget, + abiVersion: Int): List { + val result = mutableMapOf() + result.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it }) + var newDependencies: Map = result + do { + newDependencies = newDependencies.values.asSequence() + .flatMap { it.dependencies.asSequence() } + .map { resolve(it).absoluteFile } + .filter { it !in result } + .map { it to LibraryReaderImpl(it, abiVersion, target) }.toMap() + + result.putAll(newDependencies) + } while (newDependencies.isNotEmpty()) + return result.values.toList() +} + +fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List, + target: KonanTarget, + abiVersion: Int = 1, + noStdLib: Boolean = false, + noDefaultLibs: Boolean = false): List { + return resolveLibrariesRecursive( + resolveImmediateLibraries(libraryNames, target, abiVersion, noStdLib, noDefaultLibs, true), + target, abiVersion + ) +} + class KonanLibrarySearchPathResolver(repositories: List, val distributionKlib: String?, val localKonanDir: String?, val skipCurrentDir: Boolean = false): SearchPathResolver { @@ -48,21 +109,22 @@ class KonanLibrarySearchPathResolver(repositories: List, } private fun found(candidate: File): File? { + fun check(file: File): Boolean = + file.exists && (file.isFile || File(file, "manifest").exists) + val noSuffix = File(candidate.path.removeSuffixIfPresent(".klib")) val withSuffix = File(candidate.path.suffixIfNot(".klib")) - if (withSuffix.exists) { - return withSuffix + return when { + check(withSuffix) -> withSuffix + check(noSuffix) -> noSuffix + else -> null } - if (noSuffix.exists) { - return noSuffix - } - return null } override fun resolve(givenPath: String): File { val given = File(givenPath) if (given.isAbsolute) { - found(given)?.apply{return this} + found(given)?.apply{ return this } } else { searchRoots.forEach{ found(File(it, givenPath))?.apply{return this} @@ -74,7 +136,7 @@ class KonanLibrarySearchPathResolver(repositories: List, private val File.klib get() = File(this, "klib") - // The libraries from the default root are linked autimatically. + // The libraries from the default root are linked automatically. val defaultRoot: File? get() = if (distHead?.exists ?: false) distHead else null diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt index fac80851489..981f86de830 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt @@ -24,7 +24,7 @@ import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes import java.nio.file.StandardOpenOption -class File constructor(internal val javaPath: Path) { +data class File constructor(internal val javaPath: Path) { constructor(parent: Path, child: String): this(parent.resolve(child)) constructor(parent: File, child: String): this(parent.javaPath.resolve(child)) constructor(path: String): this(Paths.get(path)) diff --git a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt index f11f4f35511..f98d5bcebf8 100644 --- a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt +++ b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt @@ -1,8 +1,11 @@ package org.jetbrains.kotlin.cli.utilities +import org.jetbrains.kotlin.backend.konan.library.defaultResolver import org.jetbrains.kotlin.backend.konan.library.impl.KonanLibrary +import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.properties.loadProperties +import org.jetbrains.kotlin.konan.target.TargetManager import org.jetbrains.kotlin.utils.addIfNotNull import org.jetbrains.kotlin.cli.bc.main as konancMain import org.jetbrains.kotlin.native.interop.gen.jvm.interop @@ -12,6 +15,7 @@ fun invokeCinterop(args: Array) { var outputFileName = "nativelib" var target = "host" val libraries = mutableListOf() + val repos = mutableListOf() for (i in args.indices) { val arg = args[i] val nextArg = args.getOrNull(i + 1) @@ -21,6 +25,8 @@ fun invokeCinterop(args: Array) { target = nextArg ?: target if (arg == "-library") libraries.addIfNotNull(nextArg) + if (arg == "-r" || arg == "-repo") + repos.addIfNotNull(nextArg) } val buildDir = File("$outputFileName-build") @@ -29,14 +35,22 @@ fun invokeCinterop(args: Array) { val cstubsName ="cstubs" val manifest = File(buildDir, "manifest.properties") - val importArgs = libraries.flatMap { - val library = KonanLibrary(File(it)) + val targetManager = TargetManager(target) + val resolver = defaultResolver(repos, targetManager) + val allLibraries = resolver.resolveLibrariesRecursive( + libraries, targetManager.target, noStdLib = true, noDefaultLibs = false + ) + + val importArgs = allLibraries.flatMap { + val library = KonanLibrary(it.libraryFile) val manifestProperties = library.manifestFile.loadProperties() // TODO: handle missing properties? - val pkg = manifestProperties["pkg"] as String - val headerIds = (manifestProperties["includedHeaders"] as String).split(' ') - val arg = "$pkg:${headerIds.joinToString(",")}" - listOf("-import", arg) + manifestProperties["pkg"]?.let { + val pkg = it as String + val headerIds = (manifestProperties["includedHeaders"] as String).split(' ') + val arg = "$pkg:${headerIds.joinToString(",")}" + listOf("-import", arg) + } ?: emptyList() } val additionalArgs = listOf( @@ -58,7 +72,7 @@ fun invokeCinterop(args: Array) { "-o", outputFileName, "-target", target, "-manifest", manifest.path - ) + cinteropArgsToCompiler + libraries.flatMap { listOf("-library", it) } + ) + cinteropArgsToCompiler + libraries.flatMap { listOf("-library", it) } + repos.flatMap { listOf("-repo", it) } konancMain(konancArgs) }