Minor improvements in recursive klib dependency support

* Support -r option in the cinterop tool.
 * Move klib resolve into separate methods.
 * Don't treat directory as a library if it doesn't contain a manifest.
 * Rework recursive library search + duplicate removal.
 * Support recursive klib processing in the cinterop tool.

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -18,8 +18,7 @@ package org.jetbrains.kotlin.backend.konan
 
 import com.intellij.openapi.project.Project
 import org.jetbrains.kotlin.backend.konan.descriptors.createForwardDeclarationsModule
-import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
+import org.jetbrains.kotlin.backend.konan.library.*
-import org.jetbrains.kotlin.backend.konan.library.KonanLibrarySearchPathResolver
 import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
 import org.jetbrains.kotlin.backend.konan.util.profile
 import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent
@@ -33,8 +32,6 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.*
-import org.jetbrains.kotlin.konan.target.TargetManager.*
-import org.jetbrains.kotlin.konan.target.CompilerOutputKind.*
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.storage.StorageManager
@@ -79,40 +76,24 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         get() = configuration.getList(KonanConfigKeys.LIBRARY_FILES)
 
     private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES)
-    private val resolver = KonanLibrarySearchPathResolver(repositories, distribution.klib, distribution.localKonanDir)
+    private val resolver = defaultResolver(repositories, distribution)
 
     val immediateLibraries: List<LibraryReaderImpl> by lazy {
-        val target = targetManager.target
+        resolver.resolveImmediateLibraries(
-
+                libraryNames,
-        val defaultLibraries = resolver.defaultLinks(
+                targetManager.target,
+                currentAbiVersion,
                 configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
-                configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS)
+                configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS),
-            ) .map { LibraryReaderImpl(it, currentAbiVersion, target, isDefaultLink=true) }
+                false
-
+        ).let {
-        val userProvidedLibraries = libraryNames
+            warnOnLibraryDuplicates(it.map { it.libraryFile })
-            .map { resolver.resolve(it) }
+            it.distinctBy { it.libraryFile.absolutePath }
-            .map{ LibraryReaderImpl(it, currentAbiVersion, target) }
+        }
-
-        var resolvedLibraries = defaultLibraries + userProvidedLibraries
-        warnOnLibraryDuplicates(resolvedLibraries.map{ it.libraryFile })
-        resolvedLibraries.distinctBy { it.libraryFile.absolutePath }
     }
 
     val libraries: List<LibraryReaderImpl> by lazy {
-        val result = mutableListOf<LibraryReaderImpl>()
+        resolver.resolveLibrariesRecursive(immediateLibraries, targetManager.target, currentAbiVersion)
-        result.addAll(immediateLibraries)
-        do {
-            val dependencies = result 
-                .map { it.dependencies } .flatten()
-                .map { resolver.resolve(it) }
-                .map { LibraryReaderImpl(it, currentAbiVersion, targetManager.target) }
-                .distinctBy { it.libraryFile.absolutePath }
-
-            val newDependencies = dependencies.deleteMatching(result, { it.libraryFile.absolutePath })
-            result.addAll(newDependencies)
-        } while (newDependencies.size > 0)
-
-        result
     }
 
     private val loadedDescriptors = loadLibMetadata()
@@ -170,10 +151,5 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
     }
 }
 
-private fun <T, K> List<T>.deleteMatching(other: List<T>, transform: (T) -> K): List<T> {
-    val transformed = other.map { transform(it) }
-    return this.filterNot { transformed.contains( transform(it) ) }
-}
-
 fun CompilerConfiguration.report(priority: CompilerMessageSeverity, message: String) 
     = this.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(priority, message)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt

@@ -16,9 +16,13 @@
 
 package org.jetbrains.kotlin.backend.konan.library
 
+import org.jetbrains.kotlin.backend.konan.Distribution
+import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
 import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent
 import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
 import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.konan.target.TargetManager
 
 interface SearchPathResolver {
     val searchRoots: List<File>
@@ -26,6 +30,63 @@ interface SearchPathResolver {
     fun defaultLinks(nostdlib: Boolean, noDefaultLibs: Boolean): List<File>
 }
 
+fun defaultResolver(repositories: List<String>, targetManager: TargetManager): SearchPathResolver =
+        defaultResolver(repositories, Distribution(targetManager))
+
+fun defaultResolver(repositories: List<String>, distribution: Distribution): SearchPathResolver =
+        KonanLibrarySearchPathResolver(repositories, distribution.klib, distribution.localKonanDir)
+
+fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
+                                                 target: KonanTarget,
+                                                 abiVersion: Int = 1,
+                                                 noStdLib: Boolean = false,
+                                                 noDefaultLibs: Boolean = false,
+                                                 removeDuplicates: Boolean = true): List<LibraryReaderImpl> {
+
+    val defaultLibraries = defaultLinks(noStdLib, noDefaultLibs).map {
+        LibraryReaderImpl(it, abiVersion, target, isDefaultLink = true)
+    }
+
+    val userProvidedLibraries = libraryNames
+            .map { resolve(it) }
+            .map{ LibraryReaderImpl(it, abiVersion, target) }
+
+    val resolvedLibraries = defaultLibraries + userProvidedLibraries
+
+    return resolvedLibraries.let {
+        if (removeDuplicates) it.distinctBy { it.libraryFile.absolutePath } else it
+    }
+}
+
+fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<LibraryReaderImpl>,
+                                                 target: KonanTarget,
+                                                 abiVersion: Int): List<LibraryReaderImpl> {
+    val result = mutableMapOf<File, LibraryReaderImpl>()
+    result.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it })
+    var newDependencies: Map<File, LibraryReaderImpl> = result
+    do {
+        newDependencies = newDependencies.values.asSequence()
+                .flatMap { it.dependencies.asSequence() }
+                .map { resolve(it).absoluteFile }
+                .filter { it !in result }
+                .map { it to LibraryReaderImpl(it, abiVersion, target) }.toMap()
+
+        result.putAll(newDependencies)
+    } while (newDependencies.isNotEmpty())
+    return result.values.toList()
+}
+
+fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List<String>,
+                                                 target: KonanTarget,
+                                                 abiVersion: Int = 1,
+                                                 noStdLib: Boolean = false,
+                                                 noDefaultLibs: Boolean = false): List<LibraryReaderImpl> {
+    return resolveLibrariesRecursive(
+            resolveImmediateLibraries(libraryNames, target, abiVersion, noStdLib, noDefaultLibs, true),
+            target, abiVersion
+    )
+}
+
 class KonanLibrarySearchPathResolver(repositories: List<String>,
     val distributionKlib: String?, val localKonanDir: String?, val skipCurrentDir: Boolean = false): SearchPathResolver {
 
@@ -48,21 +109,22 @@ class KonanLibrarySearchPathResolver(repositories: List<String>,
     }
 
     private fun found(candidate: File): File? {
+        fun check(file: File): Boolean =
+                file.exists && (file.isFile || File(file, "manifest").exists)
+
         val noSuffix = File(candidate.path.removeSuffixIfPresent(".klib"))
         val withSuffix = File(candidate.path.suffixIfNot(".klib"))
-        if (withSuffix.exists) {
+        return when {
-            return withSuffix
+            check(withSuffix) -> withSuffix
+            check(noSuffix) -> noSuffix
+            else -> null
         }
-        if (noSuffix.exists) {
-            return noSuffix
-        }
-        return null
     }
 
     override fun resolve(givenPath: String): File {
         val given = File(givenPath)
         if (given.isAbsolute) {
-            found(given)?.apply{return this}
+            found(given)?.apply{ return this }
         } else {
             searchRoots.forEach{ 
                 found(File(it, givenPath))?.apply{return this}
@@ -74,7 +136,7 @@ class KonanLibrarySearchPathResolver(repositories: List<String>,
     private val File.klib
         get() = File(this, "klib")
 
-    // The libraries from the default root are linked autimatically.
+    // The libraries from the default root are linked automatically.
     val defaultRoot: File?
         get() = if (distHead?.exists ?: false) distHead else null
 

shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt

@@ -24,7 +24,7 @@ import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.StandardOpenOption
 
-class File constructor(internal val javaPath: Path) {
+data class File constructor(internal val javaPath: Path) {
     constructor(parent: Path, child: String): this(parent.resolve(child))
     constructor(parent: File, child: String): this(parent.javaPath.resolve(child))
     constructor(path: String): this(Paths.get(path))

utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt

@@ -1,8 +1,11 @@
 package org.jetbrains.kotlin.cli.utilities
 
+import org.jetbrains.kotlin.backend.konan.library.defaultResolver
 import org.jetbrains.kotlin.backend.konan.library.impl.KonanLibrary
+import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.properties.loadProperties
+import org.jetbrains.kotlin.konan.target.TargetManager
 import org.jetbrains.kotlin.utils.addIfNotNull
 import org.jetbrains.kotlin.cli.bc.main as konancMain
 import org.jetbrains.kotlin.native.interop.gen.jvm.interop 
@@ -12,6 +15,7 @@ fun invokeCinterop(args: Array<String>) {
     var outputFileName = "nativelib"
     var target = "host"
     val libraries = mutableListOf<String>()
+    val repos = mutableListOf<String>()
     for (i in args.indices) {
         val arg = args[i]
         val nextArg = args.getOrNull(i + 1)
@@ -21,6 +25,8 @@ fun invokeCinterop(args: Array<String>) {
             target = nextArg ?: target
         if (arg == "-library")
             libraries.addIfNotNull(nextArg)
+        if (arg == "-r" || arg == "-repo")
+            repos.addIfNotNull(nextArg)
     }
 
     val buildDir = File("$outputFileName-build")
@@ -29,14 +35,22 @@ fun invokeCinterop(args: Array<String>) {
     val cstubsName ="cstubs"
     val manifest = File(buildDir, "manifest.properties")
 
-    val importArgs = libraries.flatMap {
+    val targetManager = TargetManager(target)
-        val library = KonanLibrary(File(it))
+    val resolver = defaultResolver(repos, targetManager)
+    val allLibraries = resolver.resolveLibrariesRecursive(
+            libraries, targetManager.target, noStdLib = true, noDefaultLibs = false
+    )
+
+    val importArgs = allLibraries.flatMap {
+        val library = KonanLibrary(it.libraryFile)
         val manifestProperties = library.manifestFile.loadProperties()
         // TODO: handle missing properties?
-        val pkg = manifestProperties["pkg"] as String
+        manifestProperties["pkg"]?.let {
-        val headerIds = (manifestProperties["includedHeaders"] as String).split(' ')
+            val pkg = it as String
-        val arg = "$pkg:${headerIds.joinToString(",")}"
+            val headerIds = (manifestProperties["includedHeaders"] as String).split(' ')
-        listOf("-import", arg)
+            val arg = "$pkg:${headerIds.joinToString(",")}"
+            listOf("-import", arg)
+        } ?: emptyList()
     }
 
     val additionalArgs = listOf<String>(
@@ -58,7 +72,7 @@ fun invokeCinterop(args: Array<String>) {
         "-o", outputFileName,
         "-target", target,
         "-manifest", manifest.path
-    ) + cinteropArgsToCompiler + libraries.flatMap { listOf("-library", it) }
+    ) + cinteropArgsToCompiler + libraries.flatMap { listOf("-library", it) } + repos.flatMap { listOf("-repo", it) }
     konancMain(konancArgs)
 }