KLIB reader: separate library reading from dependency resolving (#1925)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -17,9 +17,6 @@
 package org.jetbrains.kotlin.backend.konan
 
 import com.intellij.openapi.project.Project
-import org.jetbrains.kotlin.backend.konan.library.resolveImmediateLibraries
-import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
-import org.jetbrains.kotlin.backend.konan.library.withResolvedDependencies
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
@@ -34,6 +31,7 @@ import org.jetbrains.kotlin.konan.TempFiles
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.defaultResolver
+import org.jetbrains.kotlin.konan.library.libraryResolver
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.konan.util.profile
 import org.jetbrains.kotlin.serialization.konan.DefaultKonanModuleDescriptorFactory
@@ -91,29 +89,25 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         get() = configuration.getList(KonanConfigKeys.LIBRARY_FILES)
 
     private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES)
-    private val resolver = defaultResolver(repositories, target, distribution)
+    private val resolver = defaultResolver(repositories, target, distribution).libraryResolver(currentAbiVersion)
 
-    internal val immediateLibraries: List<KonanLibrary> by lazy {
-        val result = resolver.resolveImmediateLibraries(
+    internal val resolvedLibraries by lazy {
+        resolver.resolveWithDependencies(
                 libraryNames,
-                target,
-                currentAbiVersion,
-                configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
-                configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS),
-                { msg -> configuration.report(STRONG_WARNING, msg) })
-        resolver.resolveLibrariesRecursive(result, target, currentAbiVersion)
-        result
+                noStdLib = configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
+                noDefaultLibs = configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS) ) { msg ->
+            configuration.report(STRONG_WARNING, msg) }
     }
 
     fun librariesWithDependencies(moduleDescriptor: ModuleDescriptor?): List<KonanLibrary> {
         if (moduleDescriptor == null) error("purgeUnneeded() only works correctly after resolve is over, and we have successfully marked package files as needed or not needed.")
 
-        return immediateLibraries.purgeUnneeded(this).withResolvedDependencies()
+        return resolvedLibraries.filterRoots { (!it.isDefault && !this.purgeUserLibs) || it.isNeededForLink }.getFullList()
     }
 
     private val loadedDescriptors = loadLibMetadata()
 
-    internal lateinit var friends:Set<ModuleDescriptorImpl>
+    internal lateinit var friends: Set<ModuleDescriptorImpl>
 
     internal val defaultNativeLibraries = 
         if (produce == CompilerOutputKind.PROGRAM) 
@@ -128,24 +122,25 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
     fun loadLibMetadata(): List<ModuleDescriptorImpl> {
 
-        val allMetadata = mutableListOf<ModuleDescriptorImpl>()
         val specifics = configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)!!
+        val friendLibsSet = configuration.get(KonanConfigKeys.FRIEND_MODULES)?.map { File(it) }?.toSet()
 
-        val libraries = immediateLibraries.withResolvedDependencies()
-        val friendLibsSet = configuration.get(KonanConfigKeys.FRIEND_MODULES)?.map{File(it)}?.toSet()
+        val allMetadata = mutableListOf<ModuleDescriptorImpl>()
         val friends = mutableListOf<ModuleDescriptorImpl>()
-        for (klib in libraries) {
-            profile("Loading ${klib.libraryName}") {
+
+        resolvedLibraries.forEach { library, packageAccessedHandler ->
+            profile("Loading ${library.libraryName}") {
                 // MutableModuleContext needs ModuleDescriptorImpl, rather than ModuleDescriptor.
-                val moduleDescriptor = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(klib, specifics)
+                val moduleDescriptor = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(library, specifics, packageAccessedHandler)
                 allMetadata.add(moduleDescriptor)
                 friendLibsSet?.apply {
-                    if (contains(klib.libraryFile))
-                        friends.add(moduleDescriptor)
+                    if (contains(library.libraryFile)) friends.add(moduleDescriptor)
                 }
             }
         }
+
         this.friends = friends.toSet()
+
         return allMetadata
     }
 
@@ -176,6 +171,3 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
 fun CompilerConfiguration.report(priority: CompilerMessageSeverity, message: String) 
     = this.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(priority, message)
-
-private fun <T: KonanLibrary> List<T>.purgeUnneeded(config: KonanConfig): List<T> =
-        this.filter{ (!it.isDefaultLibrary && !config.purgeUserLibs) || it.isNeededForLink }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt

@@ -19,6 +19,8 @@ package org.jetbrains.kotlin.backend.konan.library
 import llvm.LLVMModuleRef
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 
+const val KLIB_CURRENT_ABI_VERSION = 1
+
 interface KonanLibraryWriter {
     fun addLinkData(linkData: LinkData)
     fun addNativeBitcode(library: String)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolverExtensions.kt

@@ -1,127 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.backend.konan.library
-
-import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.konan.library.SearchPathResolver
-import org.jetbrains.kotlin.konan.library.createKonanLibraryReader
-import org.jetbrains.kotlin.konan.library.unresolvedDependencies
-import org.jetbrains.kotlin.konan.target.KonanTarget
-
-const val KONAN_CURRENT_ABI_VERSION = 1
-
-/**
- * Returns the list of [KonanLibrary]s given the list of user provided [libraryNames] along with
- * other parameters: [target], [abiVersion], [noStdLib], [noDefaultLibs].
- */
-fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
-                                                 target: KonanTarget,
-                                                 abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
-                                                 noStdLib: Boolean = false,
-                                                 noDefaultLibs: Boolean = false,
-                                                 logger: ((String) -> Unit)?): List<KonanLibrary> {
-    val userProvidedLibraries = libraryNames
-            .map { resolve(it) }
-            .map{ createKonanLibraryReader(it, abiVersion, target) }
-
-    val defaultLibraries = defaultLinks(noStdLib = noStdLib, noDefaultLibs = noDefaultLibs).map {
-        createKonanLibraryReader(it, abiVersion, target, isDefaultLibrary = true)
-    }
-
-    // Make sure the user provided ones appear first, so that 
-    // they have precedence over defaults when duplicates are eliminated.
-    val resolvedLibraries = userProvidedLibraries + defaultLibraries
-
-    warnOnLibraryDuplicates(resolvedLibraries.map { it.libraryFile }, logger)
-
-    return resolvedLibraries.distinctBy { it.libraryFile.absolutePath }
-}
-
-private fun warnOnLibraryDuplicates(resolvedLibraries: List<File>, logger: ((String) -> Unit)? ) {
-
-    if (logger == null) return
-
-    val duplicates = resolvedLibraries.groupBy { it.absolutePath } .values.filter { it.size > 1 }
-
-    duplicates.forEach {
-        logger("library included more than once: ${it.first().absolutePath}")
-    }
-}
-
-/**
- * For each of the given [immediateLibraries] fills in `resolvedDependencies` field with the
- * [KonanLibrary]s the library !!directly!! depends on.
- */
-fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanLibrary>,
-                                                 target: KonanTarget,
-                                                 abiVersion: Int) {
-    val cache = mutableMapOf<File, KonanLibrary>()
-    cache.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it })
-    var newDependencies = cache.values.toList()
-    do {
-        newDependencies = newDependencies.map { library: KonanLibrary ->
-            library.unresolvedDependencies
-                    .map { resolve(it).absoluteFile }
-                    .mapNotNull {
-                        if (it in cache) {
-                            library.resolvedDependencies.add(cache[it]!!)
-                            null
-                        } else {
-                            val reader = createKonanLibraryReader(it, abiVersion, target)
-                            cache[it] = reader
-                            library.resolvedDependencies.add(reader)
-                            reader
-                        }
-            }
-        } .flatten()
-    } while (newDependencies.isNotEmpty())
-}
-
-/**
- * For the given list of [KonanLibrary]s returns the list of [KonanLibrary]s
- * that includes the same libraries plus all their (transitive) dependencies.
- */
-fun List<KonanLibrary>.withResolvedDependencies(): List<KonanLibrary> {
-    val result = mutableSetOf<KonanLibrary>()
-    result.addAll(this)
-    var newDependencies = result.toList()
-    do {
-        newDependencies = newDependencies
-            .map { it -> it.resolvedDependencies }.flatten()
-            .filter { it !in result }
-        result.addAll(newDependencies)
-    } while (newDependencies.isNotEmpty())
-    return result.toList()
-}
-
-fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List<String>,
-                                                 target: KonanTarget,
-                                                 abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
-                                                 noStdLib: Boolean = false,
-                                                 noDefaultLibs: Boolean = false): List<KonanLibrary> {
-    val immediateLibraries = resolveImmediateLibraries(
-                    libraryNames = libraryNames,
-                    target = target,
-                    abiVersion = abiVersion,
-                    noStdLib = noStdLib,
-                    noDefaultLibs = noDefaultLibs,
-                    logger = null
-            )
-    resolveLibrariesRecursive(immediateLibraries, target, abiVersion)
-    return immediateLibraries.withResolvedDependencies()
-}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -25,14 +25,14 @@ import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.descriptors.findPackage
 import org.jetbrains.kotlin.backend.konan.hash.GlobalHash
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
-import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.backend.konan.library.withResolvedDependencies
 import org.jetbrains.kotlin.descriptors.konan.CompiledKonanModuleOrigin
 import org.jetbrains.kotlin.descriptors.konan.CurrentKonanModuleOrigin
 import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
 import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
 import org.jetbrains.kotlin.ir.declarations.IrField
 import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.resolver.TopologicalLibraryOrder
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -330,47 +330,23 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
         private val allLibraries = context.librariesWithDependencies.toSet()
 
         override fun add(origin: CompiledKonanModuleOrigin) {
-            val reader = when (origin) {
+            val library = when (origin) {
                 CurrentKonanModuleOrigin -> return
-                is DeserializedKonanModuleOrigin -> origin.reader as KonanLibrary
+                is DeserializedKonanModuleOrigin -> origin.library as KonanLibrary
             }
 
-            if (reader !in allLibraries) {
-                error("$reader (${reader.libraryName}) is used but not requested")
+            if (library !in allLibraries) {
+                error("Library (${library.libraryName}) is used but not requested.\nRequested libraries: ${allLibraries.joinToString { it.libraryName }}")
             }
 
-            usedLibraries.add(reader)
+            usedLibraries.add(library)
         }
     }
 
-    val librariesToLink: List<KonanLibrary>  by lazy {
-        context.config.immediateLibraries
-                .filter { (!it.isDefaultLibrary && !context.config.purgeUserLibs) || it in usedLibraries }
-                .withResolvedDependencies()
-                .topoSort()
-    }
-
-    private fun List<KonanLibrary>.topoSort(): List<KonanLibrary> {
-        var sorted = mutableListOf<KonanLibrary>()
-        val visited = mutableSetOf<KonanLibrary>()
-        val tempMarks = mutableSetOf<KonanLibrary>()
-
-        fun visit(node: KonanLibrary, result: MutableList<KonanLibrary>) {
-            if (visited.contains(node)) return
-            if (tempMarks.contains(node)) error("Cyclic dependency in library graph.")
-            tempMarks.add(node)
-            node.resolvedDependencies.forEach {
-                visit(it, result)
-            }
-            visited.add(node)
-            result += node
-        }
-
-        this.forEach next@{
-            if (visited.contains(it)) return@next
-            visit(it, sorted)
-        }
-        return sorted
+    val librariesToLink: List<KonanLibrary> by lazy {
+        context.config.resolvedLibraries
+                .filterRoots { (!it.isDefault && !context.config.purgeUserLibs) || it.library in usedLibraries }
+                .getFullList(TopologicalLibraryOrder)
     }
 
     val librariesForLibraryManifest: List<KonanLibrary> get() {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIr.proto

@@ -2,7 +2,7 @@
 syntax = "proto2";
 package org.jetbrains.kotlin.metadata;
 
-// FIXME: ddol: fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
+// FIXME(ddol): fix this import after moving `metadata` to main Kotlin repo - it should refer to the actual metadata.proto file from Kotlin project
 import "org/jetbrains/kotlin/backend/konan/serialization/metadata.proto1";
 option java_outer_classname = "KonanIr";
 option optimize_for = LITE_RUNTIME;