[caches] Added saving actual dependencies for caches

It is bad to link more libraries than it is really needed. For the whole world
compilation there is LllvmImports, which tracks all dependencies on bitcode level,
but there has not been such a mechanism for compilations against cached libraries.

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt

@@ -21,6 +21,11 @@ class CachedLibraries(
 
     class Cache(val kind: Kind, val path: String) {
         enum class Kind { DYNAMIC, STATIC }
+
+        val bitcodeDependencies by lazy {
+            val directory = File(path).absoluteFile.parent
+            File(directory, BITCODE_DEPENDENCIES_FILE_NAME).readStrings()
+        }
     }
 
     private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? {
@@ -81,5 +86,6 @@ class CachedLibraries(
     companion object {
         fun getCachedLibraryName(library: KotlinLibrary): String = getCachedLibraryName(library.uniqueName)
         fun getCachedLibraryName(libraryName: String): String = "$libraryName-cache"
+        const val BITCODE_DEPENDENCIES_FILE_NAME = "bitcode_deps"
     }
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -79,7 +79,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
     internal val resolvedLibraries get() = resolve.resolvedLibraries
 
-    private val cacheSupport = CacheSupport(configuration, resolvedLibraries, target, produce)
+    internal val cacheSupport = CacheSupport(configuration, resolvedLibraries, target, produce)
 
     internal val cachedLibraries: CachedLibraries
         get() = cacheSupport.cachedLibraries

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt

@@ -6,6 +6,9 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.LinkerOutputKind
 import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder
+import org.jetbrains.kotlin.library.uniqueName
+import org.jetbrains.kotlin.utils.addToStdlib.cast
 
 internal fun determineLinkerOutput(context: Context): LinkerOutputKind =
         when (context.config.produce) {
@@ -44,13 +47,33 @@ internal class Linker(val context: Context) {
 
         val libraryProvidedLinkerFlags = context.llvm.allNativeDependencies.map { it.linkerOpts }.flatten()
 
-        if (context.config.produce.isCache)
+        if (context.config.produce.isCache) {
             context.config.outputFiles.tempCacheDirectory!!.mkdirs()
+            saveAdditionalInfoForCache()
+        }
+
         runLinker(objectFiles, includedBinaries, libraryProvidedLinkerFlags)
 
         renameOutput()
     }
 
+    private fun saveAdditionalInfoForCache() {
+        saveCacheBitcodeDependencies()
+    }
+
+    private fun saveCacheBitcodeDependencies() {
+        val outputFiles = context.config.outputFiles
+        val bitcodeDependenciesFile = File(outputFiles.bitcodeDependenciesFile!!)
+        val bitcodeDependencies = context.config.resolvedLibraries
+                .getFullList(TopologicalLibraryOrder)
+                .filter {
+                    require(it is KonanLibrary)
+                    context.llvmImports.bitcodeIsUsed(it)
+                            && it !in context.config.cacheSupport.librariesToCache // Skip loops.
+                }.cast<List<KonanLibrary>>()
+        bitcodeDependenciesFile.writeLines(bitcodeDependencies.map { it.uniqueName })
+    }
+
     private fun renameOutput() {
         if (context.config.produce.isCache) {
             val outputFiles = context.config.outputFiles
@@ -155,28 +178,21 @@ private fun determineCachesToLink(context: Context): CachesToLink {
     val staticCaches = mutableListOf<String>()
     val dynamicCaches = mutableListOf<String>()
 
-    // TODO: suboptimal, see e.g. [LlvmImports].
+    context.llvm.allCachedBitcodeDependencies.forEach { library ->
-    context.librariesWithDependencies.forEach { library ->
         val currentBinaryContainsLibrary = context.llvmModuleSpecification.containsLibrary(library)
         val cache = context.config.cachedLibraries.getLibraryCache(library)
-        val libraryIsCached = cache != null
+                ?: error("Library $library is expected to be cached")
 
         // Consistency check. Generally guaranteed by implementation.
-        if (currentBinaryContainsLibrary && libraryIsCached) {
+        if (currentBinaryContainsLibrary)
             error("Library ${library.libraryName} is found in both cache and current binary")
-        } else if (!currentBinaryContainsLibrary && !libraryIsCached) {
+
-            error("Library ${library.libraryName} is not found neither in cache nor in current binary")
+        val list = when (cache.kind) {
+            CachedLibraries.Cache.Kind.DYNAMIC -> dynamicCaches
+            CachedLibraries.Cache.Kind.STATIC -> staticCaches
         }
 
-        if (cache != null) {
+        list += cache.path
-            val list = when (cache.kind) {
-                CachedLibraries.Cache.Kind.DYNAMIC -> dynamicCaches
-                CachedLibraries.Cache.Kind.STATIC -> staticCaches
-            }
-
-            list += cache.path
-        }
     }
-
+    return CachesToLink(static = staticCaches, dynamic = dynamicCaches)
-    return CachesToLink(static = staticCaches.distinct(), dynamic = dynamicCaches.distinct())
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt

@@ -51,6 +51,11 @@ class OutputFiles(outputPath: String?, target: KonanTarget, val produce: Compile
 
     val symbolicInfoFile = "$nativeBinaryFile.dSYM"
 
+    val bitcodeDependenciesFile =
+            if (produce.isCache)
+                tempCacheDirectory!!.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME).absolutePath
+            else null
+
     private fun String.fullOutputName() = prefixBaseNameIfNeeded(prefix).suffixIfNeeded(suffix)
 
     private fun String.prefixBaseNameIfNeeded(prefix: String) =

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -10,6 +10,8 @@ import kotlinx.cinterop.get
 import kotlinx.cinterop.memScoped
 import kotlinx.cinterop.toKString
 import llvm.*
+import org.jetbrains.kotlin.backend.konan.CachedLibraries
+import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder
 import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.hash.GlobalHash
 import org.jetbrains.kotlin.backend.konan.ir.llvmSymbolOrigin
@@ -24,6 +26,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder
+import org.jetbrains.kotlin.library.uniqueName
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.utils.addToStdlib.cast
@@ -395,14 +398,46 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
                     require(it is KonanLibrary)
                     (!it.isDefault && !context.config.purgeUserLibs) || imports.nativeDependenciesAreUsed(it)
                 }.cast<List<KonanLibrary>>()
+    }
 
+    private val immediateBitcodeDependencies: List<KonanLibrary> by lazy {
+        context.config.resolvedLibraries.getFullList(TopologicalLibraryOrder).cast<List<KonanLibrary>>()
+                .filter { (!it.isDefault && !context.config.purgeUserLibs) || imports.bitcodeIsUsed(it) }
+    }
+
+    val allCachedBitcodeDependencies: List<KonanLibrary> by lazy {
+        val allLibraries = context.config.resolvedLibraries.getFullList().associateBy { it.uniqueName }
+        val result = mutableSetOf<KonanLibrary>()
+
+        fun addDependencies(cachedLibrary: CachedLibraries.Cache) {
+            cachedLibrary.bitcodeDependencies.forEach {
+                val library = allLibraries[it] ?: error("Bitcode dependency to an unknown library: $it")
+                result.add(library as KonanLibrary)
+                addDependencies(context.config.cachedLibraries.getLibraryCache(library)
+                        ?: error("Library $it is expected to be cached"))
+            }
+        }
+
+        for (library in immediateBitcodeDependencies) {
+            val cache = context.config.cachedLibraries.getLibraryCache(library)
+            if (cache != null) {
+                result += library
+                addDependencies(cache)
+            }
+        }
+
+        result.toList()
     }
 
     val allNativeDependencies: List<KonanLibrary> by lazy {
-        val cachedLibraries = context.librariesWithDependencies.filter {
+        (nativeDependenciesToLink + allCachedBitcodeDependencies).distinct()
-            context.config.cachedLibraries.isLibraryCached(it)
+    }
+
+    val allBitcodeDependencies: List<KonanLibrary> by lazy {
+        val allNonCachedDependencies = context.librariesWithDependencies.filter {
+            context.config.cachedLibraries.getLibraryCache(it) == null
         }
-        (nativeDependenciesToLink + cachedLibraries).distinct()
+        (allNonCachedDependencies + allCachedBitcodeDependencies).distinct()
     }
 
     val bitcodeToLink: List<KonanLibrary> by lazy {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2375,7 +2375,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
     //-------------------------------------------------------------------------//
     fun appendStaticInitializers() {
         // Null for "current" non-library module:
-        val libraries = (context.librariesWithDependencies + listOf(null))
+        val libraries = (context.llvm.allBitcodeDependencies + listOf(null))
 
         val libraryToInitializers = libraries.associateWith {
             mutableListOf<LLVMValueRef>()