[K/N] Implemented cache orchestration machinery in the compiler

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt

@@ -141,6 +141,23 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
     )
     var cachedLibraries: Array<String>? = null
 
+    @Argument(
+        value = "-Xauto-cache-from",
+        valueDescription = "<path>",
+        description = "Path to the root directory from which dependencies are to be cached automatically.\n" +
+                "By default caches will be placed into the kotlin-native system cache directory.",
+        delimiter = ""
+    )
+    var autoCacheableFrom: Array<String>? = null
+
+    @Argument(
+        value = "-Xauto-cache-dir",
+        valueDescription = "<path>",
+        description = "Path to the directory where to put caches for auto-cacheable dependencies",
+        delimiter = ""
+    )
+    var autoCacheDir: String? = null
+
     @Argument(value="-Xcheck-dependencies", deprecatedName = "--check_dependencies", description = "Check dependencies and download the missing ones")
     var checkDependencies: Boolean = false
 

kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.analyzer.CompilationErrorException
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.cli.common.*
 import org.jetbrains.kotlin.cli.common.arguments.K2NativeCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments
 import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoot
 import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
@@ -57,40 +58,23 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
         }
 
         val pluginLoadResult =
-            PluginCliParser.loadPluginsSafe(arguments.pluginClasspaths, arguments.pluginOptions, arguments.pluginConfigurations, configuration)
+                PluginCliParser.loadPluginsSafe(arguments.pluginClasspaths, arguments.pluginOptions, arguments.pluginConfigurations, configuration)
         if (pluginLoadResult != ExitCode.OK) return pluginLoadResult
 
-        val environment = KotlinCoreEnvironment.createForProduction(rootDisposable,
-            configuration, EnvironmentConfigFiles.NATIVE_CONFIG_FILES)
-        val project = environment.project
         val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) ?: MessageCollector.NONE
         if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR)) {
             messageCollector.report(WARNING, "New compiler pipeline K2 is experimental in Kotlin/Native. No compatibility guarantees are yet provided")
         }
-        configuration.put(CLIConfigurationKeys.FLEXIBLE_PHASE_CONFIG, createFlexiblePhaseConfig(arguments))
-
 
         val enoughArguments = arguments.freeArgs.isNotEmpty() || arguments.isUsefulWithoutFreeArgs
         if (!enoughArguments) {
-            configuration.report(ERROR, "You have not specified any compilation arguments. No output has been produced.")
+            messageCollector.report(ERROR, "You have not specified any compilation arguments. No output has been produced.")
         }
 
-        /* Set default version of metadata version */
+        val environment = prepareEnvironment(arguments, configuration, rootDisposable)
-        val metadataVersionString = arguments.metadataVersion
-        if (metadataVersionString == null) {
-            configuration.put(CommonConfigurationKeys.METADATA_VERSION, KlibMetadataVersion.INSTANCE)
-        }
-
-        val relativePathBases = arguments.relativePathBases
-        if (relativePathBases != null) {
-            configuration.put(CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES, relativePathBases.toList())
-        }
-
-        configuration.put(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH, arguments.normalizeAbsolutePath)
-        configuration.put(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME, arguments.renderInternalDiagnosticNames)
 
         try {
-            KonanDriver(project, environment, configuration).run()
+            runKonanDriver(configuration, environment, rootDisposable)
         } catch (e: Throwable) {
             if (e is KonanCompilationException || e is CompilationErrorException)
                 return ExitCode.COMPILATION_ERROR
@@ -109,6 +93,53 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
         return ExitCode.OK
     }
 
+    private fun prepareEnvironment(
+            arguments: K2NativeCompilerArguments,
+            configuration: CompilerConfiguration,
+            rootDisposable: Disposable
+    ): KotlinCoreEnvironment {
+        val environment = KotlinCoreEnvironment.createForProduction(rootDisposable,
+                configuration, EnvironmentConfigFiles.NATIVE_CONFIG_FILES)
+
+        configuration.put(CLIConfigurationKeys.FLEXIBLE_PHASE_CONFIG, createFlexiblePhaseConfig(arguments))
+
+        /* Set default version of metadata version */
+        val metadataVersionString = arguments.metadataVersion
+        if (metadataVersionString == null) {
+            configuration.put(CommonConfigurationKeys.METADATA_VERSION, KlibMetadataVersion.INSTANCE)
+        }
+
+        val relativePathBases = arguments.relativePathBases
+        if (relativePathBases != null) {
+            configuration.put(CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES, relativePathBases.toList())
+        }
+
+        configuration.put(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH, arguments.normalizeAbsolutePath)
+        configuration.put(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME, arguments.renderInternalDiagnosticNames)
+
+        return environment
+    }
+
+    private fun runKonanDriver(
+            configuration: CompilerConfiguration,
+            environment: KotlinCoreEnvironment,
+            rootDisposable: Disposable
+    ) {
+        val konanDriver = KonanDriver(environment.project, environment, configuration) { args, setupConfiguration ->
+            val spawnedArguments = K2NativeCompilerArguments()
+            parseCommandLineArguments(args, spawnedArguments)
+            val spawnedConfiguration = CompilerConfiguration()
+
+            spawnedConfiguration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY))
+            spawnedConfiguration.setupCommonArguments(spawnedArguments, this::createMetadataVersion)
+            setupPlatformSpecificArgumentsAndServices(spawnedConfiguration, spawnedArguments, Services.EMPTY)
+            spawnedConfiguration.setupConfiguration()
+            val spawnedEnvironment = prepareEnvironment(spawnedArguments, spawnedConfiguration, rootDisposable)
+            runKonanDriver(spawnedConfiguration, spawnedEnvironment, rootDisposable)
+        }
+        konanDriver.run()
+    }
+
     val K2NativeCompilerArguments.isUsefulWithoutFreeArgs: Boolean
         get() = listTargets || listPhases || checkDependencies || !includes.isNullOrEmpty() ||
                 libraryToAddToCache != null || !exportedLibraries.isNullOrEmpty()
@@ -276,12 +307,12 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
                 val libraryToAddToCache = parseLibraryToAddToCache(arguments, configuration, outputKind)
                 if (libraryToAddToCache != null && !arguments.outputName.isNullOrEmpty())
                     configuration.report(ERROR, "${K2NativeCompilerArguments.ADD_CACHE} already implicitly sets output file name")
-                val cacheDirectories = arguments.cacheDirectories.toNonNullList()
                 libraryToAddToCache?.let { put(LIBRARY_TO_ADD_TO_CACHE, it) }
-                put(CACHE_DIRECTORIES, cacheDirectories)
                 put(CACHED_LIBRARIES, parseCachedLibraries(arguments, configuration))
-                val filesToCache = arguments.filesToCache
+                put(CACHE_DIRECTORIES, arguments.cacheDirectories.toNonNullList())
-                filesToCache?.let { put(FILES_TO_CACHE, it.toList()) }
+                put(AUTO_CACHEABLE_FROM, arguments.autoCacheableFrom.toNonNullList())
+                arguments.autoCacheDir?.let { put(AUTO_CACHE_DIR, it) }
+                arguments.filesToCache?.let { put(FILES_TO_CACHE, it.toList()) }
                 put(MAKE_PER_FILE_CACHE, arguments.makePerFileCache)
 
                 parseShortModuleName(arguments, configuration, outputKind)?.let {

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.backend.konan
+
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.target.CompilerOutputKind
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
+import org.jetbrains.kotlin.library.unresolvedDependencies
+import org.jetbrains.kotlin.library.uniqueName
+import org.jetbrains.kotlin.backend.konan.descriptors.isInteropLibrary
+
+class CacheBuilder(
+        val konanConfig: KonanConfig,
+        val spawnCompilation: (List<String>, CompilerConfiguration.() -> Unit) -> Unit
+) {
+    private val configuration = konanConfig.configuration
+    private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!!.map { File(it) }
+
+    fun needToBuild() = konanConfig.isFinalBinary && !konanConfig.optimizationsEnabled && autoCacheableFrom.isNotEmpty()
+
+    fun build() {
+        val allLibraries = konanConfig.resolvedLibraries.getFullList(TopologicalLibraryOrder)
+        val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName }
+        val caches = mutableMapOf<KotlinLibrary, String>()
+
+        allLibraries.forEach librariesLoop@{ library ->
+            konanConfig.cachedLibraries.getLibraryCache(library)?.let {
+                caches[library] = it.rootDirectory
+                return@librariesLoop
+            }
+            val libraryPath = library.libraryFile.absolutePath
+            val isLibraryAutoCacheable = library.isDefault || autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) }
+            if (!isLibraryAutoCacheable)
+                return@librariesLoop
+
+            val dependencies = library.unresolvedDependencies.map { uniqueNameToLibrary[it.path]!! }
+            val dependencyCaches = dependencies.map {
+                caches[it] ?: run {
+                    configuration.report(CompilerMessageSeverity.LOGGING,
+                            "SKIPPING ${library.libraryName} as some of the dependencies aren't cached")
+                    return@librariesLoop
+                }
+            }
+
+            // TODO: Uncomment after making per-file caches default.
+            val makePerFileCache = false//!library.isInteropLibrary()
+            configuration.report(CompilerMessageSeverity.LOGGING, "CACHING ${library.libraryName}")
+            val libraryCacheDirectory = if (library.isDefault)
+                konanConfig.systemCacheDirectory
+            else
+                CachedLibraries.computeVersionedCacheDirectory(konanConfig.autoCacheDirectory, library, allLibraries)
+            val libraryCache = libraryCacheDirectory.child(
+                    if (makePerFileCache)
+                        CachedLibraries.getPerFileCachedLibraryName(library)
+                    else
+                        CachedLibraries.getCachedLibraryName(library)
+            )
+            try {
+                // TODO: Run monolithic cache builds in parallel.
+                libraryCacheDirectory.mkdirs()
+                spawnCompilation(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) {
+                    val libraries = dependencies.filter { !it.isDefault }.map { it.libraryFile.absolutePath }
+                    val cachedLibraries = dependencies.zip(dependencyCaches).associate { it.first.libraryFile.absolutePath to it.second }
+                    configuration.report(CompilerMessageSeverity.LOGGING, "    dependencies:\n        " +
+                            libraries.joinToString("\n        "))
+                    configuration.report(CompilerMessageSeverity.LOGGING, "    caches used:\n        " +
+                            cachedLibraries.entries.joinToString("\n        ") { "${it.key}: ${it.value}" })
+                    configuration.report(CompilerMessageSeverity.LOGGING, "    cache dir: " +
+                            libraryCacheDirectory.absolutePath)
+                    put(KonanConfigKeys.TARGET, konanConfig.target.toString())
+                    put(KonanConfigKeys.DEBUG, konanConfig.debug)
+                    put(KonanConfigKeys.PRODUCE, CompilerOutputKind.STATIC_CACHE)
+                    put(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE, libraryPath)
+                    put(KonanConfigKeys.NODEFAULTLIBS, true)
+                    put(KonanConfigKeys.NOENDORSEDLIBS, true)
+                    put(KonanConfigKeys.NOSTDLIB, true)
+                    put(KonanConfigKeys.PARTIAL_LINKAGE, konanConfig.partialLinkage)
+                    putIfNotNull(KonanConfigKeys.EXTERNAL_DEPENDENCIES, configuration.get(KonanConfigKeys.EXTERNAL_DEPENDENCIES))
+                    put(KonanConfigKeys.LIBRARY_FILES, libraries)
+                    put(KonanConfigKeys.CACHED_LIBRARIES, cachedLibraries)
+                    put(KonanConfigKeys.CACHE_DIRECTORIES, listOf(libraryCacheDirectory.absolutePath))
+                    put(KonanConfigKeys.MAKE_PER_FILE_CACHE, makePerFileCache)
+                }
+                caches[library] = libraryCache.absolutePath
+            } catch (t: Throwable) {
+                configuration.report(CompilerMessageSeverity.LOGGING, "${t.message}\n${t.stackTraceToString()}")
+                configuration.report(CompilerMessageSeverity.WARNING,
+                        "Failed to build cache: ${t.message}\n${t.stackTraceToString()}\n" +
+                                "Falling back to not use cache for ${library.libraryName}")
+
+                libraryCache.deleteRecursively()
+            }
+        }
+    }
+}

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt

@@ -20,6 +20,30 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile
 
+class FileWithFqName(val filePath: String, val fqName: String)
+
+fun KotlinLibrary.getFilesWithFqNames(): List<FileWithFqName> {
+    val fileProtos = Array<ProtoFile>(fileCount()) {
+        ProtoFile.parseFrom(file(it).codedInputStream, ExtensionRegistryLite.newInstance())
+    }
+    return fileProtos.mapIndexed { index, proto ->
+        val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(this, index))
+        FileWithFqName(proto.fileEntry.name, fileReader.deserializeFqName(proto.fqNameList))
+    }
+}
+
+fun KotlinLibrary.getFileFqNames(filePaths: List<String>): List<String> {
+    val fileProtos = Array<ProtoFile>(fileCount()) {
+        ProtoFile.parseFrom(file(it).codedInputStream, ExtensionRegistryLite.newInstance())
+    }
+    val filePathToIndex = fileProtos.withIndex().associate { it.value.fileEntry.name to it.index }
+    return filePaths.map { filePath ->
+        val index = filePathToIndex[filePath] ?: error("No file with path $filePath is found in klib $libraryName")
+        val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(this, index))
+        fileReader.deserializeFqName(fileProtos[index].fqNameList)
+    }
+}
+
 sealed class CacheDeserializationStrategy {
     abstract fun contains(filePath: String): Boolean
     abstract fun contains(fqName: FqName, fileName: String): Boolean
@@ -58,6 +82,8 @@ class CacheSupport(
         private val configuration: CompilerConfiguration,
         resolvedLibraries: KotlinLibraryResolveResult,
         ignoreCacheReason: String?,
+        systemCacheDirectory: File,
+        autoCacheDirectory: File,
         target: KonanTarget,
         val produce: CompilerOutputKind
 ) {
@@ -66,12 +92,20 @@ class CacheSupport(
     // TODO: consider using [FeaturedLibraries.kt].
     private val fileToLibrary = allLibraries.associateBy { it.libraryFile }
 
-    private val implicitCacheDirectories = configuration.get(KonanConfigKeys.CACHE_DIRECTORIES)!!
+    private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!!
             .map {
                 File(it).takeIf { it.isDirectory }
-                        ?: configuration.reportCompilationError("cache directory $it is not found or not a directory")
+                        ?: configuration.reportCompilationError("auto cacheable root $it is not found or is not a directory")
             }
 
+    private val implicitCacheDirectories = buildList {
+        configuration.get(KonanConfigKeys.CACHE_DIRECTORIES)!!.forEach {
+            add(File(it).takeIf { it.isDirectory }
+                    ?: configuration.reportCompilationError("cache directory $it is not found or is not a directory"))
+        }
+        systemCacheDirectory.takeIf { autoCacheableFrom.isNotEmpty() }?.let { add(it) }
+    }
+
     internal fun tryGetImplicitOutput(cacheDeserializationStrategy: CacheDeserializationStrategy?): String? {
         val libraryToCache = libraryToCache ?: return null
         // Put the resulting library in the first cache directory.
@@ -111,7 +145,9 @@ class CacheSupport(
                 target = target,
                 allLibraries = allLibraries,
                 explicitCaches = if (ignoreCachedLibraries) emptyMap() else explicitCaches,
-                implicitCacheDirectories = if (ignoreCachedLibraries) emptyList() else implicitCacheDirectories
+                implicitCacheDirectories = if (ignoreCachedLibraries) emptyList() else implicitCacheDirectories,
+                autoCacheDirectory = autoCacheDirectory,
+                autoCacheableFrom = if (ignoreCachedLibraries) emptyList() else autoCacheableFrom
         )
     }
 
@@ -132,18 +168,8 @@ class CacheSupport(
 
             val strategy = if (filesToCache.isNullOrEmpty())
                 CacheDeserializationStrategy.WholeModule
-            else {
+            else
-                val fileProtos = Array<ProtoFile>(libraryToAddToCache.fileCount()) {
+                CacheDeserializationStrategy.MultipleFiles(filesToCache, libraryToAddToCache.getFileFqNames(filesToCache))
-                    ProtoFile.parseFrom(libraryToAddToCache.file(it).codedInputStream, ExtensionRegistryLite.newInstance())
-                }
-                val fileNameToIndex = fileProtos.withIndex().associate { it.value.fileEntry.name to it.index }
-                val fqNames = filesToCache.map { fileName ->
-                    val index = fileNameToIndex[fileName] ?: error("No file with path $fileName is found in klib ${libraryToAddToCache.libraryName}")
-                    val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(libraryToAddToCache, index))
-                    fileReader.deserializeFqName(fileProtos[index].fqNameList)
-                }
-                CacheDeserializationStrategy.MultipleFiles(filesToCache, fqNames)
-            }
             PartialCacheInfo(libraryToAddToCache, strategy)
         }
     }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt

@@ -13,6 +13,16 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.uniqueName
+import org.jetbrains.kotlin.library.unresolvedDependencies
+import java.security.MessageDigest
+
+private fun MessageDigest.digestFile(file: File) =
+        if (file.isDirectory) digestDirectory(file) else update(file.readBytes())
+
+private fun MessageDigest.digestDirectory(directory: File): Unit =
+        directory.listFiles.sortedBy { it.name }.forEach { digestFile(it) }
+
+private fun MessageDigest.digestLibrary(library: KotlinLibrary) = digestFile(library.libraryFile)
 
 private fun getArtifactName(target: KonanTarget, baseName: String, kind: CompilerOutputKind) =
         "${kind.prefix(target)}$baseName${kind.suffix(target)}"
@@ -21,11 +31,13 @@ class CachedLibraries(
         private val target: KonanTarget,
         allLibraries: List<KotlinLibrary>,
         explicitCaches: Map<KotlinLibrary, String>,
-        implicitCacheDirectories: List<File>
+        implicitCacheDirectories: List<File>,
+        autoCacheDirectory: File,
+        autoCacheableFrom: List<File>
 ) {
     enum class Kind { DYNAMIC, STATIC }
 
-    sealed class Cache(protected val target: KonanTarget, val kind: Kind, val path: String) {
+    sealed class Cache(protected val target: KonanTarget, val kind: Kind, val path: String, val rootDirectory: String) {
         val bitcodeDependencies by lazy { computeBitcodeDependencies() }
         val binariesPaths by lazy { computeBinariesPaths() }
         val serializedInlineFunctionBodies by lazy { computeSerializedInlineFunctionBodies() }
@@ -43,7 +55,9 @@ class CachedLibraries(
             Kind.STATIC -> CompilerOutputKind.STATIC_CACHE
         }
 
-        class Monolithic(target: KonanTarget, kind: Kind, path: String) : Cache(target, kind, path) {
+        class Monolithic(target: KonanTarget, kind: Kind, path: String)
+            : Cache(target, kind, path, File(path).parentFile.parentFile.absolutePath)
+        {
             override fun computeBitcodeDependencies(): List<DependenciesTracker.UnresolvedDependency> {
                 val directory = File(path).absoluteFile.parentFile
                 val data = directory.child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings()
@@ -71,9 +85,9 @@ class CachedLibraries(
             }
         }
 
-        class PerFile(target: KonanTarget, kind: Kind, path: String) : Cache(target, kind, path) {
+        class PerFile(target: KonanTarget, kind: Kind, path: String, private val fileDirs: List<File>)
-            private val fileDirs by lazy { File(path).listFiles.filter { it.isDirectory }.sortedBy { it.name } }
+            : Cache(target, kind, path, File(path).absolutePath)
-
+        {
             private val perFileBitcodeDependencies by lazy {
                 fileDirs.associate {
                     val data = it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings()
@@ -120,6 +134,7 @@ class CachedLibraries(
     }
 
     private val cacheDirsContents = mutableMapOf<String, Set<String>>()
+    private val librariesFileDirs = mutableMapOf<KotlinLibrary, List<File>>()
 
     private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? {
         // See Linker.renameOutput why is it ok to have an empty cache directory.
@@ -141,7 +156,13 @@ class CachedLibraries(
         return when {
             dynamicFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.DYNAMIC, dynamicFile.absolutePath)
             staticFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.STATIC, staticFile.absolutePath)
-            else -> Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath)
+            else -> {
+                val libraryFileDirs = librariesFileDirs.getOrPut(library) {
+                    library.getFilesWithFqNames().map { cacheDir.child(CacheSupport.cacheFileId(it.fqName, it.filePath)) }
+                }
+                Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath, libraryFileDirs)
+                        .takeIf { cacheDirContents.containsAll(libraryFileDirs.map { it.absolutePath }) }
+            }
         }
     }
 
@@ -152,9 +173,16 @@ class CachedLibraries(
             selectCache(library, File(explicitPath))
                     ?: error("No cache found for library ${library.libraryName} at $explicitPath")
         } else {
-            implicitCacheDirectories.firstNotNullOfOrNull { dir ->
+            val libraryPath = library.libraryFile.absolutePath
+            if (autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) }) {
+                val dir = computeVersionedCacheDirectory(autoCacheDirectory, library, allLibraries)
                 selectCache(library, dir.child(getPerFileCachedLibraryName(library)))
                         ?: selectCache(library, dir.child(getCachedLibraryName(library)))
+            } else {
+                implicitCacheDirectories.firstNotNullOfOrNull { dir ->
+                    selectCache(library, dir.child(getPerFileCachedLibraryName(library)))
+                            ?: selectCache(library, dir.child(getCachedLibraryName(library)))
+                }
             }
         }
 
@@ -186,6 +214,37 @@ class CachedLibraries(
         fun getCachedLibraryName(library: KotlinLibrary): String = getCachedLibraryName(library.uniqueName)
         fun getCachedLibraryName(libraryName: String): String = "$libraryName-cache"
 
+        private fun getAllDependencies(library: KotlinLibrary, allLibraries: List<KotlinLibrary>): List<KotlinLibrary> {
+            val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName }
+            val allDependencies = mutableSetOf<KotlinLibrary>()
+
+            fun traverseDependencies(library: KotlinLibrary) {
+                library.unresolvedDependencies.forEach {
+                    val dependency = uniqueNameToLibrary[it.path]!!
+                    if (dependency !in allDependencies) {
+                        allDependencies += dependency
+                        traverseDependencies(dependency)
+                    }
+                }
+            }
+
+            traverseDependencies(library)
+            return allDependencies.toList()
+        }
+
+        @OptIn(ExperimentalUnsignedTypes::class)
+        fun computeVersionedCacheDirectory(baseCacheDirectory: File, library: KotlinLibrary, allLibraries: List<KotlinLibrary>): File {
+            val dependencies = getAllDependencies(library, allLibraries)
+            val messageDigest = MessageDigest.getInstance("SHA-256")
+            messageDigest.digestLibrary(library)
+            dependencies.sortedBy { it.uniqueName }.forEach { messageDigest.digestLibrary(it) }
+
+            val version = library.versions.libraryVersion ?: "unspecified"
+            val hashString = messageDigest.digest().asUByteArray()
+                    .joinToString("") { it.toString(radix = 16).padStart(2, '0') }
+            return baseCacheDirectory.child(library.uniqueName).child(version).child(hashString)
+        }
+
         const val PER_FILE_CACHE_IR_LEVEL_DIR_NAME = "ir"
         const val PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME = "bin"
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -389,7 +389,30 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
     internal val useDebugInfoInNativeLibs= configuration.get(BinaryOptions.stripDebugInfoFromNativeLibs) == false
 
+    internal val partialLinkage = configuration.get(KonanConfigKeys.PARTIAL_LINKAGE) == true
+
+    internal val additionalCacheFlags by lazy { platformManager.loader(target).additionalCacheFlags }
+
+    private val systemCacheFlavorString = buildString {
+        append(target.toString())
+        if (debug) append("-g")
+        append("STATIC")
+    }
+
+    private val userCacheFlavorString = buildString {
+        append(target.toString())
+        if (debug) append("-g")
+        if (partialLinkage) append("-pl")
+        append("STATIC")
+    }
+
+    private val systemCacheRootDirectory = File(distribution.konanHome).child("klib").child("cache")
+    internal val systemCacheDirectory = systemCacheRootDirectory.child(systemCacheFlavorString)
+    private val autoCacheRootDirectory = configuration.get(KonanConfigKeys.AUTO_CACHE_DIR)?.let { File(it) } ?: systemCacheRootDirectory
+    internal val autoCacheDirectory = autoCacheRootDirectory.child(userCacheFlavorString)
+
     internal val cacheSupport = run {
+        // TODO: Take some of these flags as part of cache meta-directory name.
         val ignoreCacheReason = when {
             optimizationsEnabled -> "for optimized compilation"
             memoryModel != defaultMemoryModel -> "with ${memoryModel.name.lowercase()} memory model"
@@ -412,6 +435,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
                 configuration = configuration,
                 resolvedLibraries = resolvedLibraries,
                 ignoreCacheReason = ignoreCacheReason,
+                systemCacheDirectory = systemCacheDirectory,
+                autoCacheDirectory = autoCacheDirectory,
                 target = target,
                 produce = produce
         )

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt

@@ -35,6 +35,10 @@ class KonanConfigKeys {
                 = CompilerConfigurationKey.create<String?>("path to library that to be added to cache")
         val CACHE_DIRECTORIES: CompilerConfigurationKey<List<String>>
                 = CompilerConfigurationKey.create<List<String>>("paths to directories containing caches")
+        val AUTO_CACHEABLE_FROM: CompilerConfigurationKey<List<String>>
+                = CompilerConfigurationKey.create<List<String>>("paths to the root directories from which dependencies are to be cached automatically")
+        val AUTO_CACHE_DIR: CompilerConfigurationKey<String>
+                = CompilerConfigurationKey.create<String>("path to the directory where to put caches for auto-cacheable dependencies")
         val CACHED_LIBRARIES: CompilerConfigurationKey<Map<String, String>>
                 = CompilerConfigurationKey.create<Map<String, String>>("mapping from library paths to cache paths")
         val FILES_TO_CACHE: CompilerConfigurationKey<List<String>>

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt

@@ -16,7 +16,12 @@ import org.jetbrains.kotlin.konan.library.impl.createKonanLibrary
 import org.jetbrains.kotlin.library.uniqueName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 
-class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment, val configuration: CompilerConfiguration) {
+class KonanDriver(
+        val project: Project,
+        val environment: KotlinCoreEnvironment,
+        val configuration: CompilerConfiguration,
+        val spawnCompilation: (List<String>, CompilerConfiguration.() -> Unit) -> Unit
+) {
     fun run() {
         val fileNames = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { libPath ->
             if (configuration.get(KonanConfigKeys.MAKE_PER_FILE_CACHE) != true)
@@ -33,7 +38,7 @@ class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment,
             configuration.put(KonanConfigKeys.MAKE_PER_FILE_CACHE, true)
             configuration.put(KonanConfigKeys.FILES_TO_CACHE, fileNames)
         }
-        val konanConfig = KonanConfig(project, configuration)
+        var konanConfig = KonanConfig(project, configuration)
 
         if (configuration.get(KonanConfigKeys.LIST_TARGETS) == true) {
             konanConfig.targetManager.list()
@@ -41,6 +46,13 @@ class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment,
         if (konanConfig.infoArgsOnly) return
 
         ensureModuleName(konanConfig)
+
+        val cacheBuilder = CacheBuilder(konanConfig, spawnCompilation)
+        if (cacheBuilder.needToBuild()) {
+            cacheBuilder.build()
+            konanConfig = KonanConfig(project, configuration) // TODO: Just set freshly built caches.
+        }
+
         DynamicCompilerDriver().run(konanConfig, environment)
     }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2782,7 +2782,7 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
                 }
 
                 val cache = context.config.cachedLibraries.getLibraryCache(library)
-                        ?: error("Library $library is expected to be cached")
+                        ?: error("Library ${library.libraryFile} is expected to be cached")
 
                 when (cache) {
                     is CachedLibraries.Cache.Monolithic -> listOf(addCtorFunction(ctorName))