From 9c86532eb0d30cba8191c501d8d2b0838aa42b48 Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Fri, 25 Nov 2022 12:49:16 +0200 Subject: [PATCH] [K/N] Implemented cache orchestration machinery in the compiler --- .../arguments/K2NativeCompilerArguments.kt | 17 +++ .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 81 +++++++++----- .../kotlin/backend/konan/CacheBuilder.kt | 101 ++++++++++++++++++ .../kotlin/backend/konan/CacheSupport.kt | 56 +++++++--- .../kotlin/backend/konan/CachedLibraries.kt | 75 +++++++++++-- .../kotlin/backend/konan/KonanConfig.kt | 25 +++++ .../backend/konan/KonanConfigurationKeys.kt | 4 + .../kotlin/backend/konan/KonanDriver.kt | 16 ++- .../kotlin/backend/konan/llvm/IrToBitcode.kt | 2 +- 9 files changed, 326 insertions(+), 51 deletions(-) create mode 100644 kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt index 3a33033f735..e458b5e3e19 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt @@ -141,6 +141,23 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { ) var cachedLibraries: Array? = null + @Argument( + value = "-Xauto-cache-from", + valueDescription = "", + description = "Path to the root directory from which dependencies are to be cached automatically.\n" + + "By default caches will be placed into the kotlin-native system cache directory.", + delimiter = "" + ) + var autoCacheableFrom: Array? = null + + @Argument( + value = "-Xauto-cache-dir", + valueDescription = "", + description = "Path to the directory where to put caches for auto-cacheable dependencies", + delimiter = "" + ) + var autoCacheDir: String? = null + @Argument(value="-Xcheck-dependencies", deprecatedName = "--check_dependencies", description = "Check dependencies and download the missing ones") var checkDependencies: Boolean = false diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 2ec59dd0d0e..005df6a05e0 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -13,6 +13,7 @@ import org.jetbrains.kotlin.analyzer.CompilationErrorException import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.cli.common.* import org.jetbrains.kotlin.cli.common.arguments.K2NativeCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoot import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.* @@ -57,40 +58,23 @@ class K2Native : CLICompiler() { } val pluginLoadResult = - PluginCliParser.loadPluginsSafe(arguments.pluginClasspaths, arguments.pluginOptions, arguments.pluginConfigurations, configuration) + PluginCliParser.loadPluginsSafe(arguments.pluginClasspaths, arguments.pluginOptions, arguments.pluginConfigurations, configuration) if (pluginLoadResult != ExitCode.OK) return pluginLoadResult - val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, - configuration, EnvironmentConfigFiles.NATIVE_CONFIG_FILES) - val project = environment.project val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) ?: MessageCollector.NONE if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR)) { messageCollector.report(WARNING, "New compiler pipeline K2 is experimental in Kotlin/Native. No compatibility guarantees are yet provided") } - configuration.put(CLIConfigurationKeys.FLEXIBLE_PHASE_CONFIG, createFlexiblePhaseConfig(arguments)) - val enoughArguments = arguments.freeArgs.isNotEmpty() || arguments.isUsefulWithoutFreeArgs if (!enoughArguments) { - configuration.report(ERROR, "You have not specified any compilation arguments. No output has been produced.") + messageCollector.report(ERROR, "You have not specified any compilation arguments. No output has been produced.") } - /* Set default version of metadata version */ - val metadataVersionString = arguments.metadataVersion - if (metadataVersionString == null) { - configuration.put(CommonConfigurationKeys.METADATA_VERSION, KlibMetadataVersion.INSTANCE) - } - - val relativePathBases = arguments.relativePathBases - if (relativePathBases != null) { - configuration.put(CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES, relativePathBases.toList()) - } - - configuration.put(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH, arguments.normalizeAbsolutePath) - configuration.put(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME, arguments.renderInternalDiagnosticNames) + val environment = prepareEnvironment(arguments, configuration, rootDisposable) try { - KonanDriver(project, environment, configuration).run() + runKonanDriver(configuration, environment, rootDisposable) } catch (e: Throwable) { if (e is KonanCompilationException || e is CompilationErrorException) return ExitCode.COMPILATION_ERROR @@ -109,6 +93,53 @@ class K2Native : CLICompiler() { return ExitCode.OK } + private fun prepareEnvironment( + arguments: K2NativeCompilerArguments, + configuration: CompilerConfiguration, + rootDisposable: Disposable + ): KotlinCoreEnvironment { + val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, + configuration, EnvironmentConfigFiles.NATIVE_CONFIG_FILES) + + configuration.put(CLIConfigurationKeys.FLEXIBLE_PHASE_CONFIG, createFlexiblePhaseConfig(arguments)) + + /* Set default version of metadata version */ + val metadataVersionString = arguments.metadataVersion + if (metadataVersionString == null) { + configuration.put(CommonConfigurationKeys.METADATA_VERSION, KlibMetadataVersion.INSTANCE) + } + + val relativePathBases = arguments.relativePathBases + if (relativePathBases != null) { + configuration.put(CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES, relativePathBases.toList()) + } + + configuration.put(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH, arguments.normalizeAbsolutePath) + configuration.put(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME, arguments.renderInternalDiagnosticNames) + + return environment + } + + private fun runKonanDriver( + configuration: CompilerConfiguration, + environment: KotlinCoreEnvironment, + rootDisposable: Disposable + ) { + val konanDriver = KonanDriver(environment.project, environment, configuration) { args, setupConfiguration -> + val spawnedArguments = K2NativeCompilerArguments() + parseCommandLineArguments(args, spawnedArguments) + val spawnedConfiguration = CompilerConfiguration() + + spawnedConfiguration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)) + spawnedConfiguration.setupCommonArguments(spawnedArguments, this::createMetadataVersion) + setupPlatformSpecificArgumentsAndServices(spawnedConfiguration, spawnedArguments, Services.EMPTY) + spawnedConfiguration.setupConfiguration() + val spawnedEnvironment = prepareEnvironment(spawnedArguments, spawnedConfiguration, rootDisposable) + runKonanDriver(spawnedConfiguration, spawnedEnvironment, rootDisposable) + } + konanDriver.run() + } + val K2NativeCompilerArguments.isUsefulWithoutFreeArgs: Boolean get() = listTargets || listPhases || checkDependencies || !includes.isNullOrEmpty() || libraryToAddToCache != null || !exportedLibraries.isNullOrEmpty() @@ -276,12 +307,12 @@ class K2Native : CLICompiler() { val libraryToAddToCache = parseLibraryToAddToCache(arguments, configuration, outputKind) if (libraryToAddToCache != null && !arguments.outputName.isNullOrEmpty()) configuration.report(ERROR, "${K2NativeCompilerArguments.ADD_CACHE} already implicitly sets output file name") - val cacheDirectories = arguments.cacheDirectories.toNonNullList() libraryToAddToCache?.let { put(LIBRARY_TO_ADD_TO_CACHE, it) } - put(CACHE_DIRECTORIES, cacheDirectories) put(CACHED_LIBRARIES, parseCachedLibraries(arguments, configuration)) - val filesToCache = arguments.filesToCache - filesToCache?.let { put(FILES_TO_CACHE, it.toList()) } + put(CACHE_DIRECTORIES, arguments.cacheDirectories.toNonNullList()) + put(AUTO_CACHEABLE_FROM, arguments.autoCacheableFrom.toNonNullList()) + arguments.autoCacheDir?.let { put(AUTO_CACHE_DIR, it) } + arguments.filesToCache?.let { put(FILES_TO_CACHE, it.toList()) } put(MAKE_PER_FILE_CACHE, arguments.makePerFileCache) parseShortModuleName(arguments, configuration, outputKind)?.let { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt new file mode 100644 index 00000000000..b49e6789122 --- /dev/null +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.konan + +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.target.CompilerOutputKind +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder +import org.jetbrains.kotlin.library.unresolvedDependencies +import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.backend.konan.descriptors.isInteropLibrary + +class CacheBuilder( + val konanConfig: KonanConfig, + val spawnCompilation: (List, CompilerConfiguration.() -> Unit) -> Unit +) { + private val configuration = konanConfig.configuration + private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!!.map { File(it) } + + fun needToBuild() = konanConfig.isFinalBinary && !konanConfig.optimizationsEnabled && autoCacheableFrom.isNotEmpty() + + fun build() { + val allLibraries = konanConfig.resolvedLibraries.getFullList(TopologicalLibraryOrder) + val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName } + val caches = mutableMapOf() + + allLibraries.forEach librariesLoop@{ library -> + konanConfig.cachedLibraries.getLibraryCache(library)?.let { + caches[library] = it.rootDirectory + return@librariesLoop + } + val libraryPath = library.libraryFile.absolutePath + val isLibraryAutoCacheable = library.isDefault || autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) } + if (!isLibraryAutoCacheable) + return@librariesLoop + + val dependencies = library.unresolvedDependencies.map { uniqueNameToLibrary[it.path]!! } + val dependencyCaches = dependencies.map { + caches[it] ?: run { + configuration.report(CompilerMessageSeverity.LOGGING, + "SKIPPING ${library.libraryName} as some of the dependencies aren't cached") + return@librariesLoop + } + } + + // TODO: Uncomment after making per-file caches default. + val makePerFileCache = false//!library.isInteropLibrary() + configuration.report(CompilerMessageSeverity.LOGGING, "CACHING ${library.libraryName}") + val libraryCacheDirectory = if (library.isDefault) + konanConfig.systemCacheDirectory + else + CachedLibraries.computeVersionedCacheDirectory(konanConfig.autoCacheDirectory, library, allLibraries) + val libraryCache = libraryCacheDirectory.child( + if (makePerFileCache) + CachedLibraries.getPerFileCachedLibraryName(library) + else + CachedLibraries.getCachedLibraryName(library) + ) + try { + // TODO: Run monolithic cache builds in parallel. + libraryCacheDirectory.mkdirs() + spawnCompilation(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) { + val libraries = dependencies.filter { !it.isDefault }.map { it.libraryFile.absolutePath } + val cachedLibraries = dependencies.zip(dependencyCaches).associate { it.first.libraryFile.absolutePath to it.second } + configuration.report(CompilerMessageSeverity.LOGGING, " dependencies:\n " + + libraries.joinToString("\n ")) + configuration.report(CompilerMessageSeverity.LOGGING, " caches used:\n " + + cachedLibraries.entries.joinToString("\n ") { "${it.key}: ${it.value}" }) + configuration.report(CompilerMessageSeverity.LOGGING, " cache dir: " + + libraryCacheDirectory.absolutePath) + put(KonanConfigKeys.TARGET, konanConfig.target.toString()) + put(KonanConfigKeys.DEBUG, konanConfig.debug) + put(KonanConfigKeys.PRODUCE, CompilerOutputKind.STATIC_CACHE) + put(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE, libraryPath) + put(KonanConfigKeys.NODEFAULTLIBS, true) + put(KonanConfigKeys.NOENDORSEDLIBS, true) + put(KonanConfigKeys.NOSTDLIB, true) + put(KonanConfigKeys.PARTIAL_LINKAGE, konanConfig.partialLinkage) + putIfNotNull(KonanConfigKeys.EXTERNAL_DEPENDENCIES, configuration.get(KonanConfigKeys.EXTERNAL_DEPENDENCIES)) + put(KonanConfigKeys.LIBRARY_FILES, libraries) + put(KonanConfigKeys.CACHED_LIBRARIES, cachedLibraries) + put(KonanConfigKeys.CACHE_DIRECTORIES, listOf(libraryCacheDirectory.absolutePath)) + put(KonanConfigKeys.MAKE_PER_FILE_CACHE, makePerFileCache) + } + caches[library] = libraryCache.absolutePath + } catch (t: Throwable) { + configuration.report(CompilerMessageSeverity.LOGGING, "${t.message}\n${t.stackTraceToString()}") + configuration.report(CompilerMessageSeverity.WARNING, + "Failed to build cache: ${t.message}\n${t.stackTraceToString()}\n" + + "Falling back to not use cache for ${library.libraryName}") + + libraryCache.deleteRecursively() + } + } + } +} \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt index a208bd1575e..7f058038db5 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt @@ -20,6 +20,30 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile +class FileWithFqName(val filePath: String, val fqName: String) + +fun KotlinLibrary.getFilesWithFqNames(): List { + val fileProtos = Array(fileCount()) { + ProtoFile.parseFrom(file(it).codedInputStream, ExtensionRegistryLite.newInstance()) + } + return fileProtos.mapIndexed { index, proto -> + val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(this, index)) + FileWithFqName(proto.fileEntry.name, fileReader.deserializeFqName(proto.fqNameList)) + } +} + +fun KotlinLibrary.getFileFqNames(filePaths: List): List { + val fileProtos = Array(fileCount()) { + ProtoFile.parseFrom(file(it).codedInputStream, ExtensionRegistryLite.newInstance()) + } + val filePathToIndex = fileProtos.withIndex().associate { it.value.fileEntry.name to it.index } + return filePaths.map { filePath -> + val index = filePathToIndex[filePath] ?: error("No file with path $filePath is found in klib $libraryName") + val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(this, index)) + fileReader.deserializeFqName(fileProtos[index].fqNameList) + } +} + sealed class CacheDeserializationStrategy { abstract fun contains(filePath: String): Boolean abstract fun contains(fqName: FqName, fileName: String): Boolean @@ -58,6 +82,8 @@ class CacheSupport( private val configuration: CompilerConfiguration, resolvedLibraries: KotlinLibraryResolveResult, ignoreCacheReason: String?, + systemCacheDirectory: File, + autoCacheDirectory: File, target: KonanTarget, val produce: CompilerOutputKind ) { @@ -66,12 +92,20 @@ class CacheSupport( // TODO: consider using [FeaturedLibraries.kt]. private val fileToLibrary = allLibraries.associateBy { it.libraryFile } - private val implicitCacheDirectories = configuration.get(KonanConfigKeys.CACHE_DIRECTORIES)!! + private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!! .map { File(it).takeIf { it.isDirectory } - ?: configuration.reportCompilationError("cache directory $it is not found or not a directory") + ?: configuration.reportCompilationError("auto cacheable root $it is not found or is not a directory") } + private val implicitCacheDirectories = buildList { + configuration.get(KonanConfigKeys.CACHE_DIRECTORIES)!!.forEach { + add(File(it).takeIf { it.isDirectory } + ?: configuration.reportCompilationError("cache directory $it is not found or is not a directory")) + } + systemCacheDirectory.takeIf { autoCacheableFrom.isNotEmpty() }?.let { add(it) } + } + internal fun tryGetImplicitOutput(cacheDeserializationStrategy: CacheDeserializationStrategy?): String? { val libraryToCache = libraryToCache ?: return null // Put the resulting library in the first cache directory. @@ -111,7 +145,9 @@ class CacheSupport( target = target, allLibraries = allLibraries, explicitCaches = if (ignoreCachedLibraries) emptyMap() else explicitCaches, - implicitCacheDirectories = if (ignoreCachedLibraries) emptyList() else implicitCacheDirectories + implicitCacheDirectories = if (ignoreCachedLibraries) emptyList() else implicitCacheDirectories, + autoCacheDirectory = autoCacheDirectory, + autoCacheableFrom = if (ignoreCachedLibraries) emptyList() else autoCacheableFrom ) } @@ -132,18 +168,8 @@ class CacheSupport( val strategy = if (filesToCache.isNullOrEmpty()) CacheDeserializationStrategy.WholeModule - else { - val fileProtos = Array(libraryToAddToCache.fileCount()) { - ProtoFile.parseFrom(libraryToAddToCache.file(it).codedInputStream, ExtensionRegistryLite.newInstance()) - } - val fileNameToIndex = fileProtos.withIndex().associate { it.value.fileEntry.name to it.index } - val fqNames = filesToCache.map { fileName -> - val index = fileNameToIndex[fileName] ?: error("No file with path $fileName is found in klib ${libraryToAddToCache.libraryName}") - val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(libraryToAddToCache, index)) - fileReader.deserializeFqName(fileProtos[index].fqNameList) - } - CacheDeserializationStrategy.MultipleFiles(filesToCache, fqNames) - } + else + CacheDeserializationStrategy.MultipleFiles(filesToCache, libraryToAddToCache.getFileFqNames(filesToCache)) PartialCacheInfo(libraryToAddToCache, strategy) } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt index fc99e8293c2..fe12fef41ea 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -13,6 +13,16 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.library.unresolvedDependencies +import java.security.MessageDigest + +private fun MessageDigest.digestFile(file: File) = + if (file.isDirectory) digestDirectory(file) else update(file.readBytes()) + +private fun MessageDigest.digestDirectory(directory: File): Unit = + directory.listFiles.sortedBy { it.name }.forEach { digestFile(it) } + +private fun MessageDigest.digestLibrary(library: KotlinLibrary) = digestFile(library.libraryFile) private fun getArtifactName(target: KonanTarget, baseName: String, kind: CompilerOutputKind) = "${kind.prefix(target)}$baseName${kind.suffix(target)}" @@ -21,11 +31,13 @@ class CachedLibraries( private val target: KonanTarget, allLibraries: List, explicitCaches: Map, - implicitCacheDirectories: List + implicitCacheDirectories: List, + autoCacheDirectory: File, + autoCacheableFrom: List ) { enum class Kind { DYNAMIC, STATIC } - sealed class Cache(protected val target: KonanTarget, val kind: Kind, val path: String) { + sealed class Cache(protected val target: KonanTarget, val kind: Kind, val path: String, val rootDirectory: String) { val bitcodeDependencies by lazy { computeBitcodeDependencies() } val binariesPaths by lazy { computeBinariesPaths() } val serializedInlineFunctionBodies by lazy { computeSerializedInlineFunctionBodies() } @@ -43,7 +55,9 @@ class CachedLibraries( Kind.STATIC -> CompilerOutputKind.STATIC_CACHE } - class Monolithic(target: KonanTarget, kind: Kind, path: String) : Cache(target, kind, path) { + class Monolithic(target: KonanTarget, kind: Kind, path: String) + : Cache(target, kind, path, File(path).parentFile.parentFile.absolutePath) + { override fun computeBitcodeDependencies(): List { val directory = File(path).absoluteFile.parentFile val data = directory.child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings() @@ -71,9 +85,9 @@ class CachedLibraries( } } - class PerFile(target: KonanTarget, kind: Kind, path: String) : Cache(target, kind, path) { - private val fileDirs by lazy { File(path).listFiles.filter { it.isDirectory }.sortedBy { it.name } } - + class PerFile(target: KonanTarget, kind: Kind, path: String, private val fileDirs: List) + : Cache(target, kind, path, File(path).absolutePath) + { private val perFileBitcodeDependencies by lazy { fileDirs.associate { val data = it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings() @@ -120,6 +134,7 @@ class CachedLibraries( } private val cacheDirsContents = mutableMapOf>() + private val librariesFileDirs = mutableMapOf>() private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? { // See Linker.renameOutput why is it ok to have an empty cache directory. @@ -141,7 +156,13 @@ class CachedLibraries( return when { dynamicFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.DYNAMIC, dynamicFile.absolutePath) staticFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.STATIC, staticFile.absolutePath) - else -> Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath) + else -> { + val libraryFileDirs = librariesFileDirs.getOrPut(library) { + library.getFilesWithFqNames().map { cacheDir.child(CacheSupport.cacheFileId(it.fqName, it.filePath)) } + } + Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath, libraryFileDirs) + .takeIf { cacheDirContents.containsAll(libraryFileDirs.map { it.absolutePath }) } + } } } @@ -152,9 +173,16 @@ class CachedLibraries( selectCache(library, File(explicitPath)) ?: error("No cache found for library ${library.libraryName} at $explicitPath") } else { - implicitCacheDirectories.firstNotNullOfOrNull { dir -> + val libraryPath = library.libraryFile.absolutePath + if (autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) }) { + val dir = computeVersionedCacheDirectory(autoCacheDirectory, library, allLibraries) selectCache(library, dir.child(getPerFileCachedLibraryName(library))) ?: selectCache(library, dir.child(getCachedLibraryName(library))) + } else { + implicitCacheDirectories.firstNotNullOfOrNull { dir -> + selectCache(library, dir.child(getPerFileCachedLibraryName(library))) + ?: selectCache(library, dir.child(getCachedLibraryName(library))) + } } } @@ -186,6 +214,37 @@ class CachedLibraries( fun getCachedLibraryName(library: KotlinLibrary): String = getCachedLibraryName(library.uniqueName) fun getCachedLibraryName(libraryName: String): String = "$libraryName-cache" + private fun getAllDependencies(library: KotlinLibrary, allLibraries: List): List { + val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName } + val allDependencies = mutableSetOf() + + fun traverseDependencies(library: KotlinLibrary) { + library.unresolvedDependencies.forEach { + val dependency = uniqueNameToLibrary[it.path]!! + if (dependency !in allDependencies) { + allDependencies += dependency + traverseDependencies(dependency) + } + } + } + + traverseDependencies(library) + return allDependencies.toList() + } + + @OptIn(ExperimentalUnsignedTypes::class) + fun computeVersionedCacheDirectory(baseCacheDirectory: File, library: KotlinLibrary, allLibraries: List): File { + val dependencies = getAllDependencies(library, allLibraries) + val messageDigest = MessageDigest.getInstance("SHA-256") + messageDigest.digestLibrary(library) + dependencies.sortedBy { it.uniqueName }.forEach { messageDigest.digestLibrary(it) } + + val version = library.versions.libraryVersion ?: "unspecified" + val hashString = messageDigest.digest().asUByteArray() + .joinToString("") { it.toString(radix = 16).padStart(2, '0') } + return baseCacheDirectory.child(library.uniqueName).child(version).child(hashString) + } + const val PER_FILE_CACHE_IR_LEVEL_DIR_NAME = "ir" const val PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME = "bin" diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 91c4715f40d..d77c4f680d1 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -389,7 +389,30 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val useDebugInfoInNativeLibs= configuration.get(BinaryOptions.stripDebugInfoFromNativeLibs) == false + internal val partialLinkage = configuration.get(KonanConfigKeys.PARTIAL_LINKAGE) == true + + internal val additionalCacheFlags by lazy { platformManager.loader(target).additionalCacheFlags } + + private val systemCacheFlavorString = buildString { + append(target.toString()) + if (debug) append("-g") + append("STATIC") + } + + private val userCacheFlavorString = buildString { + append(target.toString()) + if (debug) append("-g") + if (partialLinkage) append("-pl") + append("STATIC") + } + + private val systemCacheRootDirectory = File(distribution.konanHome).child("klib").child("cache") + internal val systemCacheDirectory = systemCacheRootDirectory.child(systemCacheFlavorString) + private val autoCacheRootDirectory = configuration.get(KonanConfigKeys.AUTO_CACHE_DIR)?.let { File(it) } ?: systemCacheRootDirectory + internal val autoCacheDirectory = autoCacheRootDirectory.child(userCacheFlavorString) + internal val cacheSupport = run { + // TODO: Take some of these flags as part of cache meta-directory name. val ignoreCacheReason = when { optimizationsEnabled -> "for optimized compilation" memoryModel != defaultMemoryModel -> "with ${memoryModel.name.lowercase()} memory model" @@ -412,6 +435,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration configuration = configuration, resolvedLibraries = resolvedLibraries, ignoreCacheReason = ignoreCacheReason, + systemCacheDirectory = systemCacheDirectory, + autoCacheDirectory = autoCacheDirectory, target = target, produce = produce ) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index e0c8a8fc8a4..c93e65c5cac 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -35,6 +35,10 @@ class KonanConfigKeys { = CompilerConfigurationKey.create("path to library that to be added to cache") val CACHE_DIRECTORIES: CompilerConfigurationKey> = CompilerConfigurationKey.create>("paths to directories containing caches") + val AUTO_CACHEABLE_FROM: CompilerConfigurationKey> + = CompilerConfigurationKey.create>("paths to the root directories from which dependencies are to be cached automatically") + val AUTO_CACHE_DIR: CompilerConfigurationKey + = CompilerConfigurationKey.create("path to the directory where to put caches for auto-cacheable dependencies") val CACHED_LIBRARIES: CompilerConfigurationKey> = CompilerConfigurationKey.create>("mapping from library paths to cache paths") val FILES_TO_CACHE: CompilerConfigurationKey> diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index 1ad7ee6173b..dd4a1ae411e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -16,7 +16,12 @@ import org.jetbrains.kotlin.konan.library.impl.createKonanLibrary import org.jetbrains.kotlin.library.uniqueName import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite -class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment, val configuration: CompilerConfiguration) { +class KonanDriver( + val project: Project, + val environment: KotlinCoreEnvironment, + val configuration: CompilerConfiguration, + val spawnCompilation: (List, CompilerConfiguration.() -> Unit) -> Unit +) { fun run() { val fileNames = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { libPath -> if (configuration.get(KonanConfigKeys.MAKE_PER_FILE_CACHE) != true) @@ -33,7 +38,7 @@ class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment, configuration.put(KonanConfigKeys.MAKE_PER_FILE_CACHE, true) configuration.put(KonanConfigKeys.FILES_TO_CACHE, fileNames) } - val konanConfig = KonanConfig(project, configuration) + var konanConfig = KonanConfig(project, configuration) if (configuration.get(KonanConfigKeys.LIST_TARGETS) == true) { konanConfig.targetManager.list() @@ -41,6 +46,13 @@ class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment, if (konanConfig.infoArgsOnly) return ensureModuleName(konanConfig) + + val cacheBuilder = CacheBuilder(konanConfig, spawnCompilation) + if (cacheBuilder.needToBuild()) { + cacheBuilder.build() + konanConfig = KonanConfig(project, configuration) // TODO: Just set freshly built caches. + } + DynamicCompilerDriver().run(konanConfig, environment) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 50a44073056..bbc55032fa1 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -2782,7 +2782,7 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, } val cache = context.config.cachedLibraries.getLibraryCache(library) - ?: error("Library $library is expected to be cached") + ?: error("Library ${library.libraryFile} is expected to be cached") when (cache) { is CachedLibraries.Cache.Monolithic -> listOf(addCtorFunction(ctorName))