From cf8fcda3ddef6de93d7ce132eef8a1f7f0963672 Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Thu, 11 Jun 2020 23:34:57 +0500 Subject: [PATCH] [caches] Added saving actual dependencies for caches It is bad to link more libraries than it is really needed. For the whole world compilation there is LllvmImports, which tracks all dependencies on bitcode level, but there has not been such a mechanism for compilations against cached libraries. --- .../kotlin/backend/konan/CachedLibraries.kt | 6 +++ .../kotlin/backend/konan/KonanConfig.kt | 2 +- .../jetbrains/kotlin/backend/konan/Linker.kt | 50 ++++++++++++------- .../kotlin/backend/konan/OutputFiles.kt | 5 ++ .../kotlin/backend/konan/llvm/ContextUtils.kt | 41 +++++++++++++-- .../kotlin/backend/konan/llvm/IrToBitcode.kt | 2 +- 6 files changed, 84 insertions(+), 22 deletions(-) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt index 59f5630541c..ebfe168967a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -21,6 +21,11 @@ class CachedLibraries( class Cache(val kind: Kind, val path: String) { enum class Kind { DYNAMIC, STATIC } + + val bitcodeDependencies by lazy { + val directory = File(path).absoluteFile.parent + File(directory, BITCODE_DEPENDENCIES_FILE_NAME).readStrings() + } } private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? { @@ -81,5 +86,6 @@ class CachedLibraries( companion object { fun getCachedLibraryName(library: KotlinLibrary): String = getCachedLibraryName(library.uniqueName) fun getCachedLibraryName(libraryName: String): String = "$libraryName-cache" + const val BITCODE_DEPENDENCIES_FILE_NAME = "bitcode_deps" } } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 59eca4b9ffd..11cf2fcbd50 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -79,7 +79,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val resolvedLibraries get() = resolve.resolvedLibraries - private val cacheSupport = CacheSupport(configuration, resolvedLibraries, target, produce) + internal val cacheSupport = CacheSupport(configuration, resolvedLibraries, target, produce) internal val cachedLibraries: CachedLibraries get() = cacheSupport.cachedLibraries diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt index 25d787bfc98..7bef79c7a2d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt @@ -6,6 +6,9 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.Family import org.jetbrains.kotlin.konan.target.LinkerOutputKind import org.jetbrains.kotlin.konan.library.KonanLibrary +import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder +import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.utils.addToStdlib.cast internal fun determineLinkerOutput(context: Context): LinkerOutputKind = when (context.config.produce) { @@ -44,13 +47,33 @@ internal class Linker(val context: Context) { val libraryProvidedLinkerFlags = context.llvm.allNativeDependencies.map { it.linkerOpts }.flatten() - if (context.config.produce.isCache) + if (context.config.produce.isCache) { context.config.outputFiles.tempCacheDirectory!!.mkdirs() + saveAdditionalInfoForCache() + } + runLinker(objectFiles, includedBinaries, libraryProvidedLinkerFlags) renameOutput() } + private fun saveAdditionalInfoForCache() { + saveCacheBitcodeDependencies() + } + + private fun saveCacheBitcodeDependencies() { + val outputFiles = context.config.outputFiles + val bitcodeDependenciesFile = File(outputFiles.bitcodeDependenciesFile!!) + val bitcodeDependencies = context.config.resolvedLibraries + .getFullList(TopologicalLibraryOrder) + .filter { + require(it is KonanLibrary) + context.llvmImports.bitcodeIsUsed(it) + && it !in context.config.cacheSupport.librariesToCache // Skip loops. + }.cast>() + bitcodeDependenciesFile.writeLines(bitcodeDependencies.map { it.uniqueName }) + } + private fun renameOutput() { if (context.config.produce.isCache) { val outputFiles = context.config.outputFiles @@ -155,28 +178,21 @@ private fun determineCachesToLink(context: Context): CachesToLink { val staticCaches = mutableListOf() val dynamicCaches = mutableListOf() - // TODO: suboptimal, see e.g. [LlvmImports]. - context.librariesWithDependencies.forEach { library -> + context.llvm.allCachedBitcodeDependencies.forEach { library -> val currentBinaryContainsLibrary = context.llvmModuleSpecification.containsLibrary(library) val cache = context.config.cachedLibraries.getLibraryCache(library) - val libraryIsCached = cache != null + ?: error("Library $library is expected to be cached") // Consistency check. Generally guaranteed by implementation. - if (currentBinaryContainsLibrary && libraryIsCached) { + if (currentBinaryContainsLibrary) error("Library ${library.libraryName} is found in both cache and current binary") - } else if (!currentBinaryContainsLibrary && !libraryIsCached) { - error("Library ${library.libraryName} is not found neither in cache nor in current binary") + + val list = when (cache.kind) { + CachedLibraries.Cache.Kind.DYNAMIC -> dynamicCaches + CachedLibraries.Cache.Kind.STATIC -> staticCaches } - if (cache != null) { - val list = when (cache.kind) { - CachedLibraries.Cache.Kind.DYNAMIC -> dynamicCaches - CachedLibraries.Cache.Kind.STATIC -> staticCaches - } - - list += cache.path - } + list += cache.path } - - return CachesToLink(static = staticCaches.distinct(), dynamic = dynamicCaches.distinct()) + return CachesToLink(static = staticCaches, dynamic = dynamicCaches) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt index 753042dcc7c..e592ffbf9a5 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt @@ -51,6 +51,11 @@ class OutputFiles(outputPath: String?, target: KonanTarget, val produce: Compile val symbolicInfoFile = "$nativeBinaryFile.dSYM" + val bitcodeDependenciesFile = + if (produce.isCache) + tempCacheDirectory!!.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME).absolutePath + else null + private fun String.fullOutputName() = prefixBaseNameIfNeeded(prefix).suffixIfNeeded(suffix) private fun String.prefixBaseNameIfNeeded(prefix: String) = diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index d438ff14f1a..0ccdfd5a787 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -10,6 +10,8 @@ import kotlinx.cinterop.get import kotlinx.cinterop.memScoped import kotlinx.cinterop.toKString import llvm.* +import org.jetbrains.kotlin.backend.konan.CachedLibraries +import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.backend.konan.Context import org.jetbrains.kotlin.backend.konan.hash.GlobalHash import org.jetbrains.kotlin.backend.konan.ir.llvmSymbolOrigin @@ -24,6 +26,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibrary import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder +import org.jetbrains.kotlin.library.uniqueName import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.utils.addToStdlib.cast @@ -395,14 +398,46 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { require(it is KonanLibrary) (!it.isDefault && !context.config.purgeUserLibs) || imports.nativeDependenciesAreUsed(it) }.cast>() + } + private val immediateBitcodeDependencies: List by lazy { + context.config.resolvedLibraries.getFullList(TopologicalLibraryOrder).cast>() + .filter { (!it.isDefault && !context.config.purgeUserLibs) || imports.bitcodeIsUsed(it) } + } + + val allCachedBitcodeDependencies: List by lazy { + val allLibraries = context.config.resolvedLibraries.getFullList().associateBy { it.uniqueName } + val result = mutableSetOf() + + fun addDependencies(cachedLibrary: CachedLibraries.Cache) { + cachedLibrary.bitcodeDependencies.forEach { + val library = allLibraries[it] ?: error("Bitcode dependency to an unknown library: $it") + result.add(library as KonanLibrary) + addDependencies(context.config.cachedLibraries.getLibraryCache(library) + ?: error("Library $it is expected to be cached")) + } + } + + for (library in immediateBitcodeDependencies) { + val cache = context.config.cachedLibraries.getLibraryCache(library) + if (cache != null) { + result += library + addDependencies(cache) + } + } + + result.toList() } val allNativeDependencies: List by lazy { - val cachedLibraries = context.librariesWithDependencies.filter { - context.config.cachedLibraries.isLibraryCached(it) + (nativeDependenciesToLink + allCachedBitcodeDependencies).distinct() + } + + val allBitcodeDependencies: List by lazy { + val allNonCachedDependencies = context.librariesWithDependencies.filter { + context.config.cachedLibraries.getLibraryCache(it) == null } - (nativeDependenciesToLink + cachedLibraries).distinct() + (allNonCachedDependencies + allCachedBitcodeDependencies).distinct() } val bitcodeToLink: List by lazy { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 8e8a1ed741b..b6782b4d60a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -2375,7 +2375,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map()