From 0dcf6385198f687b94d764ff4384cd78ba530474 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Thu, 16 Jun 2022 12:27:59 +0300 Subject: [PATCH] [K/N] Add runtime.bc to runtime LTO Fixes KT-52799 --- .../kotlin/backend/konan/CompilerOutput.kt | 77 ++++++++++++++----- .../backend/konan/RuntimeLinkageStrategy.kt | 21 +---- .../backend/konan/llvm/objc/linkObjC.kt | 14 +--- 3 files changed, 62 insertions(+), 50 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt index 151a329a3fe..537d36ee3e6 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt @@ -14,14 +14,16 @@ import org.jetbrains.kotlin.backend.common.phaser.BeforeOrAfter import org.jetbrains.kotlin.backend.common.serialization.KlibIrVersion import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion import org.jetbrains.kotlin.backend.konan.llvm.* -import org.jetbrains.kotlin.backend.konan.llvm.objc.linkObjC +import org.jetbrains.kotlin.backend.konan.llvm.objc.patchObjCRuntimeModule import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.konan.CURRENT import org.jetbrains.kotlin.konan.CompilerVersion import org.jetbrains.kotlin.konan.file.isBitcode +import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME import org.jetbrains.kotlin.konan.library.impl.buildLibrary import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.library.* +import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty /** * Supposed to be true for a single LLVM module within final binary. @@ -75,36 +77,73 @@ internal fun produceCStubs(context: Context) { } } -private fun linkAllDependencies(context: Context, generatedBitcodeFiles: List) { +private val BaseKotlinLibrary.isStdlib: Boolean + get() = uniqueName == KONAN_STDLIB_NAME + + +private data class LlvmModules( + val runtimeModules: List, + val additionalModules: List +) + +/** + * Deserialize, generate, patch all bitcode dependencies and classify them into two sets: + * - Runtime modules. These may be used as an input for a separate LTO (e.g. for debug builds). + * - Everything else. + */ +private fun collectLlvmModules(context: Context, generatedBitcodeFiles: List): LlvmModules { val config = context.config - // TODO: Possibly slow, maybe to a separate phase? - val runtimeModules = RuntimeLinkageStrategy.pick(context).run() + val (bitcodePartOfStdlib, bitcodeLibraries) = context.llvm.bitcodeToLink + .partition { it.isStdlib && context.producedLlvmModuleContainsStdlib } + .toList() + .map { libraries -> + libraries.flatMap { it.bitcodePaths }.filter { it.isBitcode } + } - val launcherNativeLibraries = config.launcherNativeLibraries + val nativeLibraries = config.nativeLibraries + config.launcherNativeLibraries .takeIf { config.produce == CompilerOutputKind.PROGRAM }.orEmpty() - - linkObjC(context) - - val nativeLibraries = config.nativeLibraries + launcherNativeLibraries - - val bitcodeLibraries = context.llvm.bitcodeToLink.map { it.bitcodePaths }.flatten().filter { it.isBitcode } val additionalBitcodeFilesToLink = context.llvm.additionalProducedBitcodeFiles - val exceptionsSupportNativeLibrary = config.exceptionsSupportNativeLibrary - val bitcodeFiles = (nativeLibraries + generatedBitcodeFiles + additionalBitcodeFilesToLink + bitcodeLibraries).toMutableSet() - if (config.produce == CompilerOutputKind.DYNAMIC_CACHE) - bitcodeFiles += exceptionsSupportNativeLibrary + val exceptionsSupportNativeLibrary = listOf(config.exceptionsSupportNativeLibrary) + .takeIf { config.produce == CompilerOutputKind.DYNAMIC_CACHE }.orEmpty() + val additionalBitcodeFiles = nativeLibraries + + generatedBitcodeFiles + + additionalBitcodeFilesToLink + + bitcodeLibraries + + exceptionsSupportNativeLibrary + + val runtimeNativeLibraries = context.config.runtimeNativeLibraries + .takeIf { context.producedLlvmModuleContainsStdlib }.orEmpty() + + + fun parseBitcodeFiles(files: List): List = files.map { bitcodeFile -> + val parsedModule = parseBitcodeFile(bitcodeFile) + if (!context.shouldUseDebugInfoFromNativeLibs()) { + LLVMStripModuleDebugInfo(parsedModule) + } + parsedModule + } + + val runtimeModules = parseBitcodeFiles(runtimeNativeLibraries + bitcodePartOfStdlib) + val additionalModules = parseBitcodeFiles(additionalBitcodeFiles) + return LlvmModules( + runtimeModules.ifNotEmpty { this + context.generateRuntimeConstantsModule() } ?: emptyList(), + additionalModules + listOfNotNull(patchObjCRuntimeModule(context)) + ) +} + +private fun linkAllDependencies(context: Context, generatedBitcodeFiles: List) { + val (runtimeModules, additionalModules) = collectLlvmModules(context, generatedBitcodeFiles) + // TODO: Possibly slow, maybe to a separate phase? + val optimizedRuntimeModules = RuntimeLinkageStrategy.pick(context, runtimeModules).run() val llvmModule = context.llvmModule!! - runtimeModules.forEach { + (optimizedRuntimeModules + additionalModules).forEach { val failed = llvmLinkModules2(context, llvmModule, it) if (failed != 0) { error("Failed to link ${it.getName()}") } } - bitcodeFiles.forEach { - parseAndLinkBitcodeFile(context, llvmModule, it) - } } private fun insertAliasToEntryPoint(context: Context) { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLinkageStrategy.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLinkageStrategy.kt index 3593987c3e3..a4ac3814d61 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLinkageStrategy.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLinkageStrategy.kt @@ -61,32 +61,13 @@ internal sealed class RuntimeLinkageStrategy { } } - /** - * Used in cases when runtime is not linked directly, e.g. it is a part of stdlib cache. - */ - object None : RuntimeLinkageStrategy() { - override fun run(): List = emptyList() - } - companion object { /** * Choose runtime linkage strategy based on current compiler configuration and [BinaryOptions.linkRuntime]. */ - internal fun pick(context: Context): RuntimeLinkageStrategy { + internal fun pick(context: Context, runtimeLlvmModules: List): RuntimeLinkageStrategy { val binaryOption = context.config.configuration.get(BinaryOptions.linkRuntime) - val runtimeNativeLibraries = context.config.runtimeNativeLibraries - .takeIf { context.producedLlvmModuleContainsStdlib } - val runtimeLlvmModules = runtimeNativeLibraries?.map { - val parsedModule = parseBitcodeFile(it) - if (!context.shouldUseDebugInfoFromNativeLibs()) { - LLVMStripModuleDebugInfo(parsedModule) - } - parsedModule - }?.let { - it + context.generateRuntimeConstantsModule() - } return when { - runtimeLlvmModules == null -> return None binaryOption == RuntimeLinkageStrategyBinaryOption.Raw -> Raw(runtimeLlvmModules) binaryOption == RuntimeLinkageStrategyBinaryOption.Optimize -> LinkAndOptimize(context, runtimeLlvmModules) context.config.debug -> LinkAndOptimize(context, runtimeLlvmModules) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objc/linkObjC.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objc/linkObjC.kt index d1d1fc815b8..2265b359afd 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objc/linkObjC.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objc/linkObjC.kt @@ -13,9 +13,9 @@ import org.jetbrains.kotlin.backend.konan.llvm.* import org.jetbrains.kotlin.backend.konan.objcexport.NSNumberKind import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExportNamer -internal fun linkObjC(context: Context) { +internal fun patchObjCRuntimeModule(context: Context): LLVMModuleRef? { val config = context.config - if (!(config.produce.isFinalBinary && config.target.family.isAppleFamily)) return + if (!(config.produce.isFinalBinary && config.target.family.isAppleFamily)) return null val patchBuilder = PatchBuilder(context) patchBuilder.addObjCPatches() @@ -24,15 +24,7 @@ internal fun linkObjC(context: Context) { val parsedModule = parseBitcodeFile(bitcodeFile) patchBuilder.buildAndApply(parsedModule) - - if (!context.shouldUseDebugInfoFromNativeLibs()) { - LLVMStripModuleDebugInfo(parsedModule) - } - - val failed = llvmLinkModules2(context, context.llvmModule!!, parsedModule) - if (failed != 0) { - throw Error("failed to link $bitcodeFile") - } + return parsedModule } private class PatchBuilder(val context: Context) {