diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 2e50855b351..a205feec749 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -250,6 +250,7 @@ class K2Native : CLICompiler() { put(SHORT_MODULE_NAME, it) } put(DISABLE_FAKE_OVERRIDE_VALIDATOR, arguments.disableFakeOverrideValidator) + putIfNotNull(PRE_LINK_CACHES, parsePreLinkCachesValue(configuration, arguments.preLinkCaches)) } } } @@ -299,6 +300,19 @@ private fun selectFrameworkType( } } +private fun parsePreLinkCachesValue( + configuration: CompilerConfiguration, + value: String? +): Boolean? = when (value) { + "enable" -> true + "disable" -> false + null -> null + else -> { + configuration.report(ERROR, "Unsupported `-Xpre-link-caches` value: $value. Possible values are 'enable'/'disable'") + null + } + } + private fun selectBitcodeEmbeddingMode( configuration: CompilerConfiguration, arguments: K2NativeCompilerArguments diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt index c68dc358cc9..33342783418 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt @@ -271,6 +271,13 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { @Argument(value = "-Xdebug-prefix-map", valueDescription = "", description = "Remap file source directory paths in debug info") var debugPrefixMap: Array? = null + @Argument( + value = "-Xpre-link-caches", + valueDescription = "{disable|enable}", + description = "Perform caches pre-link" + ) + var preLinkCaches: String? = null + override fun configureAnalysisFlags(collector: MessageCollector): MutableMap, Any> = super.configureAnalysisFlags(collector).also { val useExperimental = it[AnalysisFlags.useExperimental] as List<*> diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt index b9f786c9197..d91bbe46f64 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt @@ -86,6 +86,9 @@ class CacheSupport( } } + internal val preLinkCaches: Boolean = + configuration.get(KonanConfigKeys.PRE_LINK_CACHES, false) + init { // Ensure dependencies of every cached library are cached too: resolvedLibraries.getFullList { libraries -> diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index 39f7f24c918..c4e68b64aaf 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -142,6 +142,8 @@ class KonanConfigKeys { = CompilerConfigurationKey.create("write objc header with generics support") val DEBUG_PREFIX_MAP: CompilerConfigurationKey> = CompilerConfigurationKey.create("remap file source paths in debug info") + val PRE_LINK_CACHES: CompilerConfigurationKey + = CompilerConfigurationKey.create("perform compiler caches pre-link") } } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt index 1712496eeb6..aee47d3f106 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt @@ -1,6 +1,7 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.konan.KonanExternalToolFailure +import org.jetbrains.kotlin.konan.exec.Command import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.Family @@ -137,26 +138,30 @@ internal class Linker(val context: Context) { val needsProfileLibrary = context.coverage.enabled - val caches = determineCachesToLink(context) - + val linkerInput = determineLinkerInput(objectFiles, linkerOutput) try { File(executable).delete() - linker.linkCommands(objectFiles = objectFiles, executable = executable, - libraries = linker.linkStaticLibraries(includedBinaries) + - caches.static.takeIf { context.config.produce != CompilerOutputKind.STATIC_CACHE }.orEmpty(), - linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) + - BitcodeEmbedding.getLinkerOptions(context.config) + - caches.dynamic + - libraryProvidedLinkerFlags + additionalLinkerArgs, - optimize = optimize, debug = debug, kind = linkerOutput, + val linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) + + BitcodeEmbedding.getLinkerOptions(context.config) + + linkerInput.caches.dynamic + + libraryProvidedLinkerFlags + additionalLinkerArgs + val finalOutputCommands = linker.finalLinkCommands( + objectFiles = linkerInput.objectFiles, + executable = executable, + libraries = linker.linkStaticLibraries(includedBinaries) + linkerInput.caches.static, + linkerArgs = linkerArgs, + optimize = optimize, + debug = debug, + kind = linkerOutput, outputDsymBundle = context.config.outputFiles.symbolicInfoFile, - needsProfileLibrary = needsProfileLibrary).forEach { + needsProfileLibrary = needsProfileLibrary) + (linkerInput.preLinkCommands + finalOutputCommands).forEach { it.logWith(context::log) it.execute() } } catch (e: KonanExternalToolFailure) { val extraUserInfo = - if (caches.static.isNotEmpty() || caches.dynamic.isNotEmpty()) + if (linkerInput.cachingInvolved) """ Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory: @@ -170,8 +175,42 @@ internal class Linker(val context: Context) { return executable } + private fun shouldPerformPreLink(caches: CachesToLink, linkerOutputKind: LinkerOutputKind): Boolean { + // Pre-link is only useful when producing static library. Otherwise its just a waste of time. + val isStaticLibrary = linkerOutputKind == LinkerOutputKind.STATIC_LIBRARY && + context.config.produce.isFinalBinary + val enabled = context.config.cacheSupport.preLinkCaches + val nonEmptyCaches = caches.static.isNotEmpty() + return isStaticLibrary && enabled && nonEmptyCaches + } + + private fun determineLinkerInput(objectFiles: List, linkerOutputKind: LinkerOutputKind): LinkerInput { + val caches = determineCachesToLink(context) + // Since we have several linker stages that involve caching, + // we should detect cache usage early to report errors correctly. + val cachingInvolved = caches.static.isNotEmpty() || caches.dynamic.isNotEmpty() + return when { + context.config.produce == CompilerOutputKind.STATIC_CACHE -> { + // Do not link static cache dependencies. + LinkerInput(objectFiles, CachesToLink(emptyList(), caches.dynamic), emptyList(), cachingInvolved) + } + shouldPerformPreLink(caches, linkerOutputKind) -> { + val preLinkResult = context.config.tempFiles.create("withStaticCaches", ".o").absolutePath + val preLinkCommands = linker.preLinkCommands(objectFiles + caches.static, preLinkResult) + LinkerInput(listOf(preLinkResult), CachesToLink(emptyList(), caches.dynamic), preLinkCommands, cachingInvolved) + } + else -> LinkerInput(objectFiles, caches, emptyList(), cachingInvolved) + } + } } +private class LinkerInput( + val objectFiles: List, + val caches: CachesToLink, + val preLinkCommands: List, + val cachingInvolved: Boolean +) + private class CachesToLink(val static: List, val dynamic: List) private fun determineCachesToLink(context: Context): CachesToLink { diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt index fdee3c785ef..0a7e7357fec 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt @@ -68,12 +68,22 @@ abstract class LinkerFlags(val configurables: Configurables) { open val useCompilerDriverAsLinker: Boolean get() = false // TODO: refactor. + /** + * Returns list of commands that produces final linker output. + */ // TODO: Number of arguments is quite big. Better to pass args via object. - abstract fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, - kind: LinkerOutputKind, outputDsymBundle: String, - needsProfileLibrary: Boolean): List + abstract fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, + kind: LinkerOutputKind, outputDsymBundle: String, + needsProfileLibrary: Boolean): List + + /** + * Returns list of commands that link object files into a single one. + * Pre-linkage is useful for hiding dependency symbols. + */ + open fun preLinkCommands(objectFiles: List, output: ObjectFile): List = + error("Pre-link is unsupported for ${configurables.target}.") abstract fun filterStaticLibraries(binaries: List): List @@ -109,11 +119,11 @@ class AndroidLinker(targetProperties: AndroidConfigurables) override fun filterStaticLibraries(binaries: List) = binaries.filter { it.isUnixStaticLib } - override fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, - kind: LinkerOutputKind, outputDsymBundle: String, - needsProfileLibrary: Boolean): List { + override fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, + kind: LinkerOutputKind, outputDsymBundle: String, + needsProfileLibrary: Boolean): List { if (kind == LinkerOutputKind.STATIC_LIBRARY) return staticGnuArCommands(ar, executable, objectFiles, libraries) @@ -185,11 +195,21 @@ class MacOSBasedLinker(targetProperties: AppleConfigurables) override fun filterStaticLibraries(binaries: List) = binaries.filter { it.isUnixStaticLib } - override fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, kind: LinkerOutputKind, - outputDsymBundle: String, - needsProfileLibrary: Boolean): List { + // Note that may break in case of 32-bit Mach-O. See KT-37368. + override fun preLinkCommands(objectFiles: List, output: ObjectFile): List = + Command(linker).apply { + +"-r" + +listOf("-arch", arch) + +listOf("-syslibroot", absoluteTargetSysRoot) + +objectFiles + +listOf("-o", output) + }.let(::listOf) + + override fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, kind: LinkerOutputKind, + outputDsymBundle: String, + needsProfileLibrary: Boolean): List { if (kind == LinkerOutputKind.STATIC_LIBRARY) return listOf(Command(libtool).apply { +"-static" @@ -308,11 +328,11 @@ class GccBasedLinker(targetProperties: GccConfigurables) override fun filterStaticLibraries(binaries: List) = binaries.filter { it.isUnixStaticLib } - override fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, - kind: LinkerOutputKind, outputDsymBundle: String, - needsProfileLibrary: Boolean): List { + override fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, + kind: LinkerOutputKind, outputDsymBundle: String, + needsProfileLibrary: Boolean): List { if (kind == LinkerOutputKind.STATIC_LIBRARY) return staticGnuArCommands(ar, executable, objectFiles, libraries) @@ -376,11 +396,11 @@ class MingwLinker(targetProperties: MingwConfigurables) return if (dir != null) "$dir/lib/windows/libclang_rt.$libraryName-$targetSuffix.a" else null } - override fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, - kind: LinkerOutputKind, outputDsymBundle: String, - needsProfileLibrary: Boolean): List { + override fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, + kind: LinkerOutputKind, outputDsymBundle: String, + needsProfileLibrary: Boolean): List { if (kind == LinkerOutputKind.STATIC_LIBRARY) return staticGnuArCommands(ar, executable, objectFiles, libraries) @@ -412,11 +432,11 @@ class WasmLinker(targetProperties: WasmConfigurables) override fun filterStaticLibraries(binaries: List) = binaries.filter { it.isJavaScript } - override fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, - kind: LinkerOutputKind, outputDsymBundle: String, - needsProfileLibrary: Boolean): List { + override fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, + kind: LinkerOutputKind, outputDsymBundle: String, + needsProfileLibrary: Boolean): List { if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind") val linkage = Command("$llvmBin/wasm-ld").apply { @@ -464,11 +484,11 @@ open class ZephyrLinker(targetProperties: ZephyrConfigurables) override fun filterStaticLibraries(binaries: List) = emptyList() - override fun linkCommands(objectFiles: List, executable: ExecutableFile, - libraries: List, linkerArgs: List, - optimize: Boolean, debug: Boolean, - kind: LinkerOutputKind, outputDsymBundle: String, - needsProfileLibrary: Boolean): List { + override fun finalLinkCommands(objectFiles: List, executable: ExecutableFile, + libraries: List, linkerArgs: List, + optimize: Boolean, debug: Boolean, + kind: LinkerOutputKind, outputDsymBundle: String, + needsProfileLibrary: Boolean): List { if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind: $kind") return listOf(Command(linker).apply { +listOf("-r", "--gc-sections", "--entry", "main")