azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[KT-34261][Linker] Pre-link static caches

Browse Source 
Use `ld -r` on darwin-based targets to pre-link static caches
before final linker invocation. It allows to hide symbols from
caches to avoid linkage problems when linking 2 or more
Kotlin-produced binaries.

Sadly, it has several problems:
1. `ld -r` is slow.
2. There is no obvious way to make common symbols private.
These are the reasons why we don't enable `-Xpre-link-cached` by default.
This commit is contained in:
Sergey Bogolepov
2020-09-01 11:32:07 +07:00
committed by Sergey Bogolepov
parent a710a478a5
commit e9dd7cb3b6
6 changed files with 132 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
+14
View File
@@ -250,6 +250,7 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
put(SHORT_MODULE_NAME, it)
}
put(DISABLE_FAKE_OVERRIDE_VALIDATOR, arguments.disableFakeOverrideValidator)
putIfNotNull(PRE_LINK_CACHES, parsePreLinkCachesValue(configuration, arguments.preLinkCaches))
}
}
}
@@ -299,6 +300,19 @@ private fun selectFrameworkType(
}
}
private fun parsePreLinkCachesValue(
configuration: CompilerConfiguration,
value: String?
): Boolean? = when (value) {
"enable" -> true
"disable" -> false
null -> null
else -> {
configuration.report(ERROR, "Unsupported `-Xpre-link-caches` value: $value. Possible values are 'enable'/'disable'")
null
}
}
private fun selectBitcodeEmbeddingMode(
configuration: CompilerConfiguration,
arguments: K2NativeCompilerArguments
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt
+7
View File
@@ -271,6 +271,13 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
@Argument(value = "-Xdebug-prefix-map", valueDescription = "<old1=new1,old2=new2,...>", description = "Remap file source directory paths in debug info")
var debugPrefixMap: Array<String>? = null
@Argument(
value = "-Xpre-link-caches",
valueDescription = "{disable|enable}",
description = "Perform caches pre-link"
)
var preLinkCaches: String? = null
override fun configureAnalysisFlags(collector: MessageCollector): MutableMap<AnalysisFlag<*>, Any> =
super.configureAnalysisFlags(collector).also {
val useExperimental = it[AnalysisFlags.useExperimental] as List<*>
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt
+3
View File
@@ -86,6 +86,9 @@ class CacheSupport(
}
}
internal val preLinkCaches: Boolean =
configuration.get(KonanConfigKeys.PRE_LINK_CACHES, false)
init {
// Ensure dependencies of every cached library are cached too:
resolvedLibraries.getFullList { libraries ->
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt
+2
View File
@@ -142,6 +142,8 @@ class KonanConfigKeys {
= CompilerConfigurationKey.create("write objc header with generics support")
val DEBUG_PREFIX_MAP: CompilerConfigurationKey<Map<String, String>>
= CompilerConfigurationKey.create("remap file source paths in debug info")
val PRE_LINK_CACHES: CompilerConfigurationKey<Boolean>
= CompilerConfigurationKey.create("perform compiler caches pre-link")
}
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt
+51 -12
View File
@@ -1,6 +1,7 @@
package org.jetbrains.kotlin.backend.konan
import org.jetbrains.kotlin.konan.KonanExternalToolFailure
import org.jetbrains.kotlin.konan.exec.Command
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.Family
@@ -137,26 +138,30 @@ internal class Linker(val context: Context) {
val needsProfileLibrary = context.coverage.enabled
val caches = determineCachesToLink(context)
val linkerInput = determineLinkerInput(objectFiles, linkerOutput)
try {
File(executable).delete()
linker.linkCommands(objectFiles = objectFiles, executable = executable,
libraries = linker.linkStaticLibraries(includedBinaries) +
caches.static.takeIf { context.config.produce != CompilerOutputKind.STATIC_CACHE }.orEmpty(),
linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
BitcodeEmbedding.getLinkerOptions(context.config) +
caches.dynamic +
libraryProvidedLinkerFlags + additionalLinkerArgs,
optimize = optimize, debug = debug, kind = linkerOutput,
val linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
BitcodeEmbedding.getLinkerOptions(context.config) +
linkerInput.caches.dynamic +
libraryProvidedLinkerFlags + additionalLinkerArgs
val finalOutputCommands = linker.finalLinkCommands(
objectFiles = linkerInput.objectFiles,
executable = executable,
libraries = linker.linkStaticLibraries(includedBinaries) + linkerInput.caches.static,
linkerArgs = linkerArgs,
optimize = optimize,
debug = debug,
kind = linkerOutput,
outputDsymBundle = context.config.outputFiles.symbolicInfoFile,
needsProfileLibrary = needsProfileLibrary).forEach {
needsProfileLibrary = needsProfileLibrary)
(linkerInput.preLinkCommands + finalOutputCommands).forEach {
it.logWith(context::log)
it.execute()
}
} catch (e: KonanExternalToolFailure) {
val extraUserInfo =
if (caches.static.isNotEmpty() || caches.dynamic.isNotEmpty())
if (linkerInput.cachingInvolved)
"""
Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
@@ -170,8 +175,42 @@ internal class Linker(val context: Context) {
return executable
}
private fun shouldPerformPreLink(caches: CachesToLink, linkerOutputKind: LinkerOutputKind): Boolean {
// Pre-link is only useful when producing static library. Otherwise its just a waste of time.
val isStaticLibrary = linkerOutputKind == LinkerOutputKind.STATIC_LIBRARY &&
context.config.produce.isFinalBinary
val enabled = context.config.cacheSupport.preLinkCaches
val nonEmptyCaches = caches.static.isNotEmpty()
return isStaticLibrary && enabled && nonEmptyCaches
}
private fun determineLinkerInput(objectFiles: List<ObjectFile>, linkerOutputKind: LinkerOutputKind): LinkerInput {
val caches = determineCachesToLink(context)
// Since we have several linker stages that involve caching,
// we should detect cache usage early to report errors correctly.
val cachingInvolved = caches.static.isNotEmpty() || caches.dynamic.isNotEmpty()
return when {
context.config.produce == CompilerOutputKind.STATIC_CACHE -> {
// Do not link static cache dependencies.
LinkerInput(objectFiles, CachesToLink(emptyList(), caches.dynamic), emptyList(), cachingInvolved)
}
shouldPerformPreLink(caches, linkerOutputKind) -> {
val preLinkResult = context.config.tempFiles.create("withStaticCaches", ".o").absolutePath
val preLinkCommands = linker.preLinkCommands(objectFiles + caches.static, preLinkResult)
LinkerInput(listOf(preLinkResult), CachesToLink(emptyList(), caches.dynamic), preLinkCommands, cachingInvolved)
}
else -> LinkerInput(objectFiles, caches, emptyList(), cachingInvolved)
}
}
}
private class LinkerInput(
val objectFiles: List<ObjectFile>,
val caches: CachesToLink,
val preLinkCommands: List<Command>,
val cachingInvolved: Boolean
)
private class CachesToLink(val static: List<String>, val dynamic: List<String>)
private fun determineCachesToLink(context: Context): CachesToLink {
shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt
+55 -35
View File
@@ -68,12 +68,22 @@ abstract class LinkerFlags(val configurables: Configurables) {
open val useCompilerDriverAsLinker: Boolean get() = false // TODO: refactor.
/**
* Returns list of commands that produces final linker output.
*/
// TODO: Number of arguments is quite big. Better to pass args via object.
abstract fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command>
abstract fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command>
/**
* Returns list of commands that link object files into a single one.
* Pre-linkage is useful for hiding dependency symbols.
*/
open fun preLinkCommands(objectFiles: List<ObjectFile>, output: ObjectFile): List<Command> =
error("Pre-link is unsupported for ${configurables.target}.")
abstract fun filterStaticLibraries(binaries: List<String>): List<String>
@@ -109,11 +119,11 @@ class AndroidLinker(targetProperties: AndroidConfigurables)
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return staticGnuArCommands(ar, executable, objectFiles, libraries)
@@ -185,11 +195,21 @@ class MacOSBasedLinker(targetProperties: AppleConfigurables)
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean, kind: LinkerOutputKind,
outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
// Note that may break in case of 32-bit Mach-O. See KT-37368.
override fun preLinkCommands(objectFiles: List<ObjectFile>, output: ObjectFile): List<Command> =
Command(linker).apply {
+"-r"
+listOf("-arch", arch)
+listOf("-syslibroot", absoluteTargetSysRoot)
+objectFiles
+listOf("-o", output)
}.let(::listOf)
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean, kind: LinkerOutputKind,
outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return listOf(Command(libtool).apply {
+"-static"
@@ -308,11 +328,11 @@ class GccBasedLinker(targetProperties: GccConfigurables)
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return staticGnuArCommands(ar, executable, objectFiles, libraries)
@@ -376,11 +396,11 @@ class MingwLinker(targetProperties: MingwConfigurables)
return if (dir != null) "$dir/lib/windows/libclang_rt.$libraryName-$targetSuffix.a" else null
}
override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return staticGnuArCommands(ar, executable, objectFiles, libraries)
@@ -412,11 +432,11 @@ class WasmLinker(targetProperties: WasmConfigurables)
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isJavaScript }
override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind")
val linkage = Command("$llvmBin/wasm-ld").apply {
@@ -464,11 +484,11 @@ open class ZephyrLinker(targetProperties: ZephyrConfigurables)
override fun filterStaticLibraries(binaries: List<String>) = emptyList<String>()
override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean): List<Command> {
if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind: $kind")
return listOf(Command(linker).apply {
+listOf("-r", "--gc-sections", "--entry", "main")