[KT-34261][Linker] Pre-link static caches

Use `ld -r` on darwin-based targets to pre-link static caches
before final linker invocation. It allows to hide symbols from
caches to avoid linkage problems when linking 2 or more
Kotlin-produced binaries.

Sadly, it has several problems:
1. `ld -r` is slow.
2. There is no obvious way to make common symbols private.
These are the reasons why we don't enable `-Xpre-link-cached` by default.

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -250,6 +250,7 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
                     put(SHORT_MODULE_NAME, it)
                 }
                 put(DISABLE_FAKE_OVERRIDE_VALIDATOR, arguments.disableFakeOverrideValidator)
+                putIfNotNull(PRE_LINK_CACHES, parsePreLinkCachesValue(configuration, arguments.preLinkCaches))
             }
         }
     }
@@ -299,6 +300,19 @@ private fun selectFrameworkType(
     }
 }
 
+private fun parsePreLinkCachesValue(
+        configuration: CompilerConfiguration,
+        value: String?
+): Boolean? = when (value) {
+        "enable" -> true
+        "disable" -> false
+        null -> null
+        else -> {
+            configuration.report(ERROR, "Unsupported `-Xpre-link-caches` value: $value. Possible values are 'enable'/'disable'")
+            null
+        }
+    }
+
 private fun selectBitcodeEmbeddingMode(
         configuration: CompilerConfiguration,
         arguments: K2NativeCompilerArguments

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt

@@ -271,6 +271,13 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
     @Argument(value = "-Xdebug-prefix-map", valueDescription = "<old1=new1,old2=new2,...>", description = "Remap file source directory paths in debug info")
     var debugPrefixMap: Array<String>? = null
 
+    @Argument(
+            value = "-Xpre-link-caches",
+            valueDescription = "{disable|enable}",
+            description = "Perform caches pre-link"
+    )
+    var preLinkCaches: String? = null
+
     override fun configureAnalysisFlags(collector: MessageCollector): MutableMap<AnalysisFlag<*>, Any> =
             super.configureAnalysisFlags(collector).also {
                 val useExperimental = it[AnalysisFlags.useExperimental] as List<*>

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt

@@ -86,6 +86,9 @@ class CacheSupport(
         }
     }
 
+    internal val preLinkCaches: Boolean =
+            configuration.get(KonanConfigKeys.PRE_LINK_CACHES, false)
+
     init {
         // Ensure dependencies of every cached library are cached too:
         resolvedLibraries.getFullList { libraries ->

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt

@@ -142,6 +142,8 @@ class KonanConfigKeys {
                 = CompilerConfigurationKey.create("write objc header with generics support")
         val DEBUG_PREFIX_MAP: CompilerConfigurationKey<Map<String, String>>
                 = CompilerConfigurationKey.create("remap file source paths in debug info")
+        val PRE_LINK_CACHES: CompilerConfigurationKey<Boolean>
+                = CompilerConfigurationKey.create("perform compiler caches pre-link")
     }
 }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlin.backend.konan
 
 import org.jetbrains.kotlin.konan.KonanExternalToolFailure
+import org.jetbrains.kotlin.konan.exec.Command
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Family
@@ -137,26 +138,30 @@ internal class Linker(val context: Context) {
 
         val needsProfileLibrary = context.coverage.enabled
 
-        val caches = determineCachesToLink(context)
+        val linkerInput = determineLinkerInput(objectFiles, linkerOutput)
-
         try {
             File(executable).delete()
-            linker.linkCommands(objectFiles = objectFiles, executable = executable,
+            val linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
-                    libraries = linker.linkStaticLibraries(includedBinaries) +
+                    BitcodeEmbedding.getLinkerOptions(context.config) +
-                            caches.static.takeIf { context.config.produce != CompilerOutputKind.STATIC_CACHE }.orEmpty(),
+                    linkerInput.caches.dynamic +
-                    linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
+                    libraryProvidedLinkerFlags + additionalLinkerArgs
-                            BitcodeEmbedding.getLinkerOptions(context.config) +
+            val finalOutputCommands = linker.finalLinkCommands(
-                            caches.dynamic +
+                    objectFiles = linkerInput.objectFiles,
-                            libraryProvidedLinkerFlags + additionalLinkerArgs,
+                    executable = executable,
-                    optimize = optimize, debug = debug, kind = linkerOutput,
+                    libraries = linker.linkStaticLibraries(includedBinaries) + linkerInput.caches.static,
+                    linkerArgs = linkerArgs,
+                    optimize = optimize,
+                    debug = debug,
+                    kind = linkerOutput,
                     outputDsymBundle = context.config.outputFiles.symbolicInfoFile,
-                    needsProfileLibrary = needsProfileLibrary).forEach {
+                    needsProfileLibrary = needsProfileLibrary)
+            (linkerInput.preLinkCommands + finalOutputCommands).forEach {
                 it.logWith(context::log)
                 it.execute()
             }
         } catch (e: KonanExternalToolFailure) {
             val extraUserInfo =
-                    if (caches.static.isNotEmpty() || caches.dynamic.isNotEmpty())
+                    if (linkerInput.cachingInvolved)
                         """
                         Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
                             
@@ -170,8 +175,42 @@ internal class Linker(val context: Context) {
         return executable
     }
 
+    private fun shouldPerformPreLink(caches: CachesToLink, linkerOutputKind: LinkerOutputKind): Boolean {
+        // Pre-link is only useful when producing static library. Otherwise its just a waste of time.
+        val isStaticLibrary = linkerOutputKind == LinkerOutputKind.STATIC_LIBRARY &&
+                context.config.produce.isFinalBinary
+        val enabled = context.config.cacheSupport.preLinkCaches
+        val nonEmptyCaches = caches.static.isNotEmpty()
+        return isStaticLibrary && enabled && nonEmptyCaches
+    }
+
+    private fun determineLinkerInput(objectFiles: List<ObjectFile>, linkerOutputKind: LinkerOutputKind): LinkerInput {
+        val caches = determineCachesToLink(context)
+        // Since we have several linker stages that involve caching,
+        // we should detect cache usage early to report errors correctly.
+        val cachingInvolved = caches.static.isNotEmpty() || caches.dynamic.isNotEmpty()
+        return when {
+            context.config.produce == CompilerOutputKind.STATIC_CACHE -> {
+                // Do not link static cache dependencies.
+                LinkerInput(objectFiles, CachesToLink(emptyList(), caches.dynamic), emptyList(), cachingInvolved)
+            }
+            shouldPerformPreLink(caches, linkerOutputKind) -> {
+                val preLinkResult = context.config.tempFiles.create("withStaticCaches", ".o").absolutePath
+                val preLinkCommands = linker.preLinkCommands(objectFiles + caches.static, preLinkResult)
+                LinkerInput(listOf(preLinkResult), CachesToLink(emptyList(), caches.dynamic), preLinkCommands, cachingInvolved)
+            }
+            else -> LinkerInput(objectFiles, caches, emptyList(), cachingInvolved)
+        }
+    }
 }
 
+private class LinkerInput(
+        val objectFiles: List<ObjectFile>,
+        val caches: CachesToLink,
+        val preLinkCommands: List<Command>,
+        val cachingInvolved: Boolean
+)
+
 private class CachesToLink(val static: List<String>, val dynamic: List<String>)
 
 private fun determineCachesToLink(context: Context): CachesToLink {

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt

@@ -68,12 +68,22 @@ abstract class LinkerFlags(val configurables: Configurables) {
 
     open val useCompilerDriverAsLinker: Boolean get() = false // TODO: refactor.
 
+    /**
+     * Returns list of commands that produces final linker output.
+     */
     // TODO: Number of arguments is quite big. Better to pass args via object.
-    abstract fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    abstract fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
-                              libraries: List<String>, linkerArgs: List<String>,
+                                   libraries: List<String>, linkerArgs: List<String>,
-                              optimize: Boolean, debug: Boolean,
+                                   optimize: Boolean, debug: Boolean,
-                              kind: LinkerOutputKind, outputDsymBundle: String,
+                                   kind: LinkerOutputKind, outputDsymBundle: String,
-                              needsProfileLibrary: Boolean): List<Command>
+                                   needsProfileLibrary: Boolean): List<Command>
+
+    /**
+     * Returns list of commands that link object files into a single one.
+     * Pre-linkage is useful for hiding dependency symbols.
+     */
+    open fun preLinkCommands(objectFiles: List<ObjectFile>, output: ObjectFile): List<Command> =
+            error("Pre-link is unsupported for ${configurables.target}.")
 
     abstract fun filterStaticLibraries(binaries: List<String>): List<String>
 
@@ -109,11 +119,11 @@ class AndroidLinker(targetProperties: AndroidConfigurables)
 
     override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
 
-    override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
-                              libraries: List<String>, linkerArgs: List<String>,
+                                   libraries: List<String>, linkerArgs: List<String>,
-                              optimize: Boolean, debug: Boolean,
+                                   optimize: Boolean, debug: Boolean,
-                              kind: LinkerOutputKind, outputDsymBundle: String,
+                                   kind: LinkerOutputKind, outputDsymBundle: String,
-                              needsProfileLibrary: Boolean): List<Command> {
+                                   needsProfileLibrary: Boolean): List<Command> {
         if (kind == LinkerOutputKind.STATIC_LIBRARY)
             return staticGnuArCommands(ar, executable, objectFiles, libraries)
 
@@ -185,11 +195,21 @@ class MacOSBasedLinker(targetProperties: AppleConfigurables)
 
     override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
 
-    override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    // Note that may break in case of 32-bit Mach-O. See KT-37368.
-                              libraries: List<String>, linkerArgs: List<String>,
+    override fun preLinkCommands(objectFiles: List<ObjectFile>, output: ObjectFile): List<Command> =
-                              optimize: Boolean, debug: Boolean, kind: LinkerOutputKind,
+        Command(linker).apply {
-                              outputDsymBundle: String,
+            +"-r"
-                              needsProfileLibrary: Boolean): List<Command> {
+            +listOf("-arch", arch)
+            +listOf("-syslibroot", absoluteTargetSysRoot)
+            +objectFiles
+            +listOf("-o", output)
+        }.let(::listOf)
+
+    override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+                                   libraries: List<String>, linkerArgs: List<String>,
+                                   optimize: Boolean, debug: Boolean, kind: LinkerOutputKind,
+                                   outputDsymBundle: String,
+                                   needsProfileLibrary: Boolean): List<Command> {
         if (kind == LinkerOutputKind.STATIC_LIBRARY)
             return listOf(Command(libtool).apply {
                 +"-static"
@@ -308,11 +328,11 @@ class GccBasedLinker(targetProperties: GccConfigurables)
 
     override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
 
-    override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
-                              libraries: List<String>, linkerArgs: List<String>,
+                                   libraries: List<String>, linkerArgs: List<String>,
-                              optimize: Boolean, debug: Boolean,
+                                   optimize: Boolean, debug: Boolean,
-                              kind: LinkerOutputKind, outputDsymBundle: String,
+                                   kind: LinkerOutputKind, outputDsymBundle: String,
-                              needsProfileLibrary: Boolean): List<Command> {
+                                   needsProfileLibrary: Boolean): List<Command> {
         if (kind == LinkerOutputKind.STATIC_LIBRARY)
             return staticGnuArCommands(ar, executable, objectFiles, libraries)
 
@@ -376,11 +396,11 @@ class MingwLinker(targetProperties: MingwConfigurables)
         return if (dir != null) "$dir/lib/windows/libclang_rt.$libraryName-$targetSuffix.a" else null
     }
 
-    override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
-                              libraries: List<String>, linkerArgs: List<String>,
+                                   libraries: List<String>, linkerArgs: List<String>,
-                              optimize: Boolean, debug: Boolean,
+                                   optimize: Boolean, debug: Boolean,
-                              kind: LinkerOutputKind, outputDsymBundle: String,
+                                   kind: LinkerOutputKind, outputDsymBundle: String,
-                              needsProfileLibrary: Boolean): List<Command> {
+                                   needsProfileLibrary: Boolean): List<Command> {
         if (kind == LinkerOutputKind.STATIC_LIBRARY)
             return staticGnuArCommands(ar, executable, objectFiles, libraries)
 
@@ -412,11 +432,11 @@ class WasmLinker(targetProperties: WasmConfigurables)
 
     override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isJavaScript }
 
-    override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
-                              libraries: List<String>, linkerArgs: List<String>,
+                                   libraries: List<String>, linkerArgs: List<String>,
-                              optimize: Boolean, debug: Boolean,
+                                   optimize: Boolean, debug: Boolean,
-                              kind: LinkerOutputKind, outputDsymBundle: String,
+                                   kind: LinkerOutputKind, outputDsymBundle: String,
-                              needsProfileLibrary: Boolean): List<Command> {
+                                   needsProfileLibrary: Boolean): List<Command> {
         if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind")
 
         val linkage = Command("$llvmBin/wasm-ld").apply {
@@ -464,11 +484,11 @@ open class ZephyrLinker(targetProperties: ZephyrConfigurables)
 
     override fun filterStaticLibraries(binaries: List<String>) = emptyList<String>()
 
-    override fun linkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
+    override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
-                              libraries: List<String>, linkerArgs: List<String>,
+                                   libraries: List<String>, linkerArgs: List<String>,
-                              optimize: Boolean, debug: Boolean,
+                                   optimize: Boolean, debug: Boolean,
-                              kind: LinkerOutputKind, outputDsymBundle: String,
+                                   kind: LinkerOutputKind, outputDsymBundle: String,
-                              needsProfileLibrary: Boolean): List<Command> {
+                                   needsProfileLibrary: Boolean): List<Command> {
         if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind: $kind")
         return listOf(Command(linker).apply {
             +listOf("-r", "--gc-sections", "--entry", "main")