[K/N] Extract pre-link from linker and a bit of refactoring

Finish extracting all usages of TempFiles to the driver.

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBinariesResolver.kt

@@ -24,7 +24,9 @@ internal fun shouldPerformPreLink(config: KonanConfig, caches: ResolvedCacheBina
  * [static] is a list of static libraries (e.g. "libcache.a")
  * [dynamic] is a list of dynamic libraries (e.g. "libcache.dylib")
  */
-internal class ResolvedCacheBinaries(val static: List<String>, val dynamic: List<String>)
+internal class ResolvedCacheBinaries(val static: List<String>, val dynamic: List<String>) {
+    fun isEmpty(): Boolean = static.isEmpty() && dynamic.isEmpty()
+}
 
 /**
  * Find binary files for compiler caches that are actually required for the linkage.

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt

@@ -2,7 +2,6 @@ package org.jetbrains.kotlin.backend.konan
 
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.konan.KonanExternalToolFailure
-import org.jetbrains.kotlin.konan.TempFiles
 import org.jetbrains.kotlin.konan.exec.Command
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.KonanLibrary
@@ -36,25 +35,23 @@ internal fun determineLinkerOutput(context: PhaseContext): LinkerOutputKind =
 
 // TODO: We have a Linker.kt file in the shared module.
 internal class Linker(
-        private val context: PhaseContext,
+        private val config: KonanConfig,
+        private val linkerOutput: LinkerOutputKind,
         private val isCoverageEnabled: Boolean = false,
-        private val tempFiles: TempFiles,
         private val outputFiles: OutputFiles,
 ) {
-
-    private val config = context.config
     private val platform = config.platform
-    private val linkerOutput = determineLinkerOutput(context)
     private val linker = platform.linker
     private val target = config.target
-    private val optimize = context.shouldOptimize()
+    private val optimize = config.optimizationsEnabled
     private val debug = config.debug || config.lightDebug
 
-    fun link(
+    fun linkCommands(
             outputFile: String,
             objectFiles: List<ObjectFile>,
-            dependenciesTrackingResult: DependenciesTrackingResult
+            dependenciesTrackingResult: DependenciesTrackingResult,
-    ) {
+            caches: ResolvedCacheBinaries,
+    ): List<Command> {
         val nativeDependencies = dependenciesTrackingResult.nativeDependenciesToLink
 
         val includedBinariesLibraries = config.libraryToCache?.let { listOf(it.klib) }
@@ -62,8 +59,7 @@ internal class Linker(
         val includedBinaries = includedBinariesLibraries.map { (it as? KonanLibrary)?.includedPaths.orEmpty() }.flatten()
 
         val libraryProvidedLinkerFlags = dependenciesTrackingResult.allNativeDependencies.map { it.linkerOpts }.flatten()
-
+        return runLinker(outputFile, objectFiles, includedBinaries, libraryProvidedLinkerFlags, caches)
-        runLinker(outputFile, objectFiles, includedBinaries, libraryProvidedLinkerFlags, dependenciesTrackingResult)
     }
 
     private fun asLinkerArgs(args: List<String>): List<String> {
@@ -88,8 +84,8 @@ internal class Linker(
             objectFiles: List<ObjectFile>,
             includedBinaries: List<String>,
             libraryProvidedLinkerFlags: List<String>,
-            dependenciesTrackingResult: DependenciesTrackingResult,
+            caches: ResolvedCacheBinaries,
-    ): ExecutableFile {
+    ): List<Command> {
         val additionalLinkerArgs: List<String>
         val executable: String
 
@@ -119,77 +115,44 @@ internal class Linker(
             dylibPath.parentFile.mkdirs()
             executable = dylibPath.absolutePath
         }
+        File(executable).delete()
 
-        val mimallocEnabled = config.allocationMode == AllocationMode.MIMALLOC
+        val linkerArgs = asLinkerArgs(config.configuration.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
+                BitcodeEmbedding.getLinkerOptions(config) +
+                caches.dynamic +
+                libraryProvidedLinkerFlags + additionalLinkerArgs
 
-        val linkerInput = determineLinkerInput(objectFiles, linkerOutput, dependenciesTrackingResult)
+        return linker.finalLinkCommands(
-        try {
+                objectFiles = objectFiles,
-            File(executable).delete()
+                executable = executable,
-            val linkerArgs = asLinkerArgs(config.configuration.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
+                libraries = linker.linkStaticLibraries(includedBinaries) + caches.static,
-                    BitcodeEmbedding.getLinkerOptions(config) +
+                linkerArgs = linkerArgs,
-                    linkerInput.caches.dynamic +
+                optimize = optimize,
-                    libraryProvidedLinkerFlags + additionalLinkerArgs
+                debug = debug,
-
+                kind = linkerOutput,
-            val finalOutputCommands = linker.finalLinkCommands(
+                outputDsymBundle = outputFiles.symbolicInfoFile,
-                    objectFiles = linkerInput.objectFiles,
+                needsProfileLibrary = isCoverageEnabled,
-                    executable = executable,
+                mimallocEnabled = config.allocationMode == AllocationMode.MIMALLOC,
-                    libraries = linker.linkStaticLibraries(includedBinaries) + linkerInput.caches.static,
+                sanitizer = config.sanitizer
-                    linkerArgs = linkerArgs,
+        )
-                    optimize = optimize,
-                    debug = debug,
-                    kind = linkerOutput,
-                    outputDsymBundle = outputFiles.symbolicInfoFile,
-                    needsProfileLibrary = isCoverageEnabled,
-                    mimallocEnabled = mimallocEnabled,
-                    sanitizer = config.sanitizer
-            )
-            (linkerInput.preLinkCommands + finalOutputCommands).forEach {
-                it.logWith(context::log)
-                it.execute()
-            }
-        } catch (e: KonanExternalToolFailure) {
-            val extraUserInfo =
-                    if (linkerInput.cachingInvolved)
-                        """
-                        Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
-                            
-                            kotlin.native.cacheKind.${target.presetName}=none
-                            
-                        Also, consider filing an issue with full Gradle log here: https://kotl.in/issue
-                        """.trimIndent()
-                    else ""
-            context.reportCompilationError("${e.toolName} invocation reported errors\n$extraUserInfo\n${e.message}")
-        }
-        return executable
-    }
-
-    private fun determineLinkerInput(
-            objectFiles: List<ObjectFile>,
-            linkerOutputKind: LinkerOutputKind,
-            dependenciesTrackingResult: DependenciesTrackingResult,
-    ): LinkerInput {
-        val caches = resolveCacheBinaries(context.config.cachedLibraries, dependenciesTrackingResult)
-        // Since we have several linker stages that involve caching,
-        // we should detect cache usage early to report errors correctly.
-        val cachingInvolved = caches.static.isNotEmpty() || caches.dynamic.isNotEmpty()
-        return when {
-            config.produce == CompilerOutputKind.STATIC_CACHE -> {
-                // Do not link static cache dependencies.
-                LinkerInput(objectFiles, ResolvedCacheBinaries(emptyList(), caches.dynamic), emptyList(), cachingInvolved)
-            }
-            shouldPerformPreLink(config, caches, linkerOutputKind) -> {
-                val preLinkResult = tempFiles.create("withStaticCaches", ".o").absolutePath
-                val preLinkCommands = linker.preLinkCommands(objectFiles + caches.static, preLinkResult)
-                LinkerInput(listOf(preLinkResult), ResolvedCacheBinaries(emptyList(), caches.dynamic), preLinkCommands, cachingInvolved)
-            }
-            else -> LinkerInput(objectFiles, caches, emptyList(), cachingInvolved)
-        }
     }
 }
 
-private class LinkerInput(
+internal fun runLinkerCommands(context: PhaseContext, commands: List<Command>, cachingInvolved: Boolean) = try {
-        val objectFiles: List<ObjectFile>,
+    commands.forEach {
-        val caches: ResolvedCacheBinaries,
+        it.logWith(context::log)
-        val preLinkCommands: List<Command>,
+        it.execute()
-        val cachingInvolved: Boolean
+    }
-)
+} catch (e: KonanExternalToolFailure) {
+    val extraUserInfo =
+            if (cachingInvolved)
+                """
+                        Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
+                            
+                            kotlin.native.cacheKind.${context.config.target.presetName}=none
+                            
+                        Also, consider filing an issue with full Gradle log here: https://kotl.in/issue
+                        """.trimIndent()
+            else ""
+    context.reportCompilationError("${e.toolName} invocation reported errors\n$extraUserInfo\n${e.message}")
+}

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Linker.kt

@@ -8,14 +8,16 @@ package org.jetbrains.kotlin.backend.konan.driver.phases
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.Linker
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
-import org.jetbrains.kotlin.konan.TempFiles
+import org.jetbrains.kotlin.konan.target.LinkerOutputKind
+import java.io.File
 
-data class LinkerPhaseInput(
+internal data class LinkerPhaseInput(
         val outputFile: String,
+        val outputKind: LinkerOutputKind,
         val objectFiles: List<ObjectFile>,
         val dependenciesTrackingResult: DependenciesTrackingResult,
         val outputFiles: OutputFiles,
-        val temporaryFiles: TempFiles,
+        val resolvedCacheBinaries: ResolvedCacheBinaries,
         val isCoverageEnabled: Boolean,
 )
 
@@ -23,6 +25,32 @@ internal val LinkerPhase = createSimpleNamedCompilerPhase<PhaseContext, LinkerPh
         name = "Linker",
         description = "Linker"
 ) { context, input ->
-    val linker = Linker(context, input.isCoverageEnabled, input.temporaryFiles, input.outputFiles)
+    val linker = Linker(
-    linker.link(input.outputFile, input.objectFiles, input.dependenciesTrackingResult)
+            config = context.config,
+            linkerOutput = input.outputKind,
+            isCoverageEnabled = input.isCoverageEnabled,
+            outputFiles = input.outputFiles
+    )
+    val commands = linker.linkCommands(
+            input.outputFile,
+            input.objectFiles,
+            input.dependenciesTrackingResult,
+            input.resolvedCacheBinaries
+    )
+    runLinkerCommands(context, commands, cachingInvolved = !input.resolvedCacheBinaries.isEmpty())
+}
+
+internal data class PreLinkCachesInput(
+        val objectFiles: List<File>,
+        val caches: ResolvedCacheBinaries,
+        val outputObjectFile: File,
+)
+
+internal val PreLinkCachesPhase = createSimpleNamedCompilerPhase<PhaseContext, PreLinkCachesInput>(
+        name = "PreLinkCaches",
+        description = "Pre-link static caches",
+) { context, input ->
+    val inputFiles = input.objectFiles.map { it.absoluteFile.normalize().path } + input.caches.static
+    val commands = context.config.platform.linker.preLinkCommands(inputFiles, input.outputObjectFile.absoluteFile.normalize().path)
+    runLinkerCommands(context, commands, cachingInvolved = true)
 }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt

@@ -157,10 +157,35 @@ internal fun <C : PhaseContext> PhaseEngine<C>.compileAndLink(
         temporaryFiles: TempFiles,
         isCoverageEnabled: Boolean,
 ) {
-    val objectFile = temporaryFiles.create("result", ".o").javaFile()
+    val compilationResult = temporaryFiles.create("result", ".o").javaFile()
-    runPhase(ObjectFilesPhase, ObjectFilesPhaseInput(moduleCompilationOutput.bitcodeFile, objectFile))
+    runPhase(ObjectFilesPhase, ObjectFilesPhaseInput(moduleCompilationOutput.bitcodeFile, compilationResult))
-    val linkerPhaseInput = LinkerPhaseInput(linkerOutputFile, listOf(objectFile.canonicalPath), moduleCompilationOutput.dependenciesTrackingResult,
+    val linkerOutputKind = determineLinkerOutput(context)
-            outputFiles, temporaryFiles, isCoverageEnabled = isCoverageEnabled)
+    val (linkerInput, cacheBinaries) = run {
+        val resolvedCacheBinaries = resolveCacheBinaries(context.config.cachedLibraries, moduleCompilationOutput.dependenciesTrackingResult)
+        when {
+            context.config.produce == CompilerOutputKind.STATIC_CACHE -> {
+                compilationResult to ResolvedCacheBinaries(emptyList(), resolvedCacheBinaries.dynamic)
+            }
+            shouldPerformPreLink(context.config, resolvedCacheBinaries, linkerOutputKind) -> {
+                val prelinkResult = temporaryFiles.create("withStaticCaches", ".o").javaFile()
+                runPhase(PreLinkCachesPhase, PreLinkCachesInput(listOf(compilationResult), resolvedCacheBinaries, prelinkResult))
+                // Static caches are linked into binary, so we don't need to pass them.
+                prelinkResult to ResolvedCacheBinaries(emptyList(), resolvedCacheBinaries.dynamic)
+            }
+            else -> {
+                compilationResult to resolvedCacheBinaries
+            }
+        }
+    }
+    val linkerPhaseInput = LinkerPhaseInput(
+            linkerOutputFile,
+            linkerOutputKind,
+            listOf(linkerInput.canonicalPath),
+            moduleCompilationOutput.dependenciesTrackingResult,
+            outputFiles,
+            cacheBinaries,
+            isCoverageEnabled = isCoverageEnabled
+    )
     runPhase(LinkerPhase, linkerPhaseInput)
     if (context.config.produce.isCache) {
         runPhase(FinalizeCachePhase, outputFiles)