[K/N] Extract pre-link from linker and a bit of refactoring

Finish extracting all usages of TempFiles to the driver.
2023-01-25 17:42:06 +02:00
parent e6081e5851
commit 6a06be08cd
4 changed files with 111 additions and 93 deletions
@@ -24,7 +24,9 @@ internal fun shouldPerformPreLink(config: KonanConfig, caches: ResolvedCacheBina
 * [static] is a list of static libraries (e.g. "libcache.a")
 * [dynamic] is a list of dynamic libraries (e.g. "libcache.dylib")
 */
-internal class ResolvedCacheBinaries(val static: List<String>, val dynamic: List<String>)
+internal class ResolvedCacheBinaries(val static: List<String>, val dynamic: List<String>) {
+    fun isEmpty(): Boolean = static.isEmpty() && dynamic.isEmpty()
+}

 /**
 * Find binary files for compiler caches that are actually required for the linkage.
@@ -2,7 +2,6 @@ package org.jetbrains.kotlin.backend.konan

 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.konan.KonanExternalToolFailure
-import org.jetbrains.kotlin.konan.TempFiles
 import org.jetbrains.kotlin.konan.exec.Command
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.KonanLibrary
@@ -36,25 +35,23 @@ internal fun determineLinkerOutput(context: PhaseContext): LinkerOutputKind =

 // TODO: We have a Linker.kt file in the shared module.
 internal class Linker(
-        private val context: PhaseContext,
+        private val config: KonanConfig,
+        private val linkerOutput: LinkerOutputKind,
        private val isCoverageEnabled: Boolean = false,
-        private val tempFiles: TempFiles,
        private val outputFiles: OutputFiles,
 ) {
-
-    private val config = context.config
    private val platform = config.platform
-    private val linkerOutput = determineLinkerOutput(context)
    private val linker = platform.linker
    private val target = config.target
-    private val optimize = context.shouldOptimize()
+    private val optimize = config.optimizationsEnabled
    private val debug = config.debug || config.lightDebug

-    fun link(
+    fun linkCommands(
            outputFile: String,
            objectFiles: List<ObjectFile>,
-            dependenciesTrackingResult: DependenciesTrackingResult
-    ) {
+            dependenciesTrackingResult: DependenciesTrackingResult,
+            caches: ResolvedCacheBinaries,
+    ): List<Command> {
        val nativeDependencies = dependenciesTrackingResult.nativeDependenciesToLink

        val includedBinariesLibraries = config.libraryToCache?.let { listOf(it.klib) }
@@ -62,8 +59,7 @@ internal class Linker(
        val includedBinaries = includedBinariesLibraries.map { (it as? KonanLibrary)?.includedPaths.orEmpty() }.flatten()

        val libraryProvidedLinkerFlags = dependenciesTrackingResult.allNativeDependencies.map { it.linkerOpts }.flatten()
-
-        runLinker(outputFile, objectFiles, includedBinaries, libraryProvidedLinkerFlags, dependenciesTrackingResult)
+        return runLinker(outputFile, objectFiles, includedBinaries, libraryProvidedLinkerFlags, caches)
    }

    private fun asLinkerArgs(args: List<String>): List<String> {
@@ -88,8 +84,8 @@ internal class Linker(
            objectFiles: List<ObjectFile>,
            includedBinaries: List<String>,
            libraryProvidedLinkerFlags: List<String>,
-            dependenciesTrackingResult: DependenciesTrackingResult,
-    ): ExecutableFile {
+            caches: ResolvedCacheBinaries,
+    ): List<Command> {
        val additionalLinkerArgs: List<String>
        val executable: String

@@ -119,77 +115,44 @@ internal class Linker(
            dylibPath.parentFile.mkdirs()
            executable = dylibPath.absolutePath
        }
+        File(executable).delete()

-        val mimallocEnabled = config.allocationMode == AllocationMode.MIMALLOC
+        val linkerArgs = asLinkerArgs(config.configuration.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
+                BitcodeEmbedding.getLinkerOptions(config) +
+                caches.dynamic +
+                libraryProvidedLinkerFlags + additionalLinkerArgs

-        val linkerInput = determineLinkerInput(objectFiles, linkerOutput, dependenciesTrackingResult)
-        try {
-            File(executable).delete()
-            val linkerArgs = asLinkerArgs(config.configuration.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
-                    BitcodeEmbedding.getLinkerOptions(config) +
-                    linkerInput.caches.dynamic +
-                    libraryProvidedLinkerFlags + additionalLinkerArgs
-
-            val finalOutputCommands = linker.finalLinkCommands(
-                    objectFiles = linkerInput.objectFiles,
-                    executable = executable,
-                    libraries = linker.linkStaticLibraries(includedBinaries) + linkerInput.caches.static,
-                    linkerArgs = linkerArgs,
-                    optimize = optimize,
-                    debug = debug,
-                    kind = linkerOutput,
-                    outputDsymBundle = outputFiles.symbolicInfoFile,
-                    needsProfileLibrary = isCoverageEnabled,
-                    mimallocEnabled = mimallocEnabled,
-                    sanitizer = config.sanitizer
-            )
-            (linkerInput.preLinkCommands + finalOutputCommands).forEach {
-                it.logWith(context::log)
-                it.execute()
-            }
-        } catch (e: KonanExternalToolFailure) {
-            val extraUserInfo =
-                    if (linkerInput.cachingInvolved)
-                        """
-                        Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
-                            
-                            kotlin.native.cacheKind.${target.presetName}=none
-                            
-                        Also, consider filing an issue with full Gradle log here: https://kotl.in/issue
-                        """.trimIndent()
-                    else ""
-            context.reportCompilationError("${e.toolName} invocation reported errors\n$extraUserInfo\n${e.message}")
-        }
-        return executable
-    }
-
-    private fun determineLinkerInput(
-            objectFiles: List<ObjectFile>,
-            linkerOutputKind: LinkerOutputKind,
-            dependenciesTrackingResult: DependenciesTrackingResult,
-    ): LinkerInput {
-        val caches = resolveCacheBinaries(context.config.cachedLibraries, dependenciesTrackingResult)
-        // Since we have several linker stages that involve caching,
-        // we should detect cache usage early to report errors correctly.
-        val cachingInvolved = caches.static.isNotEmpty() || caches.dynamic.isNotEmpty()
-        return when {
-            config.produce == CompilerOutputKind.STATIC_CACHE -> {
-                // Do not link static cache dependencies.
-                LinkerInput(objectFiles, ResolvedCacheBinaries(emptyList(), caches.dynamic), emptyList(), cachingInvolved)
-            }
-            shouldPerformPreLink(config, caches, linkerOutputKind) -> {
-                val preLinkResult = tempFiles.create("withStaticCaches", ".o").absolutePath
-                val preLinkCommands = linker.preLinkCommands(objectFiles + caches.static, preLinkResult)
-                LinkerInput(listOf(preLinkResult), ResolvedCacheBinaries(emptyList(), caches.dynamic), preLinkCommands, cachingInvolved)
-            }
-            else -> LinkerInput(objectFiles, caches, emptyList(), cachingInvolved)
-        }
+        return linker.finalLinkCommands(
+                objectFiles = objectFiles,
+                executable = executable,
+                libraries = linker.linkStaticLibraries(includedBinaries) + caches.static,
+                linkerArgs = linkerArgs,
+                optimize = optimize,
+                debug = debug,
+                kind = linkerOutput,
+                outputDsymBundle = outputFiles.symbolicInfoFile,
+                needsProfileLibrary = isCoverageEnabled,
+                mimallocEnabled = config.allocationMode == AllocationMode.MIMALLOC,
+                sanitizer = config.sanitizer
+        )
    }
 }

-private class LinkerInput(
-        val objectFiles: List<ObjectFile>,
-        val caches: ResolvedCacheBinaries,
-        val preLinkCommands: List<Command>,
-        val cachingInvolved: Boolean
-)
+internal fun runLinkerCommands(context: PhaseContext, commands: List<Command>, cachingInvolved: Boolean) = try {
+    commands.forEach {
+        it.logWith(context::log)
+        it.execute()
+    }
+} catch (e: KonanExternalToolFailure) {
+    val extraUserInfo =
+            if (cachingInvolved)
+                """
+                        Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
+                            
+                            kotlin.native.cacheKind.${context.config.target.presetName}=none
+                            
+                        Also, consider filing an issue with full Gradle log here: https://kotl.in/issue
+                        """.trimIndent()
+            else ""
+    context.reportCompilationError("${e.toolName} invocation reported errors\n$extraUserInfo\n${e.message}")
+}
@@ -8,14 +8,16 @@ package org.jetbrains.kotlin.backend.konan.driver.phases
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.Linker
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
-import org.jetbrains.kotlin.konan.TempFiles
+import org.jetbrains.kotlin.konan.target.LinkerOutputKind
+import java.io.File

-data class LinkerPhaseInput(
+internal data class LinkerPhaseInput(
        val outputFile: String,
+        val outputKind: LinkerOutputKind,
        val objectFiles: List<ObjectFile>,
        val dependenciesTrackingResult: DependenciesTrackingResult,
        val outputFiles: OutputFiles,
-        val temporaryFiles: TempFiles,
+        val resolvedCacheBinaries: ResolvedCacheBinaries,
        val isCoverageEnabled: Boolean,
 )

@@ -23,6 +25,32 @@ internal val LinkerPhase = createSimpleNamedCompilerPhase<PhaseContext, LinkerPh
        name = "Linker",
        description = "Linker"
 ) { context, input ->
-    val linker = Linker(context, input.isCoverageEnabled, input.temporaryFiles, input.outputFiles)
-    linker.link(input.outputFile, input.objectFiles, input.dependenciesTrackingResult)
+    val linker = Linker(
+            config = context.config,
+            linkerOutput = input.outputKind,
+            isCoverageEnabled = input.isCoverageEnabled,
+            outputFiles = input.outputFiles
+    )
+    val commands = linker.linkCommands(
+            input.outputFile,
+            input.objectFiles,
+            input.dependenciesTrackingResult,
+            input.resolvedCacheBinaries
+    )
+    runLinkerCommands(context, commands, cachingInvolved = !input.resolvedCacheBinaries.isEmpty())
+}
+
+internal data class PreLinkCachesInput(
+        val objectFiles: List<File>,
+        val caches: ResolvedCacheBinaries,
+        val outputObjectFile: File,
+)
+
+internal val PreLinkCachesPhase = createSimpleNamedCompilerPhase<PhaseContext, PreLinkCachesInput>(
+        name = "PreLinkCaches",
+        description = "Pre-link static caches",
+) { context, input ->
+    val inputFiles = input.objectFiles.map { it.absoluteFile.normalize().path } + input.caches.static
+    val commands = context.config.platform.linker.preLinkCommands(inputFiles, input.outputObjectFile.absoluteFile.normalize().path)
+    runLinkerCommands(context, commands, cachingInvolved = true)
 }
@@ -157,10 +157,35 @@ internal fun <C : PhaseContext> PhaseEngine<C>.compileAndLink(
        temporaryFiles: TempFiles,
        isCoverageEnabled: Boolean,
 ) {
-    val objectFile = temporaryFiles.create("result", ".o").javaFile()
-    runPhase(ObjectFilesPhase, ObjectFilesPhaseInput(moduleCompilationOutput.bitcodeFile, objectFile))
-    val linkerPhaseInput = LinkerPhaseInput(linkerOutputFile, listOf(objectFile.canonicalPath), moduleCompilationOutput.dependenciesTrackingResult,
-            outputFiles, temporaryFiles, isCoverageEnabled = isCoverageEnabled)
+    val compilationResult = temporaryFiles.create("result", ".o").javaFile()
+    runPhase(ObjectFilesPhase, ObjectFilesPhaseInput(moduleCompilationOutput.bitcodeFile, compilationResult))
+    val linkerOutputKind = determineLinkerOutput(context)
+    val (linkerInput, cacheBinaries) = run {
+        val resolvedCacheBinaries = resolveCacheBinaries(context.config.cachedLibraries, moduleCompilationOutput.dependenciesTrackingResult)
+        when {
+            context.config.produce == CompilerOutputKind.STATIC_CACHE -> {
+                compilationResult to ResolvedCacheBinaries(emptyList(), resolvedCacheBinaries.dynamic)
+            }
+            shouldPerformPreLink(context.config, resolvedCacheBinaries, linkerOutputKind) -> {
+                val prelinkResult = temporaryFiles.create("withStaticCaches", ".o").javaFile()
+                runPhase(PreLinkCachesPhase, PreLinkCachesInput(listOf(compilationResult), resolvedCacheBinaries, prelinkResult))
+                // Static caches are linked into binary, so we don't need to pass them.
+                prelinkResult to ResolvedCacheBinaries(emptyList(), resolvedCacheBinaries.dynamic)
+            }
+            else -> {
+                compilationResult to resolvedCacheBinaries
+            }
+        }
+    }
+    val linkerPhaseInput = LinkerPhaseInput(
+            linkerOutputFile,
+            linkerOutputKind,
+            listOf(linkerInput.canonicalPath),
+            moduleCompilationOutput.dependenciesTrackingResult,
+            outputFiles,
+            cacheBinaries,
+            isCoverageEnabled = isCoverageEnabled
+    )
    runPhase(LinkerPhase, linkerPhaseInput)
    if (context.config.produce.isCache) {
        runPhase(FinalizeCachePhase, outputFiles)