[K/N] Support caches and program outputs in dynamic driver

This commit mostly ports current phases "as-is", without any moving them to an explicit I/O. It will be done later after static driver removal.
2022-12-07 12:46:44 +02:00
parent bf89e27647
commit 4fa701f798
6 changed files with 170 additions and 22 deletions
@@ -170,7 +170,7 @@ private fun linkAllDependencies(generationState: NativeGenerationState, generate
    }
 }

-private fun insertAliasToEntryPoint(generationState: NativeGenerationState) {
+internal fun insertAliasToEntryPoint(generationState: NativeGenerationState) {
    val config = generationState.config
    val nomain = config.configuration.get(KonanConfigKeys.NOMAIN) ?: false
    if (config.produce != CompilerOutputKind.PROGRAM || nomain)
@@ -140,7 +140,8 @@ internal val buildAdditionalCacheInfoPhase = konanUnitPhase(
        },
        name = "BuildAdditionalCacheInfo",
        description = "Build additional cache info (inline functions bodies and fields of classes)",
-        prerequisite = setOf(psiToIrPhase)
+// prerequisites generally do not work in dynamic driver.
+//        prerequisite = setOf(psiToIrPhase)
 )

 internal val destroySymbolTablePhase = konanUnitPhase(
@@ -423,7 +424,7 @@ internal val bitcodePhase = SameTypeNamedCompilerPhase(
                cStubsPhase
 )

-private val bitcodePostprocessingPhase = SameTypeNamedCompilerPhase(
+internal val bitcodePostprocessingPhase = SameTypeNamedCompilerPhase(
        name = "BitcodePostprocessing",
        description = "Optimize and rewrite bitcode",
        lower = checkExternalCallsPhase then
@@ -6,11 +6,9 @@
 package org.jetbrains.kotlin.backend.konan.driver

 import kotlinx.cinterop.usingJvmCInteropCallbacks
+import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.KonanConfig
-import org.jetbrains.kotlin.backend.konan.driver.phases.runFrontend
-import org.jetbrains.kotlin.backend.konan.driver.phases.runPsiToIr
-import org.jetbrains.kotlin.backend.konan.driver.phases.runSerializer
-import org.jetbrains.kotlin.backend.konan.driver.phases.writeKlib
+import org.jetbrains.kotlin.backend.konan.driver.phases.*
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.util.usingNativeMemoryAllocator
@@ -22,7 +20,12 @@ internal class DynamicCompilerDriver : CompilerDriver() {

    companion object {
        fun supportsConfig(config: KonanConfig): Boolean =
-                config.produce == CompilerOutputKind.LIBRARY
+                config.produce in setOf(
+                        CompilerOutputKind.PROGRAM,
+                        CompilerOutputKind.LIBRARY,
+                        CompilerOutputKind.DYNAMIC_CACHE,
+                        CompilerOutputKind.STATIC_CACHE,
+                )
    }

    override fun run(config: KonanConfig, environment: KotlinCoreEnvironment) {
@@ -30,14 +33,14 @@ internal class DynamicCompilerDriver : CompilerDriver() {
            usingJvmCInteropCallbacks {
                PhaseEngine.startTopLevel(config) { engine ->
                    when (config.produce) {
-                        CompilerOutputKind.PROGRAM -> error("Dynamic compiler driver does not support `program` output yet.")
+                        CompilerOutputKind.PROGRAM -> produceBinary(engine, config, environment)
                        CompilerOutputKind.DYNAMIC -> error("Dynamic compiler driver does not support `dynamic` output yet.")
                        CompilerOutputKind.STATIC -> error("Dynamic compiler driver does not support `static` output yet.")
                        CompilerOutputKind.FRAMEWORK -> error("Dynamic compiler driver does not support `framework` output yet.")
                        CompilerOutputKind.LIBRARY -> produceKlib(engine, config, environment)
                        CompilerOutputKind.BITCODE -> error("Dynamic compiler driver does not support `bitcode` output yet.")
-                        CompilerOutputKind.DYNAMIC_CACHE -> error("Dynamic compiler driver does not support `dynamic_cache` output yet.")
-                        CompilerOutputKind.STATIC_CACHE -> error("Dynamic compiler driver does not support `static_cache` output yet.")
+                        CompilerOutputKind.DYNAMIC_CACHE -> produceBinary(engine, config, environment)
+                        CompilerOutputKind.STATIC_CACHE -> produceBinary(engine, config, environment)
                        CompilerOutputKind.PRELIMINARY_CACHE -> TODO()
                    }
                }
@@ -55,4 +58,30 @@ internal class DynamicCompilerDriver : CompilerDriver() {
        val serializerOutput = engine.runSerializer(frontendOutput.moduleDescriptor, psiToIrOutput)
        engine.writeKlib(serializerOutput)
    }
+
+    /**
+     * Produce a single binary artifact.
+     */
+    private fun produceBinary(engine: PhaseEngine<PhaseContext>, config: KonanConfig, environment: KotlinCoreEnvironment) {
+        val frontendOutput = engine.runFrontend(config, environment) ?: return
+        val psiToIrOutput = engine.runPsiToIr(frontendOutput, isProducingLibrary = false)
+        val backendContext = createBackendContext(config, frontendOutput, psiToIrOutput)
+        engine.runBackend(backendContext)
+    }
+
+    private fun createBackendContext(
+            config: KonanConfig,
+            frontendOutput: FrontendPhaseOutput.Full,
+            psiToIrOutput: PsiToIrOutput,
+            additionalDataSetter: (Context) -> Unit = {}
+    ) = Context(
+            config,
+            frontendOutput.environment,
+            frontendOutput.frontendServices,
+            frontendOutput.bindingContext,
+            frontendOutput.moduleDescriptor
+    ).also {
+        it.populateAfterPsiToIr(psiToIrOutput)
+        additionalDataSetter(it)
+    }
 }
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.backend.konan.driver.phases
+
+import llvm.LLVMWriteBitcodeToFile
+import org.jetbrains.kotlin.backend.konan.NativeGenerationState
+import org.jetbrains.kotlin.backend.konan.insertAliasToEntryPoint
+
+/**
+ * Write in-memory LLVM module to filesystem as a bitcode.
+ *
+ * TODO: Use explicit input (LLVMModule) and output (File)
+ *  after static driver removal.
+ */
+
+internal val WriteBitcodeFilePhase = createSimpleNamedCompilerPhase(
+        "WriteBitcodeFile",
+        "Write bitcode file",
+        outputIfNotEnabled = { _, _, _, _ -> }
+) { context: NativeGenerationState, _: Unit ->
+    val output = context.tempFiles.nativeBinaryFileName
+    context.bitcodeFileName = output
+    // Insert `_main` after pipeline, so we won't worry about optimizations corrupting entry point.
+    insertAliasToEntryPoint(context)
+    LLVMWriteBitcodeToFile(context.llvm.module, output)
+}
@@ -10,7 +10,6 @@ import org.jetbrains.kotlin.backend.konan.KonanConfig
 import org.jetbrains.kotlin.backend.konan.KonanReflectionTypes
 import org.jetbrains.kotlin.backend.konan.driver.BasicPhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
-import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine
 import org.jetbrains.kotlin.backend.konan.ir.KonanSymbols
 import org.jetbrains.kotlin.backend.konan.psiToIr
 import org.jetbrains.kotlin.backend.konan.serialization.KonanIdSignaturer
@@ -91,12 +90,4 @@ internal val PsiToIrPhase = createSimpleNamedCompilerPhase<PsiToIrContext, PsiTo
        outputIfNotEnabled = { _, _, _, _ -> error("PsiToIr phase cannot be disabled") }
 ) { context, input ->
    context.psiToIr(input, useLinkerWhenProducingLibrary = false)
-}
-
-internal fun <T : PsiToIrContext> PhaseEngine<T>.runPsiToIr(
-        frontendResult: FrontendPhaseOutput.Full,
-        isProducingLibrary: Boolean
-): PsiToIrOutput {
-    val psiToIrInput = PsiToIrInput(frontendResult.moduleDescriptor, frontendResult.environment, isProducingLibrary)
-    return this.runPhase(PsiToIrPhase, psiToIrInput)
 }
@@ -5,10 +5,17 @@

 package org.jetbrains.kotlin.backend.konan.driver.phases

-import org.jetbrains.kotlin.backend.konan.KonanConfig
+import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine
+import org.jetbrains.kotlin.backend.konan.driver.runPhaseInParentContext
+import org.jetbrains.kotlin.backend.konan.llvm.linkBitcodeDependenciesPhase
+import org.jetbrains.kotlin.backend.konan.llvm.printBitcodePhase
+import org.jetbrains.kotlin.backend.konan.llvm.verifyBitcodePhase
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.declarations.path
+import org.jetbrains.kotlin.konan.target.CompilerOutputKind

 internal fun PhaseEngine<PhaseContext>.runFrontend(config: KonanConfig, environment: KotlinCoreEnvironment): FrontendPhaseOutput.Full? {
    val frontendOutput = useContext(FrontendContextImpl(config)) { it.runFrontend(environment) }
@@ -29,10 +36,101 @@ internal fun <T> PhaseEngine<PhaseContext>.runPsiToIr(
    val psiToIrContext = PsiToIrContextImpl(config, frontendOutput.moduleDescriptor, frontendOutput.bindingContext)
    val (psiToIrOutput, additionalOutput) = useContext(psiToIrContext) { psiToIrEngine ->
        val additionalOutput = produceAdditionalOutput(psiToIrEngine)
-        val output = psiToIrEngine.runPsiToIr(frontendOutput, isProducingLibrary)
+        val psiToIrInput = PsiToIrInput(frontendOutput.moduleDescriptor, frontendOutput.environment, isProducingLibrary)
+        val output = psiToIrEngine.runPhase(PsiToIrPhase, psiToIrInput)
        psiToIrEngine.runSpecialBackendChecks(output)
        output to additionalOutput
    }
    runPhase(CopyDefaultValuesToActualPhase, psiToIrOutput.irModule)
    return psiToIrOutput to additionalOutput
+}
+
+internal fun <C : PhaseContext> PhaseEngine<C>.runBackend(backendContext: Context) {
+    useContext(backendContext) { backendEngine ->
+        backendEngine.runPhase(functionsWithoutBoundCheck)
+        backendEngine.processModuleFragments(backendEngine.context.irModule!!) { generationState, fragment ->
+            backendEngine.useContext(generationState) { generationStateEngine ->
+                // TODO: We can run compile part in parallel if we get rid of context.generationState.
+                generationStateEngine.runLowerAndCompile(fragment)
+            }
+        }
+    }
+}
+
+internal fun PhaseEngine<out Context>.processModuleFragments(
+        input: IrModuleFragment,
+        action: (NativeGenerationState, IrModuleFragment) -> Unit
+): Unit = if (context.config.producePerFileCache) {
+    val module = context.irModules[context.config.libraryToCache!!.klib.libraryName]
+            ?: error("No module for the library being cached: ${context.config.libraryToCache!!.klib.libraryName}")
+
+    val files = module.files.toList()
+    module.files.clear()
+    val functionInterfaceFiles = files.filter { it.isFunctionInterfaceFile }
+
+    for (file in files) {
+        if (file.isFunctionInterfaceFile) continue
+
+        context.generationState = NativeGenerationState(
+                context.config,
+                context,
+                CacheDeserializationStrategy.SingleFile(file.path, file.fqName.asString())
+        )
+
+        module.files += file
+        if (context.generationState.shouldDefineFunctionClasses)
+            module.files += functionInterfaceFiles
+
+        action(context.generationState, input)
+
+        module.files.clear()
+        context.irModule!!.files.clear() // [dependenciesLowerPhase] puts all files to [context.irModule] for codegen.
+    }
+
+    module.files += files
+} else {
+    context.generationState = NativeGenerationState(context.config, context, context.config.libraryToCache?.strategy)
+    action(context.generationState, input)
+}
+
+/**
+ * Performs all the hard work:
+ * 1. Runs IR lowerings
+ * 2. Runs LTO.
+ * 3. Translates IR to LLVM IR.
+ * 4. Optimizes it.
+ * 5. Serializes it to a bitcode file.
+ * 6. Compiles bitcode to an object file.
+ * 7. Performs binary linkage.
+ * ... And stores additional cache info.
+ *
+ * TODO: Split into more granular phases with explicit inputs and outputs.
+ */
+internal fun PhaseEngine<NativeGenerationState>.runLowerAndCompile(module: IrModuleFragment) {
+    if (context.config.produce.isCache) {
+        runPhaseInParentContext(buildAdditionalCacheInfoPhase)
+    }
+    if (context.config.produce == CompilerOutputKind.PROGRAM) {
+        runPhaseInParentContext(entryPointPhase, module)
+    }
+    runBackendCodegen(module)
+    if (context.config.produce.isCache) {
+        runPhaseInParentContext(saveAdditionalCacheInfoPhase)
+    }
+    runPhase(WriteBitcodeFilePhase)
+    runPhaseInParentContext(objectFilesPhase)
+    runPhaseInParentContext(linkerPhase)
+    if (context.config.produce.isCache) {
+        runPhaseInParentContext(finalizeCachePhase)
+    }
+}
+
+internal fun PhaseEngine<NativeGenerationState>.runBackendCodegen(module: IrModuleFragment) {
+    runPhaseInParentContext(allLoweringsPhase, module)
+    runPhaseInParentContext(dependenciesLowerPhase, module)
+    runPhaseInParentContext(bitcodePhase, module)
+    runPhaseInParentContext(verifyBitcodePhase, module)
+    runPhaseInParentContext(printBitcodePhase, module)
+    runPhaseInParentContext(linkBitcodeDependenciesPhase, module)
+    runPhaseInParentContext(bitcodePostprocessingPhase, module)
 }