From 4fa701f7980f8aa931c54c9cad045519c9d14df9 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Wed, 7 Dec 2022 12:46:44 +0200 Subject: [PATCH] [K/N] Support caches and program outputs in dynamic driver This commit mostly ports current phases "as-is", without any moving them to an explicit I/O. It will be done later after static driver removal. --- .../kotlin/backend/konan/CompilerOutput.kt | 2 +- .../kotlin/backend/konan/ToplevelPhases.kt | 5 +- .../konan/driver/DynamicCompilerDriver.kt | 45 ++++++-- .../backend/konan/driver/phases/Bitcode.kt | 29 +++++ .../backend/konan/driver/phases/PsiToIr.kt | 9 -- .../konan/driver/phases/TopLevelPhases.kt | 102 +++++++++++++++++- 6 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Bitcode.kt diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt index 6f39c440292..0eb79131df9 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt @@ -170,7 +170,7 @@ private fun linkAllDependencies(generationState: NativeGenerationState, generate } } -private fun insertAliasToEntryPoint(generationState: NativeGenerationState) { +internal fun insertAliasToEntryPoint(generationState: NativeGenerationState) { val config = generationState.config val nomain = config.configuration.get(KonanConfigKeys.NOMAIN) ?: false if (config.produce != CompilerOutputKind.PROGRAM || nomain) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt index 197a10a159c..7167567e80e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt @@ -140,7 +140,8 @@ internal val buildAdditionalCacheInfoPhase = konanUnitPhase( }, name = "BuildAdditionalCacheInfo", description = "Build additional cache info (inline functions bodies and fields of classes)", - prerequisite = setOf(psiToIrPhase) +// prerequisites generally do not work in dynamic driver. +// prerequisite = setOf(psiToIrPhase) ) internal val destroySymbolTablePhase = konanUnitPhase( @@ -423,7 +424,7 @@ internal val bitcodePhase = SameTypeNamedCompilerPhase( cStubsPhase ) -private val bitcodePostprocessingPhase = SameTypeNamedCompilerPhase( +internal val bitcodePostprocessingPhase = SameTypeNamedCompilerPhase( name = "BitcodePostprocessing", description = "Optimize and rewrite bitcode", lower = checkExternalCallsPhase then diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt index c89ff0b4be9..40aaa41554a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt @@ -6,11 +6,9 @@ package org.jetbrains.kotlin.backend.konan.driver import kotlinx.cinterop.usingJvmCInteropCallbacks +import org.jetbrains.kotlin.backend.konan.Context import org.jetbrains.kotlin.backend.konan.KonanConfig -import org.jetbrains.kotlin.backend.konan.driver.phases.runFrontend -import org.jetbrains.kotlin.backend.konan.driver.phases.runPsiToIr -import org.jetbrains.kotlin.backend.konan.driver.phases.runSerializer -import org.jetbrains.kotlin.backend.konan.driver.phases.writeKlib +import org.jetbrains.kotlin.backend.konan.driver.phases.* import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.util.usingNativeMemoryAllocator @@ -22,7 +20,12 @@ internal class DynamicCompilerDriver : CompilerDriver() { companion object { fun supportsConfig(config: KonanConfig): Boolean = - config.produce == CompilerOutputKind.LIBRARY + config.produce in setOf( + CompilerOutputKind.PROGRAM, + CompilerOutputKind.LIBRARY, + CompilerOutputKind.DYNAMIC_CACHE, + CompilerOutputKind.STATIC_CACHE, + ) } override fun run(config: KonanConfig, environment: KotlinCoreEnvironment) { @@ -30,14 +33,14 @@ internal class DynamicCompilerDriver : CompilerDriver() { usingJvmCInteropCallbacks { PhaseEngine.startTopLevel(config) { engine -> when (config.produce) { - CompilerOutputKind.PROGRAM -> error("Dynamic compiler driver does not support `program` output yet.") + CompilerOutputKind.PROGRAM -> produceBinary(engine, config, environment) CompilerOutputKind.DYNAMIC -> error("Dynamic compiler driver does not support `dynamic` output yet.") CompilerOutputKind.STATIC -> error("Dynamic compiler driver does not support `static` output yet.") CompilerOutputKind.FRAMEWORK -> error("Dynamic compiler driver does not support `framework` output yet.") CompilerOutputKind.LIBRARY -> produceKlib(engine, config, environment) CompilerOutputKind.BITCODE -> error("Dynamic compiler driver does not support `bitcode` output yet.") - CompilerOutputKind.DYNAMIC_CACHE -> error("Dynamic compiler driver does not support `dynamic_cache` output yet.") - CompilerOutputKind.STATIC_CACHE -> error("Dynamic compiler driver does not support `static_cache` output yet.") + CompilerOutputKind.DYNAMIC_CACHE -> produceBinary(engine, config, environment) + CompilerOutputKind.STATIC_CACHE -> produceBinary(engine, config, environment) CompilerOutputKind.PRELIMINARY_CACHE -> TODO() } } @@ -55,4 +58,30 @@ internal class DynamicCompilerDriver : CompilerDriver() { val serializerOutput = engine.runSerializer(frontendOutput.moduleDescriptor, psiToIrOutput) engine.writeKlib(serializerOutput) } + + /** + * Produce a single binary artifact. + */ + private fun produceBinary(engine: PhaseEngine, config: KonanConfig, environment: KotlinCoreEnvironment) { + val frontendOutput = engine.runFrontend(config, environment) ?: return + val psiToIrOutput = engine.runPsiToIr(frontendOutput, isProducingLibrary = false) + val backendContext = createBackendContext(config, frontendOutput, psiToIrOutput) + engine.runBackend(backendContext) + } + + private fun createBackendContext( + config: KonanConfig, + frontendOutput: FrontendPhaseOutput.Full, + psiToIrOutput: PsiToIrOutput, + additionalDataSetter: (Context) -> Unit = {} + ) = Context( + config, + frontendOutput.environment, + frontendOutput.frontendServices, + frontendOutput.bindingContext, + frontendOutput.moduleDescriptor + ).also { + it.populateAfterPsiToIr(psiToIrOutput) + additionalDataSetter(it) + } } \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Bitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Bitcode.kt new file mode 100644 index 00000000000..e2847ccdc8b --- /dev/null +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Bitcode.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.konan.driver.phases + +import llvm.LLVMWriteBitcodeToFile +import org.jetbrains.kotlin.backend.konan.NativeGenerationState +import org.jetbrains.kotlin.backend.konan.insertAliasToEntryPoint + +/** + * Write in-memory LLVM module to filesystem as a bitcode. + * + * TODO: Use explicit input (LLVMModule) and output (File) + * after static driver removal. + */ + +internal val WriteBitcodeFilePhase = createSimpleNamedCompilerPhase( + "WriteBitcodeFile", + "Write bitcode file", + outputIfNotEnabled = { _, _, _, _ -> } +) { context: NativeGenerationState, _: Unit -> + val output = context.tempFiles.nativeBinaryFileName + context.bitcodeFileName = output + // Insert `_main` after pipeline, so we won't worry about optimizations corrupting entry point. + insertAliasToEntryPoint(context) + LLVMWriteBitcodeToFile(context.llvm.module, output) +} \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/PsiToIr.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/PsiToIr.kt index e70e14af3de..596df6f614c 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/PsiToIr.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/PsiToIr.kt @@ -10,7 +10,6 @@ import org.jetbrains.kotlin.backend.konan.KonanConfig import org.jetbrains.kotlin.backend.konan.KonanReflectionTypes import org.jetbrains.kotlin.backend.konan.driver.BasicPhaseContext import org.jetbrains.kotlin.backend.konan.driver.PhaseContext -import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine import org.jetbrains.kotlin.backend.konan.ir.KonanSymbols import org.jetbrains.kotlin.backend.konan.psiToIr import org.jetbrains.kotlin.backend.konan.serialization.KonanIdSignaturer @@ -91,12 +90,4 @@ internal val PsiToIrPhase = createSimpleNamedCompilerPhase error("PsiToIr phase cannot be disabled") } ) { context, input -> context.psiToIr(input, useLinkerWhenProducingLibrary = false) -} - -internal fun PhaseEngine.runPsiToIr( - frontendResult: FrontendPhaseOutput.Full, - isProducingLibrary: Boolean -): PsiToIrOutput { - val psiToIrInput = PsiToIrInput(frontendResult.moduleDescriptor, frontendResult.environment, isProducingLibrary) - return this.runPhase(PsiToIrPhase, psiToIrInput) } \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt index 9908d646ce2..6b56b3ee7ff 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt @@ -5,10 +5,17 @@ package org.jetbrains.kotlin.backend.konan.driver.phases -import org.jetbrains.kotlin.backend.konan.KonanConfig +import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.driver.PhaseContext import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine +import org.jetbrains.kotlin.backend.konan.driver.runPhaseInParentContext +import org.jetbrains.kotlin.backend.konan.llvm.linkBitcodeDependenciesPhase +import org.jetbrains.kotlin.backend.konan.llvm.printBitcodePhase +import org.jetbrains.kotlin.backend.konan.llvm.verifyBitcodePhase import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.declarations.path +import org.jetbrains.kotlin.konan.target.CompilerOutputKind internal fun PhaseEngine.runFrontend(config: KonanConfig, environment: KotlinCoreEnvironment): FrontendPhaseOutput.Full? { val frontendOutput = useContext(FrontendContextImpl(config)) { it.runFrontend(environment) } @@ -29,10 +36,101 @@ internal fun PhaseEngine.runPsiToIr( val psiToIrContext = PsiToIrContextImpl(config, frontendOutput.moduleDescriptor, frontendOutput.bindingContext) val (psiToIrOutput, additionalOutput) = useContext(psiToIrContext) { psiToIrEngine -> val additionalOutput = produceAdditionalOutput(psiToIrEngine) - val output = psiToIrEngine.runPsiToIr(frontendOutput, isProducingLibrary) + val psiToIrInput = PsiToIrInput(frontendOutput.moduleDescriptor, frontendOutput.environment, isProducingLibrary) + val output = psiToIrEngine.runPhase(PsiToIrPhase, psiToIrInput) psiToIrEngine.runSpecialBackendChecks(output) output to additionalOutput } runPhase(CopyDefaultValuesToActualPhase, psiToIrOutput.irModule) return psiToIrOutput to additionalOutput +} + +internal fun PhaseEngine.runBackend(backendContext: Context) { + useContext(backendContext) { backendEngine -> + backendEngine.runPhase(functionsWithoutBoundCheck) + backendEngine.processModuleFragments(backendEngine.context.irModule!!) { generationState, fragment -> + backendEngine.useContext(generationState) { generationStateEngine -> + // TODO: We can run compile part in parallel if we get rid of context.generationState. + generationStateEngine.runLowerAndCompile(fragment) + } + } + } +} + +internal fun PhaseEngine.processModuleFragments( + input: IrModuleFragment, + action: (NativeGenerationState, IrModuleFragment) -> Unit +): Unit = if (context.config.producePerFileCache) { + val module = context.irModules[context.config.libraryToCache!!.klib.libraryName] + ?: error("No module for the library being cached: ${context.config.libraryToCache!!.klib.libraryName}") + + val files = module.files.toList() + module.files.clear() + val functionInterfaceFiles = files.filter { it.isFunctionInterfaceFile } + + for (file in files) { + if (file.isFunctionInterfaceFile) continue + + context.generationState = NativeGenerationState( + context.config, + context, + CacheDeserializationStrategy.SingleFile(file.path, file.fqName.asString()) + ) + + module.files += file + if (context.generationState.shouldDefineFunctionClasses) + module.files += functionInterfaceFiles + + action(context.generationState, input) + + module.files.clear() + context.irModule!!.files.clear() // [dependenciesLowerPhase] puts all files to [context.irModule] for codegen. + } + + module.files += files +} else { + context.generationState = NativeGenerationState(context.config, context, context.config.libraryToCache?.strategy) + action(context.generationState, input) +} + +/** + * Performs all the hard work: + * 1. Runs IR lowerings + * 2. Runs LTO. + * 3. Translates IR to LLVM IR. + * 4. Optimizes it. + * 5. Serializes it to a bitcode file. + * 6. Compiles bitcode to an object file. + * 7. Performs binary linkage. + * ... And stores additional cache info. + * + * TODO: Split into more granular phases with explicit inputs and outputs. + */ +internal fun PhaseEngine.runLowerAndCompile(module: IrModuleFragment) { + if (context.config.produce.isCache) { + runPhaseInParentContext(buildAdditionalCacheInfoPhase) + } + if (context.config.produce == CompilerOutputKind.PROGRAM) { + runPhaseInParentContext(entryPointPhase, module) + } + runBackendCodegen(module) + if (context.config.produce.isCache) { + runPhaseInParentContext(saveAdditionalCacheInfoPhase) + } + runPhase(WriteBitcodeFilePhase) + runPhaseInParentContext(objectFilesPhase) + runPhaseInParentContext(linkerPhase) + if (context.config.produce.isCache) { + runPhaseInParentContext(finalizeCachePhase) + } +} + +internal fun PhaseEngine.runBackendCodegen(module: IrModuleFragment) { + runPhaseInParentContext(allLoweringsPhase, module) + runPhaseInParentContext(dependenciesLowerPhase, module) + runPhaseInParentContext(bitcodePhase, module) + runPhaseInParentContext(verifyBitcodePhase, module) + runPhaseInParentContext(printBitcodePhase, module) + runPhaseInParentContext(linkBitcodeDependenciesPhase, module) + runPhaseInParentContext(bitcodePostprocessingPhase, module) } \ No newline at end of file