From 74c57e6057fbd004e3665dfd1d774f1e0c879b00 Mon Sep 17 00:00:00 2001 From: Vladimir Sukharev Date: Wed, 24 May 2023 13:07:12 +0000 Subject: [PATCH] [K2/N] Compile to native binary in two stages ^KT-56855 Fixed Merge-request: KT-MR-10219 Merged-by: Vladimir Sukharev --- .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 39 +++++++++++++++++++ .../backend/konan/driver/phases/Frontend.kt | 16 ++------ .../backend.native/tests/build.gradle | 7 +--- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index bcb3365ba0a..dd42172edd5 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -13,6 +13,7 @@ import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.cli.common.* import org.jetbrains.kotlin.cli.common.arguments.K2NativeCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments +import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.* import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.cli.common.messages.MessageRenderer @@ -26,11 +27,14 @@ import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.ir.linkage.partial.partialLinkageConfig import org.jetbrains.kotlin.ir.linkage.partial.setupPartialLinkageConfig import org.jetbrains.kotlin.ir.util.IrMessageLogger +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.util.profile import org.jetbrains.kotlin.utils.KotlinPaths +import java.nio.file.Files private class K2NativeCompilerPerformanceManager: CommonCompilerPerformanceManager("Kotlin to Native Compiler") @@ -64,7 +68,42 @@ class K2Native : CLICompiler() { if (!enoughArguments) { messageCollector.report(ERROR, "You have not specified any compilation arguments. No output has been produced.") } + if(configuration.get(KonanConfigKeys.PRODUCE) != CompilerOutputKind.LIBRARY && + configuration.getBoolean(CommonConfigurationKeys.USE_FIR) && + configuration.kotlinSourceRoots.isNotEmpty()) { + // K2/Native backend cannot produce binary directly from FIR frontend output, since descriptors, deserialized from KLib, are needed + // So, such compilation is split to two stages: + // - source files are compiled to intermediate KLib by FIR frontend + // - intermediate Klib is compiled to binary by K2/Native backend + // In this implementation, 'arguments' is not changed accordingly to changes in `firstStageConfiguration` and `configuration`, + // since values of fields `produce`, `output`, `freeArgs`, `includes` does not seem to matter downstream in prepareEnvironment() + val firstStageConfiguration = configuration.copy() + // For the first stage, use "-p library" produce mode + firstStageConfiguration.put(KonanConfigKeys.PRODUCE, CompilerOutputKind.LIBRARY) + // For the first stage, construct a temporary file name for an intermediate KLib + val originalOutput = firstStageConfiguration.get(KonanConfigKeys.OUTPUT) + val intermediateKLib = Files.createTempFile(File(originalOutput ?: "intermediate").name, ".klib").toString() + firstStageConfiguration.put(KonanConfigKeys.OUTPUT, intermediateKLib) + + val firstStageExitCode = executeStage(firstStageConfiguration, arguments, rootDisposable) + if (firstStageExitCode != ExitCode.OK) + return firstStageExitCode + + // For the second stage, remove already compiled source files from the configuration + configuration.put(CLIConfigurationKeys.CONTENT_ROOTS, listOf()) + // For the second stage, provide just compiled intermediate KLib as "-Xinclude=" param + configuration.put(KonanConfigKeys.INCLUDED_LIBRARIES, listOf(intermediateKLib)) + // Now, `configuration` param is prepared for the second stage of compilation, and `arguments` param does not need changes, as noted above. + } + return executeStage(configuration, arguments, rootDisposable) + } + + private fun executeStage( + configuration: CompilerConfiguration, + arguments: K2NativeCompilerArguments, + rootDisposable: Disposable + ): ExitCode { val environment = prepareEnvironment(arguments, configuration, rootDisposable) try { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt index 2cf8e667608..acad0908d16 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt @@ -57,19 +57,9 @@ internal val FrontendPhase = createSimpleNamedCompilerPhase( val sourceFiles = input.getSourceFiles() if (sourceFiles.isNotEmpty()) { - if (input.configuration.getBoolean(CommonConfigurationKeys.USE_FIR)) { - context.reportCompilationError( - "language version 2.0 doesn't support compiling sources " + - "(${sourceFiles.first().virtualFilePath} in particular) " + - "directly to native binaries " + - "(e.g. a ${context.config.produce.name.toLowerCaseAsciiOnly()}).\n" + - "If you are using the command-line compiler (e.g. konanc or kotlinc-native), then " + - "compile the sources to a klib first with '-p library' compiler flag, " + - "and then use '-Xinclude=' flag to compile this to a binary.\n" + - "See more details at https://youtrack.jetbrains.com/issue/KT-56855\n" + - "If you are seeing this error message when compiling with Gradle, " + - "please report this here: https://kotl.in/issue" - ) + require (!input.configuration.getBoolean(CommonConfigurationKeys.USE_FIR)) { + "For K2 compiler, no source files should have been passed here. " + + "K2Native.doExecute() must transform such compilation into two-stage compilation" } } else { // TODO: we shouldn't be here in this case. diff --git a/kotlin-native/backend.native/tests/build.gradle b/kotlin-native/backend.native/tests/build.gradle index 4215b6abf50..207e3c4a498 100644 --- a/kotlin-native/backend.native/tests/build.gradle +++ b/kotlin-native/backend.native/tests/build.gradle @@ -3999,14 +3999,12 @@ standaloneTest("testing_stacktrace") { // Just check that the driver is able to produce runnable binaries. tasks.register("driver0", KonanDriverTest) { disabled = isAggressiveGC // No need to test with different GC schedulers - || isK2(project) // KT-56855 useGoldenData = true source = "runtime/basic/driver0.kt" } tasks.register("driver_opt", KonanDriverTest) { disabled = (cacheTesting != null) // Cache is not compatible with -opt. - || isK2(project) // KT-56855 || isAggressiveGC // No need to test with different GC schedulers useGoldenData = true source = "runtime/basic/driver_opt.kt" @@ -5548,8 +5546,7 @@ interopTest("interop_exceptions_cCallback") { } tasks.register("library_ir_provider_mismatch", KonanDriverTest) { - disabled = isAggressiveGC || // No need to test with different GC schedulers - isK2(project) // KT-56855 + disabled = isAggressiveGC // No need to test with different GC schedulers def dir = buildDir.absolutePath def lib = "$projectDir/link/ir_providers/library/empty.kt" @@ -6150,7 +6147,6 @@ standaloneTest("local_ea_arraysfieldwrite") { tasks.register("override_konan_properties0", KonanDriverTest) { disabled = isAggressiveGC // No need to test with different GC schedulers - || isK2(project) // KT-56855 def overrides = PlatformInfo.isWindows() ? '-Xoverride-konan-properties="llvmInlineThreshold=76"' : '-Xoverride-konan-properties=llvmInlineThreshold=76' @@ -6174,7 +6170,6 @@ tasks.register("llvm_variant_dev", KonanDriverTest) { // so it is hard to reliably detect LLVM variant for these targets. disabled = isAppleTarget(project) || isAggressiveGC // No need to test with different GC schedulers - || isK2(project) // KT-56855 flags = [ "-Xllvm-variant=dev", "-Xverbose-phases=ObjectFiles"