diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index e4df2fab765..5f995b9a0cc 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -13,7 +13,6 @@ import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.cli.common.* import org.jetbrains.kotlin.cli.common.arguments.K2NativeCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments -import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.* import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.cli.common.messages.MessageRenderer @@ -25,14 +24,11 @@ import org.jetbrains.kotlin.ir.linkage.partial.PartialLinkageConfig import org.jetbrains.kotlin.ir.linkage.partial.partialLinkageConfig import org.jetbrains.kotlin.ir.linkage.partial.setupPartialLinkageConfig import org.jetbrains.kotlin.ir.util.IrMessageLogger -import org.jetbrains.kotlin.konan.file.File -import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.util.profile import org.jetbrains.kotlin.utils.KotlinPaths -import java.util.* private class K2NativeCompilerPerformanceManager: CommonCompilerPerformanceManager("Kotlin to Native Compiler") @@ -67,64 +63,6 @@ class K2Native : CLICompiler() { if (!enoughArguments) { messageCollector.report(ERROR, "You have not specified any compilation arguments. No output has been produced.") } - if (configuration.get(KonanConfigKeys.PRODUCE) != CompilerOutputKind.LIBRARY && - configuration.kotlinSourceRoots.isNotEmpty() && - arguments.compileFromBitcode == null) { - // K2/Native backend cannot produce binary directly from FIR frontend output, since descriptors, deserialized from KLib, are needed - // So, such compilation is split to two stages: - // - source files are compiled to intermediate KLib by FIR frontend - // - intermediate Klib is compiled to binary by K2/Native backend - // In this implementation, 'arguments' is not changed accordingly to changes in `firstStageConfiguration` and `configuration`, - // since values of fields `produce`, `output`, `freeArgs`, `includes` does not seem to matter downstream in prepareEnvironment() - - if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR) && - configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS) - ?.getFeatureSupport(LanguageFeature.MultiPlatformProjects) == LanguageFeature.State.ENABLED) - messageCollector.report(ERROR, - """ - Producing a multiplatform library directly from sources is not allowed since language version 2.0. - - If you use the command-line compiler, then first compile the sources to a KLIB with - the `-p library` compiler flag. Then, use '-Xinclude=' to pass the KLIB to - the compiler to produce the required type of binary artifact. - """.trimIndent()) - - val firstStageConfiguration = configuration.copy() - // For the first stage, use "-p library" produce mode - firstStageConfiguration.put(KonanConfigKeys.PRODUCE, CompilerOutputKind.LIBRARY) - // For the first stage, construct a temporary file name for an intermediate KLib - val intermediateKLib = File(System.getProperty("java.io.tmpdir"), "${UUID.randomUUID()}.klib").also { - require(!it.exists) { "Collision writing intermediate KLib $it"} - it.deleteOnExit() - } - firstStageConfiguration.put(KonanConfigKeys.OUTPUT, intermediateKLib.absolutePath) - // Empty all cache-related keys. - firstStageConfiguration.put(KonanConfigKeys.CACHE_DIRECTORIES, emptyList()) - firstStageConfiguration.put(KonanConfigKeys.AUTO_CACHEABLE_FROM, emptyList()) - firstStageConfiguration.put(KonanConfigKeys.CACHED_LIBRARIES, emptyMap()) - firstStageConfiguration.put(KonanConfigKeys.AUTO_CACHE_DIR, "") - firstStageConfiguration.put(KonanConfigKeys.INCREMENTAL_CACHE_DIR, "") - firstStageConfiguration.setupPartialLinkageConfig(PartialLinkageConfig.DEFAULT) // Disable PL for KLIB compilation. - - val firstStageExitCode = executeStage(firstStageConfiguration, arguments, rootDisposable) - if (firstStageExitCode != ExitCode.OK) - return firstStageExitCode - - // For the second stage, remove already compiled source files from the configuration - configuration.put(CLIConfigurationKeys.CONTENT_ROOTS, listOf()) - // For the second stage, provide just compiled intermediate KLib as "-Xinclude=" param - require(intermediateKLib.exists) { "Intermediate KLib $intermediateKLib must have been created by successful first compilation stage" } - configuration.put(KonanConfigKeys.INCLUDED_LIBRARIES, listOf(intermediateKLib.absolutePath)) - // Now, `configuration` param is prepared for the second stage of compilation, and `arguments` param does not need changes, as noted above. - } - return executeStage(configuration, arguments, rootDisposable) - } - - private fun executeStage( - configuration: CompilerConfiguration, - arguments: K2NativeCompilerArguments, - rootDisposable: Disposable - ): ExitCode { val environment = prepareEnvironment(arguments, configuration, rootDisposable) try { @@ -179,20 +117,34 @@ class K2Native : CLICompiler() { environment: KotlinCoreEnvironment, rootDisposable: Disposable ) { - val konanDriver = KonanDriver(environment.project, environment, configuration) { args, setupConfiguration -> - val spawnedArguments = K2NativeCompilerArguments() - parseCommandLineArguments(args, spawnedArguments) - val spawnedConfiguration = CompilerConfiguration() + val konanDriver = KonanDriver(environment.project, environment, configuration, object : CompilationSpawner { + override fun spawn(configuration: CompilerConfiguration) { + val spawnedArguments = K2NativeCompilerArguments() + parseCommandLineArguments(emptyList(), spawnedArguments) + val spawnedEnvironment = KotlinCoreEnvironment.createForProduction( + rootDisposable, configuration, EnvironmentConfigFiles.NATIVE_CONFIG_FILES) - spawnedConfiguration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)) - spawnedConfiguration.put(IrMessageLogger.IR_MESSAGE_LOGGER, configuration.getNotNull(IrMessageLogger.IR_MESSAGE_LOGGER)) - spawnedConfiguration.setupCommonArguments(spawnedArguments, this::createMetadataVersion) - spawnedConfiguration.setupFromArguments(spawnedArguments) - spawnedConfiguration.setupPartialLinkageConfig(configuration.partialLinkageConfig) - spawnedConfiguration.setupConfiguration() - val spawnedEnvironment = prepareEnvironment(spawnedArguments, spawnedConfiguration, rootDisposable) - runKonanDriver(spawnedConfiguration, spawnedEnvironment, rootDisposable) - } + runKonanDriver(configuration, spawnedEnvironment, rootDisposable) + } + + override fun spawn(arguments: List, setupConfiguration: CompilerConfiguration.() -> Unit) { + val spawnedArguments = K2NativeCompilerArguments() + parseCommandLineArguments(arguments, spawnedArguments) + val spawnedConfiguration = CompilerConfiguration() + + spawnedConfiguration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)) + spawnedConfiguration.put(IrMessageLogger.IR_MESSAGE_LOGGER, configuration.getNotNull(IrMessageLogger.IR_MESSAGE_LOGGER)) + spawnedConfiguration.setupCommonArguments(spawnedArguments, this@K2Native::createMetadataVersion) + spawnedConfiguration.setupFromArguments(spawnedArguments) + spawnedConfiguration.setupPartialLinkageConfig(configuration.partialLinkageConfig) + configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)?.let { + spawnedConfiguration.put(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS, it) + } + spawnedConfiguration.setupConfiguration() + val spawnedEnvironment = prepareEnvironment(spawnedArguments, spawnedConfiguration, rootDisposable) + runKonanDriver(spawnedConfiguration, spawnedEnvironment, rootDisposable) + } + }) konanDriver.run() } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt index 86233437307..f7bfdccb681 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt @@ -38,7 +38,7 @@ internal fun KotlinLibrary.getAllTransitiveDependencies(allLibraries: Map, CompilerConfiguration.() -> Unit) -> Unit + val compilationSpawner: CompilationSpawner ) { private val configuration = konanConfig.configuration private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!!.map { File(it) } @@ -249,7 +249,7 @@ class CacheBuilder( try { // TODO: Run monolithic cache builds in parallel. libraryCacheDirectory.mkdirs() - spawnCompilation(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) { + compilationSpawner.spawn(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) { val libraryPath = library.libraryFile.absolutePath val libraries = dependencies.filter { !it.isDefault }.map { it.libraryFile.absolutePath } val cachedLibraries = dependencies.zip(dependencyCaches).associate { it.first.libraryFile.absolutePath to it.second } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index 5fed239c69f..8048d8e784a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -9,15 +9,20 @@ import com.intellij.openapi.project.Project import org.jetbrains.kotlin.backend.common.serialization.codedInputStream import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile import org.jetbrains.kotlin.backend.konan.driver.DynamicCompilerDriver +import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.LanguageFeature import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.library.impl.createKonanLibrary import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.uniqueName import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite +import java.util.* // Extracted from KonanTarget class to avoid problems with kotlin-native-shared. private val deprecatedTargets = setOf( @@ -35,13 +40,34 @@ private val softDeprecatedTargets = setOf( private const val DEPRECATION_LINK = "https://kotl.in/native-targets-tiers" +interface CompilationSpawner { + fun spawn(configuration: CompilerConfiguration) + fun spawn(arguments: List, setupConfiguration: CompilerConfiguration.() -> Unit) +} + class KonanDriver( val project: Project, val environment: KotlinCoreEnvironment, val configuration: CompilerConfiguration, - val spawnCompilation: (List, CompilerConfiguration.() -> Unit) -> Unit + val compilationSpawner: CompilationSpawner ) { fun run() { + val outputKind = configuration[KonanConfigKeys.PRODUCE] + val isCompilingFromBitcode = configuration[KonanConfigKeys.COMPILE_FROM_BITCODE] != null + val hasSourceRoots = configuration.kotlinSourceRoots.isNotEmpty() + + if (isCompilingFromBitcode && hasSourceRoots) { + configuration.report( + CompilerMessageSeverity.WARNING, + "Source files will be ignored by the compiler when compiling from bitcode" + ) + } + + if (outputKind != CompilerOutputKind.LIBRARY && hasSourceRoots && !isCompilingFromBitcode) { + splitOntoTwoStages() + return + } + val fileNames = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { libPath -> val filesToCache = configuration.get(KonanConfigKeys.FILES_TO_CACHE) when { @@ -80,7 +106,7 @@ class KonanDriver( ensureModuleName(konanConfig) - val cacheBuilder = CacheBuilder(konanConfig, spawnCompilation) + val cacheBuilder = CacheBuilder(konanConfig, compilationSpawner) if (cacheBuilder.needToBuild()) { cacheBuilder.build() konanConfig = KonanConfig(project, configuration) // TODO: Just set freshly built caches. @@ -102,4 +128,48 @@ class KonanDriver( } } } + + private fun splitOntoTwoStages() { + // K2/Native backend cannot produce binary directly from FIR frontend output, since descriptors, deserialized from KLib, are needed + // So, such compilation is split to two stages: + // - source files are compiled to intermediate KLib by FIR frontend + // - intermediate Klib is compiled to binary by K2/Native backend + + if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR) && + configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS) + ?.getFeatureSupport(LanguageFeature.MultiPlatformProjects) == LanguageFeature.State.ENABLED) + configuration.report(CompilerMessageSeverity.ERROR, + """ + Producing a multiplatform library directly from sources is not allowed since language version 2.0. + + If you use the command-line compiler, then first compile the sources to a KLIB with + the `-p library` compiler flag. Then, use '-Xinclude=' to pass the KLIB to + the compiler to produce the required type of binary artifact. + """.trimIndent()) + + // For the first stage, construct a temporary file name for an intermediate KLib. + val intermediateKLib = File(System.getProperty("java.io.tmpdir"), "${UUID.randomUUID()}.klib").also { + require(!it.exists) { "Collision writing intermediate KLib $it" } + it.deleteOnExit() + } + compilationSpawner.spawn(emptyList()) { + // For the first stage, use "-p library" produce mode. + put(KonanConfigKeys.PRODUCE, CompilerOutputKind.LIBRARY) + configuration.get(KonanConfigKeys.TARGET)?.let { put(KonanConfigKeys.TARGET, it) } + put(KonanConfigKeys.OUTPUT, intermediateKLib.absolutePath) + put(CLIConfigurationKeys.CONTENT_ROOTS, configuration.getNotNull(CLIConfigurationKeys.CONTENT_ROOTS)) + put(KonanConfigKeys.LIBRARY_FILES, configuration.getNotNull(KonanConfigKeys.LIBRARY_FILES)) + put(KonanConfigKeys.REPOSITORIES, configuration.getNotNull(KonanConfigKeys.REPOSITORIES)) + configuration.get(KonanConfigKeys.FRIEND_MODULES)?.let { put(KonanConfigKeys.FRIEND_MODULES, it) } + configuration.get(KonanConfigKeys.REFINES_MODULES)?.let { put(KonanConfigKeys.REFINES_MODULES, it) } + } + + // For the second stage, remove already compiled source files from the configuration. + configuration.put(CLIConfigurationKeys.CONTENT_ROOTS, listOf()) + // For the second stage, provide just compiled intermediate KLib as "-Xinclude=" param. + require(intermediateKLib.exists) { "Intermediate KLib $intermediateKLib must have been created by successful first compilation stage" } + configuration.put(KonanConfigKeys.INCLUDED_LIBRARIES, + configuration.get(KonanConfigKeys.INCLUDED_LIBRARIES).orEmpty() + listOf(intermediateKLib.absolutePath)) + compilationSpawner.spawn(configuration) // Need to spawn a new compilation to create fresh environment (without sources). + } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt index 3d7067c0b1a..6fdae523e78 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt @@ -57,8 +57,8 @@ internal val FrontendPhase = createSimpleNamedCompilerPhase( require(context.config.produce == CompilerOutputKind.LIBRARY || sourceFiles.isEmpty()) { "Internal error: no source files should have been passed here (${sourceFiles.first().virtualFilePath} in particular)\n" + - "to produce binary (e.g. a ${context.config.produce.name.toLowerCaseAsciiOnly()})\n" + - "K2Native.doExecute() must transform such compilation into two-stage compilation. Please report this here: https://kotl.in/issue" + "to produce binary (e.g. a ${context.config.produce.name.toLowerCaseAsciiOnly()})\n" + + "KonanDriver.kt::splitOntoTwoStages() must transform such compilation into two-stage compilation. Please report this here: https://kotl.in/issue" } // Build AST and binding info.