[K/N] Refactored 2-stage splitting

kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -13,7 +13,6 @@ import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.cli.common.*
 import org.jetbrains.kotlin.cli.common.arguments.K2NativeCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments
-import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
@@ -25,14 +24,11 @@ import org.jetbrains.kotlin.ir.linkage.partial.PartialLinkageConfig
 import org.jetbrains.kotlin.ir.linkage.partial.partialLinkageConfig
 import org.jetbrains.kotlin.ir.linkage.partial.setupPartialLinkageConfig
 import org.jetbrains.kotlin.ir.util.IrMessageLogger
-import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.util.profile
 import org.jetbrains.kotlin.utils.KotlinPaths
-import java.util.*
 
 
 private class K2NativeCompilerPerformanceManager: CommonCompilerPerformanceManager("Kotlin to Native Compiler")
@@ -67,64 +63,6 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
         if (!enoughArguments) {
             messageCollector.report(ERROR, "You have not specified any compilation arguments. No output has been produced.")
         }
-        if (configuration.get(KonanConfigKeys.PRODUCE) != CompilerOutputKind.LIBRARY &&
-                configuration.kotlinSourceRoots.isNotEmpty() &&
-                arguments.compileFromBitcode == null) {
-            // K2/Native backend cannot produce binary directly from FIR frontend output, since descriptors, deserialized from KLib, are needed
-            // So, such compilation is split to two stages:
-            // - source files are compiled to intermediate KLib by FIR frontend
-            // - intermediate Klib is compiled to binary by K2/Native backend
-            // In this implementation, 'arguments' is not changed accordingly to changes in `firstStageConfiguration` and `configuration`,
-            // since values of fields `produce`, `output`, `freeArgs`, `includes` does not seem to matter downstream in prepareEnvironment()
-
-            if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR) &&
-                    configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)
-                            ?.getFeatureSupport(LanguageFeature.MultiPlatformProjects) == LanguageFeature.State.ENABLED)
-                messageCollector.report(ERROR,
-                        """
-                            Producing a multiplatform library directly from sources is not allowed since language version 2.0.
-                        
-                            If you use the command-line compiler, then first compile the sources to a KLIB with
-                            the `-p library` compiler flag. Then, use '-Xinclude=<klib>' to pass the KLIB to
-                            the compiler to produce the required type of binary artifact.
-                        """.trimIndent())
-
-            val firstStageConfiguration = configuration.copy()
-            // For the first stage, use "-p library" produce mode
-            firstStageConfiguration.put(KonanConfigKeys.PRODUCE, CompilerOutputKind.LIBRARY)
-            // For the first stage, construct a temporary file name for an intermediate KLib
-            val intermediateKLib = File(System.getProperty("java.io.tmpdir"), "${UUID.randomUUID()}.klib").also {
-                require(!it.exists) { "Collision writing intermediate KLib $it"}
-                it.deleteOnExit()
-            }
-            firstStageConfiguration.put(KonanConfigKeys.OUTPUT, intermediateKLib.absolutePath)
-            // Empty all cache-related keys.
-            firstStageConfiguration.put(KonanConfigKeys.CACHE_DIRECTORIES, emptyList())
-            firstStageConfiguration.put(KonanConfigKeys.AUTO_CACHEABLE_FROM, emptyList())
-            firstStageConfiguration.put(KonanConfigKeys.CACHED_LIBRARIES, emptyMap())
-            firstStageConfiguration.put(KonanConfigKeys.AUTO_CACHE_DIR, "")
-            firstStageConfiguration.put(KonanConfigKeys.INCREMENTAL_CACHE_DIR, "")
-            firstStageConfiguration.setupPartialLinkageConfig(PartialLinkageConfig.DEFAULT) // Disable PL for KLIB compilation.
-
-            val firstStageExitCode = executeStage(firstStageConfiguration, arguments, rootDisposable)
-            if (firstStageExitCode != ExitCode.OK)
-                return firstStageExitCode
-
-            // For the second stage, remove already compiled source files from the configuration
-            configuration.put(CLIConfigurationKeys.CONTENT_ROOTS, listOf())
-            // For the second stage, provide just compiled intermediate KLib as "-Xinclude=" param
-            require(intermediateKLib.exists) { "Intermediate KLib $intermediateKLib must have been created by successful first compilation stage" }
-            configuration.put(KonanConfigKeys.INCLUDED_LIBRARIES, listOf(intermediateKLib.absolutePath))
-            // Now, `configuration` param is prepared for the second stage of compilation, and `arguments` param does not need changes, as noted above.
-        }
-        return executeStage(configuration, arguments, rootDisposable)
-    }
-
-    private fun executeStage(
-            configuration: CompilerConfiguration,
-            arguments: K2NativeCompilerArguments,
-            rootDisposable: Disposable
-    ): ExitCode {
         val environment = prepareEnvironment(arguments, configuration, rootDisposable)
 
         try {
@@ -179,20 +117,34 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
             environment: KotlinCoreEnvironment,
             rootDisposable: Disposable
     ) {
-        val konanDriver = KonanDriver(environment.project, environment, configuration) { args, setupConfiguration ->
+        val konanDriver = KonanDriver(environment.project, environment, configuration, object : CompilationSpawner {
-            val spawnedArguments = K2NativeCompilerArguments()
+            override fun spawn(configuration: CompilerConfiguration) {
-            parseCommandLineArguments(args, spawnedArguments)
+                val spawnedArguments = K2NativeCompilerArguments()
-            val spawnedConfiguration = CompilerConfiguration()
+                parseCommandLineArguments(emptyList(), spawnedArguments)
+                val spawnedEnvironment = KotlinCoreEnvironment.createForProduction(
+                        rootDisposable, configuration, EnvironmentConfigFiles.NATIVE_CONFIG_FILES)
 
-            spawnedConfiguration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY))
+                runKonanDriver(configuration, spawnedEnvironment, rootDisposable)
-            spawnedConfiguration.put(IrMessageLogger.IR_MESSAGE_LOGGER, configuration.getNotNull(IrMessageLogger.IR_MESSAGE_LOGGER))
+            }
-            spawnedConfiguration.setupCommonArguments(spawnedArguments, this::createMetadataVersion)
+
-            spawnedConfiguration.setupFromArguments(spawnedArguments)
+            override fun spawn(arguments: List<String>, setupConfiguration: CompilerConfiguration.() -> Unit) {
-            spawnedConfiguration.setupPartialLinkageConfig(configuration.partialLinkageConfig)
+                val spawnedArguments = K2NativeCompilerArguments()
-            spawnedConfiguration.setupConfiguration()
+                parseCommandLineArguments(arguments, spawnedArguments)
-            val spawnedEnvironment = prepareEnvironment(spawnedArguments, spawnedConfiguration, rootDisposable)
+                val spawnedConfiguration = CompilerConfiguration()
-            runKonanDriver(spawnedConfiguration, spawnedEnvironment, rootDisposable)
+
-        }
+                spawnedConfiguration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY))
+                spawnedConfiguration.put(IrMessageLogger.IR_MESSAGE_LOGGER, configuration.getNotNull(IrMessageLogger.IR_MESSAGE_LOGGER))
+                spawnedConfiguration.setupCommonArguments(spawnedArguments, this@K2Native::createMetadataVersion)
+                spawnedConfiguration.setupFromArguments(spawnedArguments)
+                spawnedConfiguration.setupPartialLinkageConfig(configuration.partialLinkageConfig)
+                configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)?.let {
+                    spawnedConfiguration.put(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS, it)
+                }
+                spawnedConfiguration.setupConfiguration()
+                val spawnedEnvironment = prepareEnvironment(spawnedArguments, spawnedConfiguration, rootDisposable)
+                runKonanDriver(spawnedConfiguration, spawnedEnvironment, rootDisposable)
+            }
+        })
         konanDriver.run()
     }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt

@@ -38,7 +38,7 @@ internal fun KotlinLibrary.getAllTransitiveDependencies(allLibraries: Map<String
 // TODO: deleteRecursively might throw an exception!
 class CacheBuilder(
         val konanConfig: KonanConfig,
-        val spawnCompilation: (List<String>, CompilerConfiguration.() -> Unit) -> Unit
+        val compilationSpawner: CompilationSpawner
 ) {
     private val configuration = konanConfig.configuration
     private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!!.map { File(it) }
@@ -249,7 +249,7 @@ class CacheBuilder(
         try {
             // TODO: Run monolithic cache builds in parallel.
             libraryCacheDirectory.mkdirs()
-            spawnCompilation(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) {
+            compilationSpawner.spawn(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) {
                 val libraryPath = library.libraryFile.absolutePath
                 val libraries = dependencies.filter { !it.isDefault }.map { it.libraryFile.absolutePath }
                 val cachedLibraries = dependencies.zip(dependencyCaches).associate { it.first.libraryFile.absolutePath to it.second }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt

@@ -9,15 +9,20 @@ import com.intellij.openapi.project.Project
 import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
 import org.jetbrains.kotlin.backend.konan.driver.DynamicCompilerDriver
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.impl.createKonanLibrary
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.uniqueName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
+import java.util.*
 
 // Extracted from KonanTarget class to avoid problems with kotlin-native-shared.
 private val deprecatedTargets = setOf(
@@ -35,13 +40,34 @@ private val softDeprecatedTargets = setOf(
 
 private const val DEPRECATION_LINK = "https://kotl.in/native-targets-tiers"
 
+interface CompilationSpawner {
+    fun spawn(configuration: CompilerConfiguration)
+    fun spawn(arguments: List<String>, setupConfiguration: CompilerConfiguration.() -> Unit)
+}
+
 class KonanDriver(
         val project: Project,
         val environment: KotlinCoreEnvironment,
         val configuration: CompilerConfiguration,
-        val spawnCompilation: (List<String>, CompilerConfiguration.() -> Unit) -> Unit
+        val compilationSpawner: CompilationSpawner
 ) {
     fun run() {
+        val outputKind = configuration[KonanConfigKeys.PRODUCE]
+        val isCompilingFromBitcode = configuration[KonanConfigKeys.COMPILE_FROM_BITCODE] != null
+        val hasSourceRoots = configuration.kotlinSourceRoots.isNotEmpty()
+
+        if (isCompilingFromBitcode && hasSourceRoots) {
+            configuration.report(
+                    CompilerMessageSeverity.WARNING,
+                    "Source files will be ignored by the compiler when compiling from bitcode"
+            )
+        }
+
+        if (outputKind != CompilerOutputKind.LIBRARY && hasSourceRoots && !isCompilingFromBitcode) {
+            splitOntoTwoStages()
+            return
+        }
+
         val fileNames = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { libPath ->
             val filesToCache = configuration.get(KonanConfigKeys.FILES_TO_CACHE)
             when {
@@ -80,7 +106,7 @@ class KonanDriver(
 
         ensureModuleName(konanConfig)
 
-        val cacheBuilder = CacheBuilder(konanConfig, spawnCompilation)
+        val cacheBuilder = CacheBuilder(konanConfig, compilationSpawner)
         if (cacheBuilder.needToBuild()) {
             cacheBuilder.build()
             konanConfig = KonanConfig(project, configuration) // TODO: Just set freshly built caches.
@@ -102,4 +128,48 @@ class KonanDriver(
             }
         }
     }
+
+    private fun splitOntoTwoStages() {
+        // K2/Native backend cannot produce binary directly from FIR frontend output, since descriptors, deserialized from KLib, are needed
+        // So, such compilation is split to two stages:
+        // - source files are compiled to intermediate KLib by FIR frontend
+        // - intermediate Klib is compiled to binary by K2/Native backend
+
+        if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR) &&
+                configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)
+                        ?.getFeatureSupport(LanguageFeature.MultiPlatformProjects) == LanguageFeature.State.ENABLED)
+            configuration.report(CompilerMessageSeverity.ERROR,
+                    """
+                            Producing a multiplatform library directly from sources is not allowed since language version 2.0.
+                        
+                            If you use the command-line compiler, then first compile the sources to a KLIB with
+                            the `-p library` compiler flag. Then, use '-Xinclude=<klib>' to pass the KLIB to
+                            the compiler to produce the required type of binary artifact.
+                        """.trimIndent())
+
+        // For the first stage, construct a temporary file name for an intermediate KLib.
+        val intermediateKLib = File(System.getProperty("java.io.tmpdir"), "${UUID.randomUUID()}.klib").also {
+            require(!it.exists) { "Collision writing intermediate KLib $it" }
+            it.deleteOnExit()
+        }
+        compilationSpawner.spawn(emptyList()) {
+            // For the first stage, use "-p library" produce mode.
+            put(KonanConfigKeys.PRODUCE, CompilerOutputKind.LIBRARY)
+            configuration.get(KonanConfigKeys.TARGET)?.let { put(KonanConfigKeys.TARGET, it) }
+            put(KonanConfigKeys.OUTPUT, intermediateKLib.absolutePath)
+            put(CLIConfigurationKeys.CONTENT_ROOTS, configuration.getNotNull(CLIConfigurationKeys.CONTENT_ROOTS))
+            put(KonanConfigKeys.LIBRARY_FILES, configuration.getNotNull(KonanConfigKeys.LIBRARY_FILES))
+            put(KonanConfigKeys.REPOSITORIES, configuration.getNotNull(KonanConfigKeys.REPOSITORIES))
+            configuration.get(KonanConfigKeys.FRIEND_MODULES)?.let { put(KonanConfigKeys.FRIEND_MODULES, it) }
+            configuration.get(KonanConfigKeys.REFINES_MODULES)?.let { put(KonanConfigKeys.REFINES_MODULES, it) }
+        }
+
+        // For the second stage, remove already compiled source files from the configuration.
+        configuration.put(CLIConfigurationKeys.CONTENT_ROOTS, listOf())
+        // For the second stage, provide just compiled intermediate KLib as "-Xinclude=" param.
+        require(intermediateKLib.exists) { "Intermediate KLib $intermediateKLib must have been created by successful first compilation stage" }
+        configuration.put(KonanConfigKeys.INCLUDED_LIBRARIES,
+                configuration.get(KonanConfigKeys.INCLUDED_LIBRARIES).orEmpty() + listOf(intermediateKLib.absolutePath))
+        compilationSpawner.spawn(configuration) // Need to spawn a new compilation to create fresh environment (without sources).
+    }
 }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Frontend.kt

@@ -57,8 +57,8 @@ internal val FrontendPhase = createSimpleNamedCompilerPhase(
 
         require(context.config.produce == CompilerOutputKind.LIBRARY || sourceFiles.isEmpty()) {
             "Internal error: no source files should have been passed here (${sourceFiles.first().virtualFilePath} in particular)\n" +
-            "to produce binary (e.g. a ${context.config.produce.name.toLowerCaseAsciiOnly()})\n" +
+                    "to produce binary (e.g. a ${context.config.produce.name.toLowerCaseAsciiOnly()})\n" +
-            "K2Native.doExecute() must transform such compilation into two-stage compilation. Please report this here: https://kotl.in/issue"
+                    "KonanDriver.kt::splitOntoTwoStages() must transform such compilation into two-stage compilation. Please report this here: https://kotl.in/issue"
         }
 
         // Build AST and binding info.