From 030e3b306fed7f667c88fbcab78132a2e719e509 Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Mon, 11 Apr 2022 15:15:34 +0500 Subject: [PATCH] [K/N][build] Added -Xmake-per-file-cache compiler option --- .../arguments/K2NativeCompilerArguments.kt | 3 + .../backend/common/phaser/PhaseConfig.kt | 44 ++++++++++++- .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 22 +------ .../backend/konan/KonanConfigurationKeys.kt | 2 + .../kotlin/backend/konan/KonanDriver.kt | 66 ++++++++++++++++++- 5 files changed, 112 insertions(+), 25 deletions(-) diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt index 78be8c451c6..6c618277dcc 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt @@ -221,6 +221,9 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { ) var fileToCache: String? = null + @Argument(value = "-Xmake-per-file-cache", description = "Force compiler to produce per-file cache") + var makePerFileCache: Boolean = false + @Argument(value = "-Xexport-kdoc", description = "Export KDoc in framework header") var exportKDoc: Boolean = false diff --git a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseConfig.kt b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseConfig.kt index 731bf9eb087..71e16d4361e 100644 --- a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseConfig.kt +++ b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseConfig.kt @@ -12,10 +12,33 @@ fun CompilerPhase<*, *, *>.toPhaseMap(): MutableMap = acc } +class PhaseConfigBuilder(private val compoundPhase: CompilerPhase<*, *, *>) { + val enabled = mutableSetOf() + val verbose = mutableSetOf() + val toDumpStateBefore = mutableSetOf() + val toDumpStateAfter = mutableSetOf() + var dumpToDirectory: String? = null + var dumpOnlyFqName: String? = null + val toValidateStateBefore = mutableSetOf() + val toValidateStateAfter = mutableSetOf() + val namesOfElementsExcludedFromDumping = mutableSetOf() + var needProfiling = false + var checkConditions = false + var checkStickyConditions = false + + fun build() = PhaseConfig( + compoundPhase, compoundPhase.toPhaseMap(), enabled, + verbose, toDumpStateBefore, toDumpStateAfter, dumpToDirectory, dumpOnlyFqName, + toValidateStateBefore, toValidateStateAfter, + namesOfElementsExcludedFromDumping, + needProfiling, checkConditions, checkStickyConditions + ) +} + class PhaseConfig( private val compoundPhase: CompilerPhase<*, *, *>, - private val phases: MutableMap = compoundPhase.toPhaseMap(), - enabled: MutableSet = phases.values.toMutableSet(), + private val phases: Map = compoundPhase.toPhaseMap(), + private val initiallyEnabled: Set = phases.values.toSet(), val verbose: Set = emptySet(), val toDumpStateBefore: Set = emptySet(), val toDumpStateAfter: Set = emptySet(), @@ -28,7 +51,22 @@ class PhaseConfig( val checkConditions: Boolean = false, val checkStickyConditions: Boolean = false ) { - private val enabledMut = enabled + fun toBuilder() = PhaseConfigBuilder(compoundPhase).also { + it.enabled.addAll(initiallyEnabled) + it.verbose.addAll(verbose) + it.toDumpStateBefore.addAll(toDumpStateBefore) + it.toDumpStateAfter.addAll(toDumpStateAfter) + it.dumpToDirectory = dumpToDirectory + it.dumpOnlyFqName = dumpOnlyFqName + it.toValidateStateBefore.addAll(toValidateStateBefore) + it.toValidateStateAfter.addAll(toValidateStateAfter) + it.namesOfElementsExcludedFromDumping.addAll(namesOfElementsExcludedFromDumping) + it.needProfiling = needProfiling + it.checkConditions = checkConditions + it.checkStickyConditions = checkStickyConditions + } + + private val enabledMut = initiallyEnabled.toMutableSet() val enabled: Set get() = enabledMut diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 638599feae9..3c65408a947 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -28,7 +28,6 @@ import org.jetbrains.kotlin.konan.CURRENT import org.jetbrains.kotlin.konan.CompilerVersion import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.CompilerOutputKind -import org.jetbrains.kotlin.library.uniqueName import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.util.profile @@ -90,13 +89,7 @@ class K2Native : CLICompiler() { configuration.put(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME, arguments.renderInternalDiagnosticNames) try { - val konanConfig = KonanConfig(project, configuration) - try { - ensureModuleName(konanConfig, environment) - runTopLevelPhases(konanConfig, environment) - } finally { - konanConfig.dispose() - } + KonanDriver(project, environment, configuration).run() } catch (e: Throwable) { if (e is KonanCompilationException || e is CompilationErrorException) return ExitCode.COMPILATION_ERROR @@ -123,18 +116,6 @@ class K2Native : CLICompiler() { return this?.asList() ?: listOf() } - private fun ensureModuleName(config: KonanConfig, environment: KotlinCoreEnvironment) { - if (environment.getSourceFiles().isEmpty()) { - val libraries = config.resolvedLibraries.getFullList() - val moduleName = config.moduleId - if (libraries.any { it.uniqueName == moduleName }) { - val kexeModuleName = "${moduleName}_kexe" - config.configuration.put(KonanConfigKeys.MODULE_NAME, kexeModuleName) - assert(libraries.none { it.uniqueName == kexeModuleName }) - } - } - } - // It is executed before doExecute(). override fun setupPlatformSpecificArgumentsAndServices( configuration: CompilerConfiguration, @@ -308,6 +289,7 @@ class K2Native : CLICompiler() { if (outputKind == CompilerOutputKind.PRELIMINARY_CACHE && fileToCache == null) configuration.report(ERROR, "preliminary_cache only supported for per-file caches") fileToCache?.let { put(FILE_TO_CACHE, it) } + put(MAKE_PER_FILE_CACHE, arguments.makePerFileCache) parseShortModuleName(arguments, configuration, outputKind)?.let { put(SHORT_MODULE_NAME, it) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index 14c0f6ae736..4daa1989120 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -46,6 +46,8 @@ class KonanConfigKeys { = CompilerConfigurationKey.create>("mapping from library paths to cache paths") val FILE_TO_CACHE: CompilerConfigurationKey = CompilerConfigurationKey.create("which file should be compiled to cache") + val MAKE_PER_FILE_CACHE: CompilerConfigurationKey + = CompilerConfigurationKey.create("make per-file cache") val FRAMEWORK_IMPORT_HEADERS: CompilerConfigurationKey> = CompilerConfigurationKey.create>("headers imported to framework header") val FRIEND_MODULES: CompilerConfigurationKey> diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index e838e0a080f..a18713b9f9a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -5,19 +5,81 @@ package org.jetbrains.kotlin.backend.konan +import com.intellij.openapi.project.Project import kotlinx.cinterop.usingJvmCInteropCallbacks import org.jetbrains.kotlin.analyzer.AnalysisResult import org.jetbrains.kotlin.backend.common.phaser.CompilerPhase import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel +import org.jetbrains.kotlin.backend.common.serialization.codedInputStream +import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment -import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.languageVersionSettings +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.library.impl.createKonanLibrary +import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.util.usingNativeMemoryAllocator +import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.utils.addToStdlib.cast -fun runTopLevelPhases(konanConfig: KonanConfig, environment: KotlinCoreEnvironment) { +class KonanDriver(val project: Project, val environment: KotlinCoreEnvironment, val configuration: CompilerConfiguration) { + fun run() { + val fileNames = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { libPath -> + if (configuration.get(KonanConfigKeys.MAKE_PER_FILE_CACHE) != true) + null + else { + val lib = createKonanLibrary(File(libPath), "default", null, true) + (0 until lib.fileCount()).map { fileIndex -> + val proto = IrFile.parseFrom(lib.file(fileIndex).codedInputStream, ExtensionRegistryLite.newInstance()) + proto.fileEntry.name + } + } + } + + if (fileNames == null) { + KonanConfig(project, configuration).runTopLevelPhases() + } else { + fileNames.forEach { buildFileCache(it, CompilerOutputKind.PRELIMINARY_CACHE) } + fileNames.forEach { buildFileCache(it, configuration.get(KonanConfigKeys.PRODUCE)!!) } + } + } + + private fun buildFileCache(fileName: String, cacheKind: CompilerOutputKind) { + val phaseConfig = configuration.get(CLIConfigurationKeys.PHASE_CONFIG)!! + val subConfiguration = configuration.copy() + subConfiguration.put(KonanConfigKeys.PRODUCE, cacheKind) + subConfiguration.put(KonanConfigKeys.FILE_TO_CACHE, fileName) + subConfiguration.put(KonanConfigKeys.MAKE_PER_FILE_CACHE, false) + subConfiguration.put(CLIConfigurationKeys.PHASE_CONFIG, phaseConfig.toBuilder().build()) + KonanConfig(project, subConfiguration).runTopLevelPhases() + } + + private fun KonanConfig.runTopLevelPhases() { + try { + ensureModuleName(this) + runTopLevelPhases(this, environment) + } finally { + dispose() + } + } + + private fun ensureModuleName(config: KonanConfig) { + if (environment.getSourceFiles().isEmpty()) { + val libraries = config.resolvedLibraries.getFullList() + val moduleName = config.moduleId + if (libraries.any { it.uniqueName == moduleName }) { + val kexeModuleName = "${moduleName}_kexe" + config.configuration.put(KonanConfigKeys.MODULE_NAME, kexeModuleName) + assert(libraries.none { it.uniqueName == kexeModuleName }) + } + } + } +} + +private fun runTopLevelPhases(konanConfig: KonanConfig, environment: KotlinCoreEnvironment) { val config = konanConfig.configuration