From b7b85177a75df3bff69cf6e503b83b516e708ed1 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Thu, 2 Feb 2023 16:32:48 +0200 Subject: [PATCH] [K/N] Refactor module splitting for backend We want to extract tempFiles from NativeGenerationState. To do so, we need to move creation of NativeGenerationState from `splitIntoFragments` function and untie CacheLlvmModuleSpecification from NativeGenerationState --- .../kotlin/backend/konan/CompilerOutput.kt | 6 +- .../konan/LlvmModuleSpecificationImpl.kt | 8 +- .../backend/konan/NativeGenerationState.kt | 8 +- .../konan/driver/phases/TopLevelPhases.kt | 85 ++++++++++++------- 4 files changed, 70 insertions(+), 37 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt index e27859c099f..441544f48bf 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt @@ -44,9 +44,11 @@ internal val NativeGenerationState.shouldDefineCachedBoxes: Boolean get() = producedLlvmModuleContainsStdlib && cacheDeserializationStrategy?.contains(KonanFqNames.internalPackageName, "Boxing.kt") != false +internal val CacheDeserializationStrategy?.containsRuntime: Boolean + get() = this?.contains(KonanFqNames.internalPackageName, "Runtime.kt") != false + internal val NativeGenerationState.shouldLinkRuntimeNativeLibraries: Boolean - get() = producedLlvmModuleContainsStdlib && - cacheDeserializationStrategy?.contains(KonanFqNames.internalPackageName, "Runtime.kt") != false + get() = producedLlvmModuleContainsStdlib && cacheDeserializationStrategy.containsRuntime val CompilerOutputKind.isCache: Boolean get() = this == CompilerOutputKind.STATIC_CACHE || this == CompilerOutputKind.DYNAMIC_CACHE diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt index 319e6f0df99..3f51e8e594a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt @@ -43,19 +43,19 @@ internal class DefaultLlvmModuleSpecification(cachedLibraries: CachedLibraries) } internal class CacheLlvmModuleSpecification( - private val generationState: NativeGenerationState, cachedLibraries: CachedLibraries, - private val libraryToCache: PartialCacheInfo + private val libraryToCache: PartialCacheInfo, + private val containsStdlib: Boolean, ) : LlvmModuleSpecificationBase(cachedLibraries) { override val isFinal = false override fun containsLibrary(library: KotlinLibrary): Boolean = library == libraryToCache.klib override fun containsDeclaration(declaration: IrDeclaration): Boolean { - if (generationState.shouldDefineFunctionClasses && declaration.getPackageFragment().isFunctionInterfaceFile) + if (containsStdlib && libraryToCache.strategy.containsKFunctionImpl && declaration.getPackageFragment().isFunctionInterfaceFile) return true if (!super.containsDeclaration(declaration)) return false - return (generationState.cacheDeserializationStrategy as? CacheDeserializationStrategy.SingleFile) + return (libraryToCache.strategy as? CacheDeserializationStrategy.SingleFile) ?.filePath.let { it == null || it == declaration.fileOrNull?.path } } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt index 2e675dfe2fc..6e5ff4fed6f 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt @@ -9,6 +9,7 @@ import llvm.* import org.jetbrains.kotlin.backend.konan.driver.BasicPhaseContext import org.jetbrains.kotlin.backend.konan.driver.utilities.BackendContextHolder import org.jetbrains.kotlin.backend.konan.driver.utilities.LlvmIrHolder +import org.jetbrains.kotlin.backend.konan.ir.konanLibrary import org.jetbrains.kotlin.backend.konan.llvm.* import org.jetbrains.kotlin.backend.konan.llvm.coverage.CoverageManager import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport @@ -79,8 +80,11 @@ internal class NativeGenerationState( val llvmModuleSpecification by lazy { if (config.produce.isCache) - CacheLlvmModuleSpecification(this, config.cachedLibraries, - PartialCacheInfo(config.libraryToCache!!.klib, cacheDeserializationStrategy!!)) + CacheLlvmModuleSpecification( + config.cachedLibraries, + PartialCacheInfo(config.libraryToCache!!.klib, cacheDeserializationStrategy!!), + containsStdlib = config.libraryToCache!!.klib == context.stdlibModule.konanLibrary + ) else DefaultLlvmModuleSpecification(config.cachedLibraries) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt index 9f02c5765c9..4d84f0a6432 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.backend.konan.driver.phases import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.driver.PhaseContext import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine +import org.jetbrains.kotlin.backend.konan.ir.konanLibrary import org.jetbrains.kotlin.backend.konan.llvm.getName import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.ir.declarations.IrClass @@ -55,10 +56,13 @@ internal fun PhaseEngine.runBackend(backendContext: Contex useContext(backendContext) { backendEngine -> backendEngine.runPhase(functionsWithoutBoundCheck) val fragments = backendEngine.splitIntoFragments(irModule) - fragments.forEach { (generationState, fragment) -> + fragments.forEach { fragment -> + val generationState = NativeGenerationState(context.config, backendContext, + fragment.cacheDeserializationStrategy + ) backendEngine.useContext(generationState) { generationStateEngine -> // TODO: Make this work if we first compile all the fragments and only after that run the link phases. - val it = generationStateEngine.compileModule(fragment) + val it = generationStateEngine.compileModule(fragment.irModule) // Split here compileAndLink(it, it.outputFiles.mainFileName, it.outputFiles, it.temporaryFiles, isCoverageEnabled = false) } @@ -74,41 +78,64 @@ private fun isReferencedByNativeRuntime(declarations: List): Bool it is IrClass && isReferencedByNativeRuntime(it.declarations) } +private data class BackendJobFragment( + val irModule: IrModuleFragment, + val cacheDeserializationStrategy: CacheDeserializationStrategy?, + val dependenciesTracker: DependenciesTracker, + val llvmModuleSpecification: LlvmModuleSpecification, +) + private fun PhaseEngine.splitIntoFragments( input: IrModuleFragment, -): Sequence> = if (context.config.producePerFileCache) { - val module = input - val files = module.files.toList() - val stdlibIsBeingCached = module.descriptor == context.stdlibModule - val functionInterfaceFiles = files.takeIf { stdlibIsBeingCached } - ?.filter { it.isFunctionInterfaceFile }.orEmpty() - val filesReferencedByNativeRuntime = files.takeIf { stdlibIsBeingCached } - ?.filter { isReferencedByNativeRuntime(it.declarations) }.orEmpty() +): Sequence { + val config = context.config + return if (context.config.producePerFileCache) { + val files = input.files.toList() + val containsStdlib = config.libraryToCache!!.klib == context.stdlibModule.konanLibrary - files.asSequence().filter { !it.isFunctionInterfaceFile }.map { file -> - val generationState = NativeGenerationState( - context.config, - context, - CacheDeserializationStrategy.SingleFile(file.path, file.fqName.asString()) - ) - val fragment = IrModuleFragmentImpl(input.descriptor, input.irBuiltins, listOf(file)) - if (generationState.shouldDefineFunctionClasses) - fragment.files += functionInterfaceFiles + files.asSequence().filter { !it.isFunctionInterfaceFile }.map { file -> + val cacheDeserializationStrategy = CacheDeserializationStrategy.SingleFile(file.path, file.fqName.asString()) + val llvmModuleSpecification = CacheLlvmModuleSpecification( + config.cachedLibraries, + PartialCacheInfo(config.libraryToCache!!.klib, cacheDeserializationStrategy), + containsStdlib = containsStdlib + ) + val dependenciesTracker = DependenciesTrackerImpl(llvmModuleSpecification, context.config, context) + val fragment = IrModuleFragmentImpl(input.descriptor, input.irBuiltins, listOf(file)) + if (containsStdlib && cacheDeserializationStrategy.containsKFunctionImpl) + fragment.files += files.filter { it.isFunctionInterfaceFile } - if (generationState.shouldLinkRuntimeNativeLibraries) { - filesReferencedByNativeRuntime.forEach { - generationState.dependenciesTracker.add(it) + if (containsStdlib && cacheDeserializationStrategy.containsRuntime) { + files.filter { isReferencedByNativeRuntime(it.declarations) } + .forEach { dependenciesTracker.add(it) } } - } - fragment.files.filterIsInstance().forEach { - it.module = fragment + fragment.files.filterIsInstance().forEach { + it.module = fragment + } + BackendJobFragment( + fragment, + cacheDeserializationStrategy, + dependenciesTracker, + llvmModuleSpecification, + ) } - generationState to fragment + } else { + val llvmModuleSpecification = if (config.produce.isCache) { + val containsStdlib = config.libraryToCache!!.klib == context.stdlibModule.konanLibrary + CacheLlvmModuleSpecification(config.cachedLibraries, context.config.libraryToCache!!, containsStdlib = containsStdlib) + } else { + DefaultLlvmModuleSpecification(config.cachedLibraries) + } + sequenceOf( + BackendJobFragment( + input, + context.config.libraryToCache?.strategy, + DependenciesTrackerImpl(llvmModuleSpecification, context.config, context), + llvmModuleSpecification + ) + ) } -} else { - val nativeGenerationState = NativeGenerationState(context.config, context, context.config.libraryToCache?.strategy) - sequenceOf(nativeGenerationState to input) } internal data class ModuleCompilationOutput(