From 05628660bad072128239a9638b80b695b707819e Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Tue, 23 Jan 2024 16:10:19 +0100 Subject: [PATCH] [Test] Implement classic & Fir Native KLIB facades for test infra With these test facades it would be possible to serialize Native test modules to KLIBs. ^KT-65117 --- .../kotlin/test/backend/ir/IrBackendInput.kt | 9 +- .../test/ClassicFrontend2NativeIrConverter.kt | 7 +- .../test/Fir2IrNativeResultsConverter.kt | 4 +- .../konan/test/NativeKlibBackendFacade.kt | 214 ++++++++++++++++++ .../library/impl/KonanLibraryWriterImpl.kt | 2 +- 5 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/test/NativeKlibBackendFacade.kt diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt index ecd23bd2a3a..cb331092966 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt @@ -119,9 +119,11 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput() { get() = state.diagnosticReporter as BaseDiagnosticsCollector } - // Actually, class won't be used as a real input for the Native backend during blackbox testing, since such testing is done via a different engine. - // In irText tests, this class is used only to hold Native-specific FIR2IR output (module fragments) to render and dump IR. - // So, other fields are actually not needed: source files, icData, error flag, serialization lambda, etc... + /** + * Note: For the classic frontend both [firMangler] and [metadataSerializer] are null. + * The latter is because the Native backend uses + * [org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer] which serializes a whole module. + */ class NativeBackendInput( override val irModuleFragment: IrModuleFragment, override val irPluginContext: IrPluginContext, @@ -129,5 +131,6 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput() { override val descriptorMangler: KotlinMangler.DescriptorMangler?, override val irMangler: KotlinMangler.IrMangler, override val firMangler: FirMangler?, + val metadataSerializer: KlibSingleFileMetadataSerializer<*>?, ) : IrBackendInput() } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/ClassicFrontend2NativeIrConverter.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/ClassicFrontend2NativeIrConverter.kt index 4968496e396..269a37c836e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/ClassicFrontend2NativeIrConverter.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/ClassicFrontend2NativeIrConverter.kt @@ -33,9 +33,9 @@ import org.jetbrains.kotlin.test.model.Frontend2BackendConverter import org.jetbrains.kotlin.test.model.FrontendKinds import org.jetbrains.kotlin.test.model.TestModule import org.jetbrains.kotlin.test.services.* +import org.jetbrains.kotlin.utils.addToStdlib.shouldNotBeCalled import kotlin.reflect.KClass - class ClassicFrontend2NativeIrConverter( testServices: TestServices, ) : Frontend2BackendConverter( @@ -103,9 +103,7 @@ class ClassicFrontend2NativeIrConverter( signature: IdSignature, kind: IrDeserializer.TopLevelSymbolKind, moduleName: Name - ): IrSymbol { - error("Should not be called") - } + ): Nothing = shouldNotBeCalled() override fun postProcess(inOrAfterLinkageStep: Boolean) = Unit } @@ -136,6 +134,7 @@ class ClassicFrontend2NativeIrConverter( descriptorMangler = (pluginContext.symbolTable as SymbolTable).signaturer!!.mangler, irMangler = KonanManglerIr, firMangler = null, + metadataSerializer = null ) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/Fir2IrNativeResultsConverter.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/Fir2IrNativeResultsConverter.kt index 5e8e0fde416..8c13861918e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/Fir2IrNativeResultsConverter.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/Fir2IrNativeResultsConverter.kt @@ -48,8 +48,7 @@ class Fir2IrNativeResultsConverter(testServices: TestServices) : AbstractFir2IrN fir2IrResult: Fir2IrActualizedResult, fir2KlibMetadataSerializer: Fir2KlibMetadataSerializer, ): IrBackendInput { - val fir2IrComponents = fir2IrResult.components - val manglers = fir2IrComponents.manglers + val manglers = fir2IrResult.components.manglers return IrBackendInput.NativeBackendInput( fir2IrResult.irModuleFragment, fir2IrResult.pluginContext, @@ -57,6 +56,7 @@ class Fir2IrNativeResultsConverter(testServices: TestServices) : AbstractFir2IrN descriptorMangler = null, irMangler = manglers.irMangler, firMangler = manglers.firMangler, + metadataSerializer = fir2KlibMetadataSerializer ) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/NativeKlibBackendFacade.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/NativeKlibBackendFacade.kt new file mode 100644 index 00000000000..808bbee55e0 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/NativeKlibBackendFacade.kt @@ -0,0 +1,214 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.test + +import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode +import org.jetbrains.kotlin.backend.common.serialization.SerializerOutput +import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer +import org.jetbrains.kotlin.backend.common.serialization.serializeModuleIntoKlib +import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer +import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.KotlinCompilerVersion +import org.jetbrains.kotlin.config.languageVersionSettings +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext +import org.jetbrains.kotlin.konan.library.impl.buildLibrary +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.library.* +import org.jetbrains.kotlin.library.metadata.KlibMetadataFactories +import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion +import org.jetbrains.kotlin.library.metadata.NullFlexibleTypeDeserializer +import org.jetbrains.kotlin.storage.LockBasedStorageManager +import org.jetbrains.kotlin.test.backend.ir.IrBackendFacade +import org.jetbrains.kotlin.test.backend.ir.IrBackendInput +import org.jetbrains.kotlin.test.frontend.classic.ModuleDescriptorProvider +import org.jetbrains.kotlin.test.frontend.classic.moduleDescriptorProvider +import org.jetbrains.kotlin.test.frontend.fir.getAllNativeDependenciesPaths +import org.jetbrains.kotlin.test.frontend.fir.resolveLibraries +import org.jetbrains.kotlin.test.model.* +import org.jetbrains.kotlin.test.services.* +import org.jetbrains.kotlin.test.services.configuration.NativeEnvironmentConfigurator.Companion.getKlibArtifactFile +import org.jetbrains.kotlin.utils.metadataVersion + +abstract class AbstractNativeKlibBackendFacade( + testServices: TestServices +) : IrBackendFacade(testServices, ArtifactKinds.KLib) { + final override fun shouldRunAnalysis(module: TestModule): Boolean { + return module.backendKind == inputKind + } + + final override fun transform(module: TestModule, inputArtifact: IrBackendInput): BinaryArtifacts.KLib { + require(inputArtifact is IrBackendInput.NativeBackendInput) { + "${this::class.java.simpleName} expects IrBackendInput.NativeBackendInput as input" + } + + val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module) + + val dependencyPaths = getAllNativeDependenciesPaths(module, testServices) + val dependencies = resolveLibraries(configuration, dependencyPaths).map { it.library } + + val serializerOutput = serialize(configuration, dependencies, module, inputArtifact) + + val outputArtifact = BinaryArtifacts.KLib(getKlibArtifactFile(testServices, module.name), inputArtifact.diagnosticReporter) + + buildLibrary( + natives = emptyList(), + included = emptyList(), + linkDependencies = serializerOutput.neededLibraries, + serializerOutput.serializedMetadata ?: testServices.assertions.fail { "expected serialized metadata" }, + serializerOutput.serializedIr, + versions = KotlinLibraryVersioning( + abiVersion = KotlinAbiVersion.CURRENT, + libraryVersion = null, + compilerVersion = KotlinCompilerVersion.getVersion(), + metadataVersion = configuration.metadataVersion().toString(), + ), + target = HostManager.host, + output = outputArtifact.outputFile.path, + moduleName = configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME), + nopack = true, + shortName = null, + manifestProperties = null, + dataFlowGraph = null + ) + + updateTestConfiguration(configuration, dependencyPaths, module, outputArtifact) + + return outputArtifact + } + + protected abstract fun serialize( + configuration: CompilerConfiguration, + dependencies: List, + module: TestModule, + inputArtifact: IrBackendInput.NativeBackendInput, + ): SerializerOutput + + protected open fun updateTestConfiguration( + configuration: CompilerConfiguration, + dependencyPaths: List, + module: TestModule, + outputArtifact: BinaryArtifacts.KLib + ) = Unit +} + +/** + * The Native KLIB facade suitable for the classic frontend. + */ +class ClassicNativeKlibBackendFacade(testServices: TestServices) : AbstractNativeKlibBackendFacade(testServices) { + override val additionalServices: List + get() = listOf(service(::LibraryProvider), service(::ModuleDescriptorProvider)) + + override fun serialize( + configuration: CompilerConfiguration, + dependencies: List, + module: TestModule, + inputArtifact: IrBackendInput.NativeBackendInput, + ): SerializerOutput { + testServices.assertions.assertTrue(inputArtifact.firMangler == null) { "unexpected Fir mangler" } + testServices.assertions.assertTrue(inputArtifact.metadataSerializer == null) { "unexpected single-file metadata serializer" } + + val frontendOutput = testServices.dependencyProvider.getArtifact(module, FrontendKinds.ClassicFrontend) + + val serializedMetadata = KlibMetadataMonolithicSerializer( + configuration.languageVersionSettings, + metadataVersion = configuration[CommonConfigurationKeys.METADATA_VERSION] as? KlibMetadataVersion + ?: KlibMetadataVersion.INSTANCE, + frontendOutput.project, + exportKDoc = false, + skipExpects = true, + allowErrorTypes = false, + ).serializeModule(frontendOutput.analysisResult.moduleDescriptor) + + val serializerIr = KonanIrModuleSerializer( + KtDiagnosticReporterWithImplicitIrBasedContext(inputArtifact.diagnosticReporter, configuration.languageVersionSettings), + inputArtifact.irModuleFragment.irBuiltins, + CompatibilityMode.CURRENT, + normalizeAbsolutePaths = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false, + sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES].orEmpty(), + configuration.languageVersionSettings, + shouldCheckSignaturesOnUniqueness = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true + ).serializedIrModule(inputArtifact.irModuleFragment) + + return SerializerOutput( + serializedMetadata, + serializerIr, + dataFlowGraph = null, + neededLibraries = dependencies + ) + } + + override fun updateTestConfiguration( + configuration: CompilerConfiguration, + dependencyPaths: List, + module: TestModule, + outputArtifact: BinaryArtifacts.KLib + ) { + val nativeFactories = KlibMetadataFactories(::KonanBuiltIns, NullFlexibleTypeDeserializer) + + val library = resolveLibraries( + configuration, dependencyPaths + outputArtifact.outputFile.path, + ).last().library + + val moduleDescriptor = nativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns( + library, + configuration.languageVersionSettings, + LockBasedStorageManager("ModulesStructure"), + testServices.moduleDescriptorProvider.getModuleDescriptor(module).builtIns, + packageAccessHandler = null, + lookupTracker = LookupTracker.DO_NOTHING + ) + moduleDescriptor.setDependencies(dependencyPaths.map { testServices.libraryProvider.getDescriptorByPath(it) as ModuleDescriptorImpl } + moduleDescriptor) + + testServices.moduleDescriptorProvider.replaceModuleDescriptorForModule(module, moduleDescriptor) + testServices.libraryProvider.setDescriptorAndLibraryByName(outputArtifact.outputFile.path, moduleDescriptor, library) + } +} + +/** + * The Native KLIB facade suitable for FIR frontend. + */ +class FirNativeKlibBackendFacade(testServices: TestServices) : AbstractNativeKlibBackendFacade(testServices) { + override fun serialize( + configuration: CompilerConfiguration, + dependencies: List, + module: TestModule, + inputArtifact: IrBackendInput.NativeBackendInput, + ) = serializeModuleIntoKlib( + moduleName = inputArtifact.irModuleFragment.name.asString(), + inputArtifact.irModuleFragment, + configuration, + inputArtifact.diagnosticReporter, + CompatibilityMode.CURRENT, + cleanFiles = emptyList(), + dependencies, + createModuleSerializer = { + irDiagnosticReporter, + irBuiltIns, + compatibilityMode, + normalizeAbsolutePaths, + sourceBaseDirs, + languageVersionSettings, + shouldCheckSignaturesOnUniqueness, + -> + KonanIrModuleSerializer( + diagnosticReporter = irDiagnosticReporter, + irBuiltIns = irBuiltIns, + compatibilityMode = compatibilityMode, + normalizeAbsolutePaths = normalizeAbsolutePaths, + sourceBaseDirs = sourceBaseDirs, + languageVersionSettings = languageVersionSettings, + bodiesOnlyForInlines = false, + publicAbiOnly = false, + shouldCheckSignaturesOnUniqueness = shouldCheckSignaturesOnUniqueness, + ) + }, + inputArtifact.metadataSerializer ?: error("expected metadata serializer"), + ) +} diff --git a/native/utils/src/org/jetbrains/kotlin/konan/library/impl/KonanLibraryWriterImpl.kt b/native/utils/src/org/jetbrains/kotlin/konan/library/impl/KonanLibraryWriterImpl.kt index 719273be57c..cf04ce2ac43 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/library/impl/KonanLibraryWriterImpl.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/library/impl/KonanLibraryWriterImpl.kt @@ -44,7 +44,7 @@ class KonanLibraryWriterImpl( fun buildLibrary( natives: List, included: List, - linkDependencies: List, + linkDependencies: List, metadata: SerializedMetadata, ir: SerializedIrModule?, versions: KotlinLibraryVersioning,