From 8bf108dc17504da38eee62d235458aceae60b6d5 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Tue, 7 Jun 2022 17:05:54 +0200 Subject: [PATCH] FIR JS: Prepare JS/KLib infrastructure for FIR integration rearrange declarations, abstract things that should be done differently with FIR, etc. --- .../jetbrains/kotlin/cli/js/K2JsIrCompiler.kt | 36 ++++- .../js/klib}/TopDownAnalyzerFacadeForJSIR.kt | 5 +- .../jetbrains/kotlin/cli/js/klib/irForKlib.kt | 100 ++++++++++++ .../js/klib/prepareAnalyzedSourceModule.kt | 30 ++++ .../KlibMetadataIncrementalSerializer.kt | 30 ++-- .../metadata/KlibMetadataSerializer.kt | 41 +++-- .../jetbrains/kotlin/ir/backend/js/klib.kt | 152 +++++++++++------- .../kotlin/test/backend/ir/IrBackendInput.kt | 10 +- .../classic/ClassicFrontend2IrConverter.kt | 26 ++- .../frontend/classic/ClassicFrontendFacade.kt | 1 + .../test/frontend/fir/FirFrontendFacade.kt | 2 +- .../kotlin/klib/AbstractKlibTextTestCase.kt | 3 +- .../kotlin/klib/FilePathsInKlibTest.kt | 41 ++++- .../kotlin/benchmarks/GenerateIrRuntime.kt | 10 +- .../incremental/AbstractInvalidationTest.kt | 34 +++- .../test/AbstractJsBlackBoxCodegenTestBase.kt | 2 +- .../js/test/AbstractJsKLibABITestCase.kt | 35 +++- .../js/test/converters/JsKlibBackendFacade.kt | 12 +- .../kotlin/js/test/ir/AbstractJsIrTest.kt | 101 +++++------- .../box/multiModule/samePackageNames.kt | 1 - 20 files changed, 479 insertions(+), 193 deletions(-) rename compiler/{ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js => cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib}/TopDownAnalyzerFacadeForJSIR.kt (89%) create mode 100644 compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt create mode 100644 compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/prepareAnalyzedSourceModule.kt diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index 8cf1ca8e917..de3c696227c 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -38,12 +38,15 @@ import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.incremental.js.IncrementalNextRoundChecker import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.backend.wasm.dce.eliminateDeadDeclarations +import org.jetbrains.kotlin.cli.js.klib.generateIrForKlibSerialization +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity import org.jetbrains.kotlin.ir.backend.js.ic.* import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.* import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC +import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult import org.jetbrains.kotlin.js.config.* import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion @@ -175,6 +178,8 @@ class K2JsIrCompiler : CLICompiler() { configuration.put(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH, arguments.normalizeAbsolutePath) configuration.put(CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS, arguments.enableSignatureClashChecks) + // ---- + val environmentForJS = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JS_CONFIG_FILES) val projectJs = environmentForJS.project @@ -301,17 +306,44 @@ class K2JsIrCompiler : CLICompiler() { } if (arguments.irProduceKlibDir || arguments.irProduceKlibFile) { + if (arguments.irProduceKlibFile) { + require(outputFile.extension == KLIB_FILE_EXTENSION) { "Please set up .klib file as output" } + } + val moduleSourceFiles = (sourceModule.mainModule as MainModule.SourceFiles).files + val icData = configurationJs.incrementalDataProvider?.getSerializedData(moduleSourceFiles) ?: emptyList() + val expectDescriptorToSymbol = mutableMapOf() + + val moduleFragment = generateIrForKlibSerialization( + projectJs, + moduleSourceFiles, + configurationJs, + sourceModule.jsFrontEndResult.jsAnalysisResult, + sortDependencies(sourceModule.descriptors), + icData, + expectDescriptorToSymbol, + IrFactoryImpl, + verifySignatures = true + ) { + sourceModule.getModuleDescriptor(it) + } + + val metadataSerializer = + KlibMetadataIncrementalSerializer(configuration, sourceModule.project, sourceModule.jsFrontEndResult.hasErrors) generateKLib( sourceModule, - irFactory = IrFactoryImpl, outputKlibPath = if (arguments.irProduceKlibFile) outputDir.resolve("$outputName.klib").normalize().absolutePath else outputDirPath, nopack = arguments.irProduceKlibDir, jsOutputName = arguments.irPerModuleOutputName, - ) + icData = icData, + expectDescriptorToSymbol = expectDescriptorToSymbol, + moduleFragment = moduleFragment + ) { file -> + metadataSerializer.serializeScope(file, sourceModule.jsFrontEndResult.bindingContext, moduleFragment.descriptor) + } } if (arguments.irProduceJs) { diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/TopDownAnalyzerFacadeForJSIR.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/TopDownAnalyzerFacadeForJSIR.kt similarity index 89% rename from compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/TopDownAnalyzerFacadeForJSIR.kt rename to compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/TopDownAnalyzerFacadeForJSIR.kt index 52e63ad57cd..97e5b43b0cd 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/TopDownAnalyzerFacadeForJSIR.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/TopDownAnalyzerFacadeForJSIR.kt @@ -1,15 +1,16 @@ /* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.ir.backend.js +package org.jetbrains.kotlin.cli.js.klib import org.jetbrains.kotlin.config.LanguageVersionSettings import org.jetbrains.kotlin.context.ModuleContext import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider +import org.jetbrains.kotlin.ir.backend.js.JsFactories import org.jetbrains.kotlin.js.analyze.AbstractTopDownAnalyzerFacadeForJS import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt new file mode 100644 index 00000000000..70d437b0220 --- /dev/null +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.cli.js.klib + +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.analyzer.AnalysisResult +import org.jetbrains.kotlin.backend.common.lower.ExpectDeclarationRemover +import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideChecker +import org.jetbrains.kotlin.backend.common.serialization.ICData +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnboundSymbols +import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker +import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter +import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.languageVersionSettings +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData +import org.jetbrains.kotlin.ir.backend.js.generateModuleFragmentWithPlugins +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerIr +import org.jetbrains.kotlin.ir.declarations.IrFactory +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.ir.util.IrMessageLogger +import org.jetbrains.kotlin.ir.util.SymbolTable +import org.jetbrains.kotlin.ir.visitors.acceptVoid +import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy +import org.jetbrains.kotlin.js.config.JSConfigurationKeys +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration +import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator + +fun generateIrForKlibSerialization( + project: Project, + files: List, + configuration: CompilerConfiguration, + analysisResult: AnalysisResult, + sortedDependencies: Collection, + icData: List, + expectDescriptorToSymbol: MutableMap, + irFactory: IrFactory, + verifySignatures: Boolean = true, + getDescriptorByLibrary: (KotlinLibrary) -> ModuleDescriptor, +): IrModuleFragment { + val errorPolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT + val messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None + val allowUnboundSymbols = configuration[JSConfigurationKeys.PARTIAL_LINKAGE] ?: false + val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), irFactory) + val psi2Ir = Psi2IrTranslator( + configuration.languageVersionSettings, + Psi2IrConfiguration(errorPolicy.allowErrors, allowUnboundSymbols), + messageLogger::checkNoUnboundSymbols + ) + val psi2IrContext = psi2Ir.createGeneratorContext(analysisResult.moduleDescriptor, analysisResult.bindingContext, symbolTable) + val irBuiltIns = psi2IrContext.irBuiltIns + + val feContext = psi2IrContext.run { + JsIrLinker.JsFePluginContext(moduleDescriptor, symbolTable, typeTranslator, irBuiltIns) + } + val irLinker = JsIrLinker( + psi2IrContext.moduleDescriptor, + messageLogger, + psi2IrContext.irBuiltIns, + psi2IrContext.symbolTable, + partialLinkageEnabled = configuration[JSConfigurationKeys.PARTIAL_LINKAGE] ?: false, + feContext, + ICData(icData.map { it.irData }, errorPolicy.allowErrors) + ) + + sortedDependencies.map { irLinker.deserializeOnlyHeaderModule(getDescriptorByLibrary(it), it) } + + val moduleFragment = psi2IrContext.generateModuleFragmentWithPlugins(project, files, irLinker, messageLogger, expectDescriptorToSymbol) + + if (verifySignatures) { + moduleFragment.acceptVoid(ManglerChecker(JsManglerIr, Ir2DescriptorManglerAdapter(JsManglerDesc))) + } + if (configuration.getBoolean(JSConfigurationKeys.FAKE_OVERRIDE_VALIDATOR)) { + val fakeOverrideChecker = FakeOverrideChecker(JsManglerIr, JsManglerDesc) + irLinker.modules.forEach { fakeOverrideChecker.check(it) } + } + + if (configuration.get(CommonConfigurationKeys.EXPECT_ACTUAL_LINKER) != true) { + moduleFragment.transform(ExpectDeclarationRemover(psi2IrContext.symbolTable, false), null) + } + + return moduleFragment +} + diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/prepareAnalyzedSourceModule.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/prepareAnalyzedSourceModule.kt new file mode 100644 index 00000000000..b42a3c8f804 --- /dev/null +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/prepareAnalyzedSourceModule.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js + +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.analyzer.AbstractAnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.js.klib.TopDownAnalyzerFacadeForJSIR +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy +import org.jetbrains.kotlin.js.config.JSConfigurationKeys +import org.jetbrains.kotlin.psi.KtFile + +fun prepareAnalyzedSourceModule( + project: Project, + files: List, + configuration: CompilerConfiguration, + dependencies: List, + friendDependencies: List, + analyzer: AbstractAnalyzerWithCompilerReport, + errorPolicy: ErrorTolerancePolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT, +): ModulesStructure { + val mainModule = MainModule.SourceFiles(files) + val sourceModule = ModulesStructure(project, mainModule, configuration, dependencies, friendDependencies) + return sourceModule.apply { + runAnalysis(errorPolicy, analyzer, TopDownAnalyzerFacadeForJSIR) + } +} \ No newline at end of file diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataIncrementalSerializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataIncrementalSerializer.kt index 1ae029cdeed..e45430f19d2 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataIncrementalSerializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataIncrementalSerializer.kt @@ -56,21 +56,6 @@ class KlibMetadataIncrementalSerializer( return serializeDescriptors(fqName, classifierDescriptors, topLevelDescriptors).single() } - fun serializedMetadata( - fragments: Map>, - header: ByteArray - ): SerializedMetadata { - val fragmentNames = mutableListOf() - val fragmentParts = mutableListOf>() - - for ((fqName, fragment) in fragments.entries.sortedBy { it.key }) { - fragmentNames += fqName - fragmentParts += fragment - } - - return SerializedMetadata(header, fragmentParts, fragmentNames) - } - // TODO: For now, in the incremental serializer, we assume // there is only a single package fragment per file. // This is no always the case, actually. @@ -78,4 +63,19 @@ class KlibMetadataIncrementalSerializer( // See monolithic serializer for details. override val TOP_LEVEL_DECLARATION_COUNT_PER_FILE = null override val TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE = null +} + +fun makeSerializedKlibMetadata( + fragments: Map>, + header: ByteArray +): SerializedMetadata { + val fragmentNames = mutableListOf() + val fragmentParts = mutableListOf>() + + for ((fqName, fragment) in fragments.entries.sortedBy { it.key }) { + fragmentNames += fqName + fragmentParts += fragment + } + + return SerializedMetadata(header, fragmentParts, fragmentNames) } \ No newline at end of file diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt index d9e10f1e13b..01e7ffaac7a 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt @@ -253,22 +253,7 @@ abstract class KlibMetadataSerializer( fragmentNames: List, emptyPackages: List ): KlibMetadataProtoBuf.Header { - val header = KlibMetadataProtoBuf.Header.newBuilder() - - header.moduleName = moduleDescriptor.name.asString() - - if (languageVersionSettings.isPreRelease()) { - header.flags = 1 - } - - fragmentNames.forEach { - header.addPackageFragmentName(it) - } - emptyPackages.forEach { - header.addEmptyPackage(it) - } - - return header.build() + return serializeKlibHeader(languageVersionSettings, moduleDescriptor, fragmentNames, emptyPackages) } // For platform libraries we get HUGE files. @@ -278,6 +263,30 @@ abstract class KlibMetadataSerializer( abstract protected val TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE: Int? } +fun serializeKlibHeader( + languageVersionSettings: LanguageVersionSettings, + moduleDescriptor: ModuleDescriptor, + fragmentNames: List, + emptyPackages: List +): KlibMetadataProtoBuf.Header { + val header = KlibMetadataProtoBuf.Header.newBuilder() + + header.moduleName = moduleDescriptor.name.asString() + + if (languageVersionSettings.isPreRelease()) { + header.flags = 1 + } + + fragmentNames.forEach { + header.addPackageFragmentName(it) + } + emptyPackages.forEach { + header.addEmptyPackage(it) + } + + return header.build() +} + fun DeclarationDescriptor.extractFileId(): Int? = when (this) { is DeserializedClassDescriptor -> classProto.getExtension(KlibMetadataProtoBuf.classFile) is DeserializedSimpleFunctionDescriptor -> proto.getExtension(KlibMetadataProtoBuf.functionFile) diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index c51fc5b8f29..ca514452f83 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -8,6 +8,10 @@ package org.jetbrains.kotlin.ir.backend.js import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.VfsUtilCore import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtPsiSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.analyzer.AbstractAnalyzerWithCompilerReport import org.jetbrains.kotlin.analyzer.AnalysisResult import org.jetbrains.kotlin.analyzer.CompilationErrorException @@ -19,9 +23,7 @@ import org.jetbrains.kotlin.backend.common.serialization.* import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnboundSymbols import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter -import org.jetbrains.kotlin.backend.common.serialization.metadata.DynamicTypeDeserializer -import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataIncrementalSerializer -import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion +import org.jetbrains.kotlin.backend.common.serialization.metadata.* import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.config.* @@ -30,6 +32,7 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.ir.IrBuiltIns import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.* @@ -41,6 +44,7 @@ import org.jetbrains.kotlin.ir.linkage.IrDeserializer import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.visitors.acceptVoid +import org.jetbrains.kotlin.js.analyze.AbstractTopDownAnalyzerFacadeForJS import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy import org.jetbrains.kotlin.js.config.JSConfigurationKeys @@ -205,42 +209,24 @@ fun generateIrForKlibSerialization( fun generateKLib( depsDescriptors: ModulesStructure, - irFactory: IrFactory, outputKlibPath: String, nopack: Boolean, - verifySignatures: Boolean = true, abiVersion: KotlinAbiVersion = KotlinAbiVersion.CURRENT, - jsOutputName: String? + jsOutputName: String?, + icData: List, + expectDescriptorToSymbol: MutableMap, + moduleFragment: IrModuleFragment, + serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment ) { - val project = depsDescriptors.project - val files = (depsDescriptors.mainModule as MainModule.SourceFiles).files + val files = (depsDescriptors.mainModule as MainModule.SourceFiles).files.map(::KtPsiSourceFile) val configuration = depsDescriptors.compilerConfiguration val allDependencies = depsDescriptors.allDependencies.map { it.library } val messageLogger = configuration.irMessageLogger - val icData = mutableListOf() - val expectDescriptorToSymbol = mutableMapOf() - - val moduleFragment = generateIrForKlibSerialization( - project, - files, - configuration, - depsDescriptors.jsFrontEndResult.jsAnalysisResult, - sortDependencies(depsDescriptors.descriptors), - icData, - expectDescriptorToSymbol, - irFactory, - verifySignatures - ) { - depsDescriptors.getModuleDescriptor(it) - } - serializeModuleIntoKlib( configuration[CommonConfigurationKeys.MODULE_NAME]!!, - project, configuration, messageLogger, - depsDescriptors.jsFrontEndResult.bindingContext, files, outputKlibPath, allDependencies, @@ -251,7 +237,8 @@ fun generateKLib( perFile = false, depsDescriptors.jsFrontEndResult.hasErrors, abiVersion, - jsOutputName + jsOutputName, + serializeSingleFile ) } @@ -484,22 +471,6 @@ fun getIrModuleInfoForSourceFiles( ) } -fun prepareAnalyzedSourceModule( - project: Project, - files: List, - configuration: CompilerConfiguration, - dependencies: List, - friendDependencies: List, - analyzer: AbstractAnalyzerWithCompilerReport, - errorPolicy: ErrorTolerancePolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT, -): ModulesStructure { - val mainModule = MainModule.SourceFiles(files) - val sourceModule = ModulesStructure(project, mainModule, configuration, dependencies, friendDependencies) - return sourceModule.apply { - runAnalysis(errorPolicy, analyzer) - } -} - private fun preparePsi2Ir( depsDescriptors: ModulesStructure, errorIgnorancePolicy: ErrorTolerancePolicy, @@ -633,12 +604,12 @@ class ModulesStructure( lateinit var jsFrontEndResult: JsFrontEndResult - fun runAnalysis(errorPolicy: ErrorTolerancePolicy, analyzer: AbstractAnalyzerWithCompilerReport) { + fun runAnalysis(errorPolicy: ErrorTolerancePolicy, analyzer: AbstractAnalyzerWithCompilerReport, analyzerFacade: AbstractTopDownAnalyzerFacadeForJS) { require(mainModule is MainModule.SourceFiles) val files = mainModule.files analyzer.analyzeAndReport(files) { - TopDownAnalyzerFacadeForJSIR.analyzeFiles( + analyzerFacade.analyzeFiles( files, project, compilerConfiguration, @@ -665,7 +636,7 @@ class ModulesStructure( else hasErrors = true } - hasErrors = TopDownAnalyzerFacadeForJSIR.checkForErrors(files, analysisResult.bindingContext, errorPolicy) || hasErrors + hasErrors = analyzerFacade.checkForErrors(files, analysisResult.bindingContext, errorPolicy) || hasErrors jsFrontEndResult = JsFrontEndResult(analysisResult, hasErrors) } @@ -712,11 +683,9 @@ private fun getDescriptorForElement( fun serializeModuleIntoKlib( moduleName: String, - project: Project, configuration: CompilerConfiguration, messageLogger: IrMessageLogger, - bindingContext: BindingContext, - files: List, + files: List, klibPath: String, dependencies: List, moduleFragment: IrModuleFragment, @@ -727,6 +696,7 @@ fun serializeModuleIntoKlib( containsErrorCode: Boolean = false, abiVersion: KotlinAbiVersion, jsOutputName: String?, + serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment ) { assert(files.size == moduleFragment.files.size) @@ -748,7 +718,6 @@ fun serializeModuleIntoKlib( ).serializedIrModule(moduleFragment) val moduleDescriptor = moduleFragment.descriptor - val metadataSerializer = KlibMetadataIncrementalSerializer(configuration, project, containsErrorCode) val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER) val empty = ByteArray(0) @@ -764,33 +733,38 @@ fun serializeModuleIntoKlib( val additionalFiles = mutableListOf() - for ((ktFile, binaryFile) in files.zip(serializedIr.files)) { - assert(ktFile.virtualFilePath == binaryFile.path) { + for ((ktSourceFile, binaryFile) in files.zip(serializedIr.files)) { + assert(ktSourceFile.path == binaryFile.path) { """The Kt and Ir files are put in different order - Kt: ${ktFile.virtualFilePath} + Kt: ${ktSourceFile.path} Ir: ${binaryFile.path} """.trimMargin() } - val packageFragment = metadataSerializer.serializeScope(ktFile, bindingContext, moduleDescriptor) + val packageFragment = serializeSingleFile(ktSourceFile) val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile) additionalFiles += compiledKotlinFile - processCompiledFileData(VfsUtilCore.virtualToIoFile(ktFile.virtualFile), compiledKotlinFile) + val ioFile = ktSourceFile.toIoFileOrNull() + assert(ioFile != null) { + "No file found for source ${ktSourceFile.path}" + } + processCompiledFileData(ioFile!!, compiledKotlinFile) } val compiledKotlinFiles = (cleanFiles + additionalFiles) - val header = metadataSerializer.serializeHeader( - moduleDescriptor, + val header = serializeKlibHeader( + configuration.languageVersionSettings, moduleDescriptor, compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(), emptyList() ).toByteArray() + incrementalResultsConsumer?.run { processHeader(header) } val serializedMetadata = - metadataSerializer.serializedMetadata( + makeSerializedKlibMetadata( compiledKotlinFiles.groupBy { it.irData.fqName } .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(), header @@ -832,7 +806,7 @@ fun serializeModuleIntoKlib( const val KLIB_PROPERTY_JS_OUTPUT_NAME = "jsOutputName" -private fun KlibMetadataIncrementalSerializer.serializeScope( +fun KlibMetadataIncrementalSerializer.serializeScope( ktFile: KtFile, bindingContext: BindingContext, moduleDescriptor: ModuleDescriptor @@ -841,6 +815,16 @@ private fun KlibMetadataIncrementalSerializer.serializeScope( return serializePackageFragment(moduleDescriptor, memberScope, ktFile.packageFqName) } +fun KlibMetadataIncrementalSerializer.serializeScope( + ktSourceFile: KtSourceFile, + bindingContext: BindingContext, + moduleDescriptor: ModuleDescriptor +): ProtoBuf.PackageFragment { + val ktFile = (ktSourceFile as KtPsiSourceFile).psiFile as KtFile + val memberScope = ktFile.declarations.map { getDescriptorForElement(bindingContext, it) } + return serializePackageFragment(moduleDescriptor, memberScope, ktFile.packageFqName) +} + private fun compareMetadataAndGoToNextICRoundIfNeeded( analysisResult: AnalysisResult, config: CompilerConfiguration, @@ -861,7 +845,7 @@ private fun compareMetadataAndGoToNextICRoundIfNeeded( if (nextRoundChecker.shouldGoToNextRound()) throw IncrementalNextRoundException() } -private fun KlibMetadataIncrementalSerializer(configuration: CompilerConfiguration, project: Project, allowErrors: Boolean) = +fun KlibMetadataIncrementalSerializer(configuration: CompilerConfiguration, project: Project, allowErrors: Boolean) = KlibMetadataIncrementalSerializer( languageVersionSettings = configuration.languageVersionSettings, metadataVersion = configuration.metadataVersion, @@ -876,3 +860,49 @@ private fun Map.getUniqueNameForEachFragment(): klib.jsOutputName?.let { moduleFragment to it } }.toMap() } + +fun KtSourceFile.toIoFileOrNull(): File? = when (this) { + is KtIoFileSourceFile -> file + is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile) + is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile) + else -> path?.let(::File) +} + +fun IncrementalDataProvider.getSerializedData(newSources: List): List { + val nonCompiledSources = newSources.associateBy { it.toIoFileOrNull()!! } + val compiledIrFiles = serializedIrFiles + val compiledMetaFiles = compiledPackageParts + + assert(compiledIrFiles.size == compiledMetaFiles.size) + + val storage = mutableListOf() + + for (f in compiledIrFiles.keys) { + if (f in nonCompiledSources) continue + + val irData = compiledIrFiles[f] ?: error("No Ir Data found for file $f") + val metaFile = compiledMetaFiles[f] ?: error("No Meta Data found for file $f") + val irFile = with(irData) { + SerializedIrFile( + fileData, + String(fqn), + f.path.replace('\\', '/'), + types, + signatures, + strings, + bodies, + declarations, + debugInfo + ) + } + storage.add(KotlinFileSerializedData(metaFile.metadata, irFile)) + } + return storage +} + +@JvmName("getSerializedDataByPsiFiles") +fun IncrementalDataProvider.getSerializedData(newSources: List): List = + getSerializedData(newSources.map(::KtPsiSourceFile)) + +val CompilerConfiguration.incrementalDataProvider : IncrementalDataProvider? + get() = get(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER) \ No newline at end of file diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt index 2da82e565d0..ca548f46ea8 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt @@ -7,14 +7,13 @@ package org.jetbrains.kotlin.test.backend.ir import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory -import org.jetbrains.kotlin.codegen.CodegenFactory import org.jetbrains.kotlin.codegen.state.GenerationState import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.metadata.ProtoBuf import org.jetbrains.kotlin.psi.KtFile -import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.test.model.BackendKinds import org.jetbrains.kotlin.test.model.ResultingArtifact @@ -27,10 +26,11 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput() { data class JsIrBackendInput( override val irModuleFragment: IrModuleFragment, - val sourceFiles: List, - val bindingContext: BindingContext, + val sourceFiles: List, val icData: List, - val expectDescriptorToSymbol: MutableMap, + val expectDescriptorToSymbol: MutableMap, // TODO: abstract from descriptors + val hasErrors: Boolean, + val serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment ) : IrBackendInput() data class JvmIrBackendInput( diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt index 6821f0c83d6..0f1055727c9 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt @@ -5,18 +5,21 @@ package org.jetbrains.kotlin.test.frontend.classic +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.cli.js.klib.TopDownAnalyzerFacadeForJSIR +import org.jetbrains.kotlin.cli.js.klib.generateIrForKlibSerialization import org.jetbrains.kotlin.codegen.ClassBuilderFactories import org.jetbrains.kotlin.codegen.CodegenFactory import org.jetbrains.kotlin.codegen.state.GenerationState import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory -import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData -import org.jetbrains.kotlin.ir.backend.js.generateIrForKlibSerialization -import org.jetbrains.kotlin.ir.backend.js.sortDependencies +import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy +import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.test.TargetBackend import org.jetbrains.kotlin.test.backend.ir.IrBackendInput import org.jetbrains.kotlin.test.directives.CodegenTestDirectives @@ -75,11 +78,12 @@ class ClassicFrontend2IrConverter( val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module) val verifySignatures = JsEnvironmentConfigurationDirectives.SKIP_MANGLE_VERIFICATION !in module.directives - val icData = mutableListOf() + val sourceFiles = psiFiles.values.toList() + val icData = configuration.incrementalDataProvider?.getSerializedData(sourceFiles) ?: emptyList() val expectDescriptorToSymbol = mutableMapOf() val moduleFragment = generateIrForKlibSerialization( project, - psiFiles.values.toList(), + sourceFiles, configuration, analysisResult, sortDependencies(JsEnvironmentConfigurator.getAllRecursiveLibrariesFor(module, testServices)), @@ -91,12 +95,18 @@ class ClassicFrontend2IrConverter( testServices.jsLibraryProvider.getDescriptorByCompiledLibrary(it) } + val errorPolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT + val hasErrors = TopDownAnalyzerFacadeForJSIR.checkForErrors(sourceFiles, analysisResult.bindingContext, errorPolicy) + val metadataSerializer = KlibMetadataIncrementalSerializer(configuration, project, hasErrors) + return IrBackendInput.JsIrBackendInput( moduleFragment, - psiFiles.values.toList(), - bindingContext = analysisResult.bindingContext, + sourceFiles.map(::KtPsiSourceFile), icData, expectDescriptorToSymbol = expectDescriptorToSymbol, - ) + hasErrors + ) { file -> + metadataSerializer.serializeScope(file, analysisResult.bindingContext, moduleFragment.descriptor) + } } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontendFacade.kt index 9d6d385031f..ceecf0ab255 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontendFacade.kt @@ -13,6 +13,7 @@ import org.jetbrains.kotlin.builtins.DefaultBuiltIns import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.builtins.jvm.JvmBuiltIns import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.js.klib.TopDownAnalyzerFacadeForJSIR import org.jetbrains.kotlin.cli.jvm.compiler.JvmPackagePartProvider import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt index 57d7aa56703..dc9f340c9e6 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt @@ -249,7 +249,7 @@ private fun getJsDependencies(module: TestModule, testServices: TestServices): T return Triple(runtimeKlibsPaths, transitiveLibraries, friendLibraries) } -private fun getAllJsDependenciesPaths(module: TestModule, testServices: TestServices): List { +internal fun getAllJsDependenciesPaths(module: TestModule, testServices: TestServices): List { val (runtimeKlibsPaths, transitiveLibraries, friendLibraries) = getJsDependencies(module, testServices) return runtimeKlibsPaths + transitiveLibraries.map { it.path } + friendLibraries.map { it.path } } diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibTextTestCase.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibTextTestCase.kt index 2f3383f021c..f0b17482698 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibTextTestCase.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibTextTestCase.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.backend.common.serialization.KlibIrVersion import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnboundSymbols import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataIncrementalSerializer import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion +import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment @@ -172,7 +173,7 @@ abstract class AbstractKlibTextTestCase : CodegenTestCase() { ).toByteArray() - val serializedMetadata = metadataSerializer.serializedMetadata( + val serializedMetadata = makeSerializedKlibMetadata( compiledKotlinFiles.groupBy { it.irData.fqName } .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(), header diff --git a/compiler/tests/org/jetbrains/kotlin/klib/FilePathsInKlibTest.kt b/compiler/tests/org/jetbrains/kotlin/klib/FilePathsInKlibTest.kt index ef9065274c1..6e0fc7797bc 100644 --- a/compiler/tests/org/jetbrains/kotlin/klib/FilePathsInKlibTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/klib/FilePathsInKlibTest.kt @@ -11,17 +11,17 @@ import com.intellij.psi.PsiManager import org.jetbrains.kotlin.backend.common.serialization.codedInputStream import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.js.klib.generateIrForKlibSerialization import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.codegen.CodegenTestCase import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.incremental.md5 -import org.jetbrains.kotlin.ir.backend.js.ModulesStructure -import org.jetbrains.kotlin.ir.backend.js.generateKLib -import org.jetbrains.kotlin.ir.backend.js.jsResolveLibraries -import org.jetbrains.kotlin.ir.backend.js.prepareAnalyzedSourceModule +import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl +import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.test.util.KtTestUtil @@ -58,7 +58,38 @@ class FilePathsInKlibTest : CodegenTestCase() { } private fun produceKlib(module: ModulesStructure, destination: File) { - generateKLib(module, irFactory = IrFactoryImpl, outputKlibPath = destination.path, nopack = false, jsOutputName = MODULE_NAME) + // TODO: improve API for generateIrForKlibSerialization and related functionality and remove code duplication here and in similar places in the code + val sourceFiles = (module.mainModule as MainModule.SourceFiles).files + val icData = module.compilerConfiguration.incrementalDataProvider?.getSerializedData(sourceFiles) ?: emptyList() + val expectDescriptorToSymbol = mutableMapOf() + val moduleFragment = generateIrForKlibSerialization( + module.project, + sourceFiles, + module.compilerConfiguration, + module.jsFrontEndResult.jsAnalysisResult, + sortDependencies(module.descriptors), + icData, + expectDescriptorToSymbol, + IrFactoryImpl, + verifySignatures = true + ) { + module.getModuleDescriptor(it) + } + + val metadataSerializer = + KlibMetadataIncrementalSerializer(module.compilerConfiguration, module.project, module.jsFrontEndResult.hasErrors) + + generateKLib( + module, + outputKlibPath = destination.path, + nopack = false, + jsOutputName = MODULE_NAME, + icData = icData, + expectDescriptorToSymbol = expectDescriptorToSymbol, + moduleFragment = moduleFragment + ) { file -> + metadataSerializer.serializeScope(file, module.jsFrontEndResult.bindingContext, moduleFragment.descriptor) + } } private fun setupEnvironment(): CompilerConfiguration { diff --git a/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt b/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt index 81bdc23d193..6daf3dce87e 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/benchmarks/GenerateIrRuntime.kt @@ -11,6 +11,7 @@ import com.intellij.openapi.vfs.StandardFileSystems import com.intellij.openapi.vfs.VfsUtilCore import com.intellij.openapi.vfs.VirtualFileManager import com.intellij.psi.PsiManager +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.analyzer.AnalysisResult import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel @@ -492,13 +493,12 @@ class GenerateIrRuntime { perFile: Boolean = false ): String { val tmpKlibDir = createTempDirectory().also { it.toFile().deleteOnExit() }.toString() + val metadataSerializer = KlibMetadataIncrementalSerializer(configuration, project, false) serializeModuleIntoKlib( moduleName, - project, configuration, IrMessageLogger.None, - bindingContext, - files, + files.map(::KtPsiSourceFile), tmpKlibDir, emptyList(), moduleFragment, @@ -508,7 +508,9 @@ class GenerateIrRuntime { perFile, abiVersion = KotlinAbiVersion.CURRENT, jsOutputName = null - ) + ) { file -> + metadataSerializer.serializeScope(file, bindingContext, moduleFragment.descriptor) + } return tmpKlibDir } diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index 4a751c1da09..0f101b0d240 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -11,6 +11,7 @@ import com.intellij.openapi.vfs.local.CoreLocalFileSystem import com.intellij.psi.PsiManager import com.intellij.psi.SingleRootFileViewProvider import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.js.klib.generateIrForKlibSerialization import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.codegen.* @@ -18,6 +19,7 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.LanguageFeature import org.jetbrains.kotlin.config.languageVersionSettings +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.backend.js.ic.* import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo @@ -25,6 +27,7 @@ import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.safeModuleName import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC +import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.js.testOld.V8IrJsTestChecker import org.jetbrains.kotlin.name.FqName @@ -294,7 +297,36 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { AnalyzerWithCompilerReport(configuration) ) - generateKLib(sourceModule, IrFactoryImpl, outputKlibFile.canonicalPath, nopack = false, jsOutputName = moduleName) + val moduleSourceFiles = (sourceModule.mainModule as MainModule.SourceFiles).files + val icData = sourceModule.compilerConfiguration.incrementalDataProvider?.getSerializedData(moduleSourceFiles) ?: emptyList() + val expectDescriptorToSymbol = mutableMapOf() + val moduleFragment = generateIrForKlibSerialization( + environment.project, + moduleSourceFiles, + configuration, + sourceModule.jsFrontEndResult.jsAnalysisResult, + sortDependencies(sourceModule.descriptors), + icData, + expectDescriptorToSymbol, + IrFactoryImpl, + verifySignatures = true + ) { + sourceModule.getModuleDescriptor(it) + } + val metadataSerializer = + KlibMetadataIncrementalSerializer(configuration, sourceModule.project, sourceModule.jsFrontEndResult.hasErrors) + + generateKLib( + sourceModule, + outputKlibFile.canonicalPath, + nopack = false, + jsOutputName = moduleName, + icData = icData, + expectDescriptorToSymbol = expectDescriptorToSymbol, + moduleFragment = moduleFragment + ) { file -> + metadataSerializer.serializeScope(file, sourceModule.jsFrontEndResult.bindingContext, moduleFragment.descriptor) + } } private fun initializeWorkingDir(projectInfo: ProjectInfo, testDir: File, sourceDir: File, buildDir: File) { diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsBlackBoxCodegenTestBase.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsBlackBoxCodegenTestBase.kt index e8a1a802b08..318fc16763c 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsBlackBoxCodegenTestBase.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsBlackBoxCodegenTestBase.kt @@ -32,7 +32,7 @@ abstract class AbstractJsBlackBoxCodegenTestBase> abstract val frontendToBackendConverter: Constructor> diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsKLibABITestCase.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsKLibABITestCase.kt index 53a5e87a3b4..b3948d8a273 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsKLibABITestCase.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsKLibABITestCase.kt @@ -12,11 +12,13 @@ import com.intellij.psi.PsiManager import com.intellij.psi.SingleRootFileViewProvider import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.js.klib.generateIrForKlibSerialization import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.codegen.ProjectInfo import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdater @@ -25,6 +27,7 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransf import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.TranslationMode import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC +import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.js.testOld.V8IrJsTestChecker import org.jetbrains.kotlin.klib.KlibABITestUtils @@ -118,7 +121,37 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() { AnalyzerWithCompilerReport(config) ) - generateKLib(sourceModule, IrFactoryImpl, klibFile.path, nopack = false, jsOutputName = moduleName) + val moduleSourceFiles = (sourceModule.mainModule as MainModule.SourceFiles).files + val icData = sourceModule.compilerConfiguration.incrementalDataProvider?.getSerializedData(moduleSourceFiles) ?: emptyList() + val expectDescriptorToSymbol = mutableMapOf() + val moduleFragment = generateIrForKlibSerialization( + environment.project, + moduleSourceFiles, + config, + sourceModule.jsFrontEndResult.jsAnalysisResult, + sortDependencies(sourceModule.descriptors), + icData, + expectDescriptorToSymbol, + IrFactoryImpl, + verifySignatures = true + ) { + sourceModule.getModuleDescriptor(it) + } + + val metadataSerializer = + KlibMetadataIncrementalSerializer(config, sourceModule.project, sourceModule.jsFrontEndResult.hasErrors) + + generateKLib( + sourceModule, + klibFile.path, + nopack = false, + jsOutputName = moduleName, + icData = icData, + expectDescriptorToSymbol = expectDescriptorToSymbol, + moduleFragment = moduleFragment + ) { file -> + metadataSerializer.serializeScope(file, sourceModule.jsFrontEndResult.bindingContext, moduleFragment.descriptor) + } } private fun buildBinaryAndRun(mainModuleKlibFile: File, allDependencies: KlibABITestUtils.Dependencies) { diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsKlibBackendFacade.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsKlibBackendFacade.kt index b76d394bfdb..2172853308f 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsKlibBackendFacade.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsKlibBackendFacade.kt @@ -9,6 +9,7 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.ir.backend.js.* +import org.jetbrains.kotlin.ir.util.IrMessageLogger import org.jetbrains.kotlin.ir.util.irMessageLogger import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy import org.jetbrains.kotlin.js.config.JSConfigurationKeys @@ -48,16 +49,10 @@ class JsKlibBackendFacade( val outputFile = JsEnvironmentConfigurator.getJsKlibArtifactPath(testServices, module.name) if (firstTimeCompilation) { - val project = testServices.compilerConfigurationProvider.getProject(module) - val errorPolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT - val hasErrors = TopDownAnalyzerFacadeForJSIR.checkForErrors(inputArtifact.sourceFiles, inputArtifact.bindingContext, errorPolicy) - serializeModuleIntoKlib( configuration[CommonConfigurationKeys.MODULE_NAME]!!, - project, configuration, configuration.irMessageLogger, - inputArtifact.bindingContext, inputArtifact.sourceFiles, klibPath = outputFile, JsEnvironmentConfigurator.getAllRecursiveLibrariesFor(module, testServices).keys.toList(), @@ -66,9 +61,10 @@ class JsKlibBackendFacade( cleanFiles = inputArtifact.icData, nopack = true, perFile = false, - containsErrorCode = hasErrors, + containsErrorCode = inputArtifact.hasErrors, abiVersion = KotlinAbiVersion.CURRENT, // TODO get from test file data - jsOutputName = null + jsOutputName = null, + inputArtifact.serializeSingleFile ) } diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsIrTest.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsIrTest.kt index e6617e80abd..fd1e85e06f0 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsIrTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsIrTest.kt @@ -6,14 +6,12 @@ package org.jetbrains.kotlin.js.test.ir import org.jetbrains.kotlin.js.test.AbstractJsBlackBoxCodegenTestBase -import org.jetbrains.kotlin.js.test.JsAdditionalSourceProvider import org.jetbrains.kotlin.js.test.JsSteppingTestAdditionalSourceProvider import org.jetbrains.kotlin.js.test.converters.JsIrBackendFacade import org.jetbrains.kotlin.js.test.converters.JsKlibBackendFacade import org.jetbrains.kotlin.js.test.converters.incremental.RecompileModuleJsIrBackendFacade import org.jetbrains.kotlin.js.test.handlers.* import org.jetbrains.kotlin.parsing.parseBoolean -import org.jetbrains.kotlin.platform.js.JsPlatforms import org.jetbrains.kotlin.test.Constructor import org.jetbrains.kotlin.test.TargetBackend import org.jetbrains.kotlin.test.backend.ir.IrBackendInput @@ -27,15 +25,9 @@ import org.jetbrains.kotlin.test.frontend.fir.FirFrontendFacade import org.jetbrains.kotlin.test.frontend.fir.FirOutputArtifact import org.jetbrains.kotlin.test.frontend.fir.handlers.* import org.jetbrains.kotlin.test.model.* -import org.jetbrains.kotlin.test.runners.AbstractKotlinCompilerWithTargetBackendTest -import org.jetbrains.kotlin.test.services.JsLibraryProvider import org.jetbrains.kotlin.test.services.MetaTestConfigurator import org.jetbrains.kotlin.test.services.TestServices -import org.jetbrains.kotlin.test.services.configuration.CommonEnvironmentConfigurator import org.jetbrains.kotlin.test.services.configuration.JsEnvironmentConfigurator -import org.jetbrains.kotlin.test.services.moduleStructure -import org.jetbrains.kotlin.test.services.sourceProviders.CoroutineHelpersSourceFilesProvider -import org.jetbrains.kotlin.test.utils.isDirectiveDefined import java.lang.Boolean.getBoolean abstract class AbstractJsIrTest( @@ -194,68 +186,46 @@ private class SkipMultiModuleTestsMetaConfigurator(testServices: TestServices) : } } -open class AbstractFirJsTest : AbstractKotlinCompilerWithTargetBackendTest(TargetBackend.JS_IR) { - private val pathToTestDir = "${JsEnvironmentConfigurator.TEST_DATA_DIR_PATH}/box/" - private val testGroupOutputDirPrefix = "box/" - - val targetFrontend = FrontendKinds.FIR - private val skipMinification: Boolean = getBoolean("kotlin.js.skipMinificationTest") - - val frontendFacade: Constructor> +abstract class AbstractFirJsTest( + pathToTestDir: String = "${JsEnvironmentConfigurator.TEST_DATA_DIR_PATH}/box/", + testGroupOutputDirPrefix: String = "box/", +) : AbstractJsBlackBoxCodegenTestBase( + FrontendKinds.FIR, TargetBackend.JS_IR, pathToTestDir, testGroupOutputDirPrefix, skipMinification = true +) { + override val frontendFacade: Constructor> get() = ::FirFrontendFacade - private val frontendToBackendConverter: Constructor> + override val frontendToBackendConverter: Constructor> get() = ::Fir2IrResultsConverter - override fun TestConfigurationBuilder.configuration() { - globalDefaults { - frontend = targetFrontend - targetPlatform = JsPlatforms.defaultJsPlatform - dependencyKind = DependencyKind.Binary - } + override val backendFacade: Constructor> + get() = ::JsKlibBackendFacade - val pathToRootOutputDir = System.getProperty("kotlin.js.test.root.out.dir") ?: error("'kotlin.js.test.root.out.dir' is not set") - defaultDirectives { - +DiagnosticsDirectives.REPORT_ONLY_EXPLICITLY_DEFINED_DEBUG_INFO - JsEnvironmentConfigurationDirectives.PATH_TO_ROOT_OUTPUT_DIR with pathToRootOutputDir - JsEnvironmentConfigurationDirectives.PATH_TO_TEST_DIR with pathToTestDir - JsEnvironmentConfigurationDirectives.TEST_GROUP_OUTPUT_DIR_PREFIX with testGroupOutputDirPrefix - +JsEnvironmentConfigurationDirectives.TYPED_ARRAYS - if (skipMinification) +JsEnvironmentConfigurationDirectives.SKIP_MINIFICATION - if (getBoolean("kotlin.js.ir.skipRegularMode")) +JsEnvironmentConfigurationDirectives.SKIP_REGULAR_MODE - +ConfigurationDirectives.WITH_STDLIB - +LanguageSettingsDirectives.ALLOW_KOTLIN_PACKAGE - } + override val afterBackendFacade: Constructor>? + get() = ::JsIrBackendFacade - useMetaTestConfigurators(::SkipMultiModuleTestsMetaConfigurator) + override val recompileFacade: Constructor> + get() = { RecompileModuleJsIrBackendFacade(it) } - forTestsNotMatching("compiler/testData/codegen/box/diagnostics/functions/tailRecursion/*") { + private fun getBoolean(s: String, default: Boolean) = System.getProperty(s)?.let { parseBoolean(it) } ?: default + + override fun configure(builder: TestConfigurationBuilder) { + super.configure(builder) + with (builder) { defaultDirectives { - DiagnosticsDirectives.DIAGNOSTICS with "-warnings" + +ConfigurationDirectives.WITH_STDLIB + +LanguageSettingsDirectives.ALLOW_KOTLIN_PACKAGE + val runIc = getBoolean("kotlin.js.ir.icMode") + if (runIc) +JsEnvironmentConfigurationDirectives.RUN_IC + if (getBoolean("kotlin.js.ir.klibMainModule")) +JsEnvironmentConfigurationDirectives.KLIB_MAIN_MODULE + if (getBoolean("kotlin.js.ir.perModule", true)) +JsEnvironmentConfigurationDirectives.PER_MODULE + if (getBoolean("kotlin.js.ir.dce", true)) +JsEnvironmentConfigurationDirectives.RUN_IR_DCE + +LanguageSettingsDirectives.ALLOW_KOTLIN_PACKAGE + -JsEnvironmentConfigurationDirectives.GENERATE_NODE_JS_RUNNER } - } - - forTestsNotMatching("compiler/testData/codegen/boxError/*") { - enableMetaInfoHandler() - } - - useConfigurators( - ::CommonEnvironmentConfigurator, - ::JsEnvironmentConfigurator, - ) - - useAdditionalSourceProviders( - ::JsAdditionalSourceProvider, - ::CoroutineHelpersSourceFilesProvider, - ) - - useAdditionalService(::JsLibraryProvider) - - facadeStep(frontendFacade) firHandlersStep { useHandlers( - ::FirDiagnosticsHandler, ::FirDumpHandler, ::FirCfgDumpHandler, ::FirCfgConsistencyHandler, @@ -263,8 +233,17 @@ open class AbstractFirJsTest : AbstractKotlinCompilerWithTargetBackendTest(Targe ) } - // There were some problems not covered by - // the FIR handlers above - facadeStep(frontendToBackendConverter) + configureJsArtifactsHandlersStep { + useHandlers( + ::JsIrRecompiledArtifactsIdentityHandler, + ) + } + + forTestsMatching("${JsEnvironmentConfigurator.TEST_DATA_DIR_PATH}/box/closure/inlineAnonymousFunctions/*") { + defaultDirectives { + +JsEnvironmentConfigurationDirectives.GENERATE_INLINE_ANONYMOUS_FUNCTIONS + } + } + } } } diff --git a/js/js.translator/testData/box/multiModule/samePackageNames.kt b/js/js.translator/testData/box/multiModule/samePackageNames.kt index f36f2ce1590..42bea4c664f 100644 --- a/js/js.translator/testData/box/multiModule/samePackageNames.kt +++ b/js/js.translator/testData/box/multiModule/samePackageNames.kt @@ -1,4 +1,3 @@ -// IGNORE_FIR // KJS_WITH_FULL_RUNTIME // EXPECTED_REACHABLE_NODES: 1304 // MODULE: lib1