[JS IR] Support function type interfaces in incremental cache infrastructure

The patch removes logic of generating extra IrFiles (fake file) into
 IrModuleFragment for the function type interfaces during klib deserialization,
 because IC infrastructure can not process files which do not exist in klib.

 Instead of adding extra IrFiles during deserialization, the empty files
 with required packages are added into Kotlin/JS stdlib physically.
 These files are used as containers for function type interface declarations.

 Since Kotlin/WASM uses the same klib loading infrastructure as Kotlin/JS,
 the the empty files are added into Kotlin/WASM stdlib as well.

 The patch also adds a check that IrModuleFagment has files only from klib.

^KT-55720 Fixed
This commit is contained in:
Alexander Korepanov
2023-02-03 12:23:48 +00:00
committed by Space Team
parent 1c3c5417ad
commit 9324cf3360
46 changed files with 574 additions and 81 deletions
@@ -9,7 +9,6 @@ import org.jetbrains.kotlin.backend.common.serialization.cityHash64
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.*
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
@@ -241,18 +240,15 @@ class CacheUpdater(
}
private fun KotlinSourceFileMutableMap<DirtyFileMetadata>.getExportedSignaturesAndAddMetadata(
jsIrLinker: JsIrLinker,
irModule: IrModuleFragment,
symbolProviders: List<FileSignatureProvider>,
libFile: KotlinLibraryFile,
dirtySrcFiles: Set<KotlinSourceFile>
): Map<IdSignature, IdSignatureSource> {
val idSignatureToFile = hashMapOf<IdSignature, IdSignatureSource>()
val moduleDeserializer = jsIrLinker.moduleDeserializer(irModule.descriptor)
val incrementalCache = getLibIncrementalCache(libFile)
for (fileDeserializer in moduleDeserializer.fileDeserializers()) {
val reachableSignatures = fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
val maybeImportedSignatures = HashSet(reachableSignatures.keys)
val implementedSymbols = collectImplementedSymbol(fileDeserializer.symbolDeserializer.deserializedSymbols)
for (fileSymbolProvider in symbolProviders) {
val maybeImportedSignatures = fileSymbolProvider.getReachableSignatures().toHashSet()
val implementedSymbols = fileSymbolProvider.getImplementedSymbols()
for ((signature, symbol) in implementedSymbols) {
var symbolCanBeExported = maybeImportedSignatures.remove(signature)
resolveFakeOverrideFunction(symbol)?.let { resolvedSignature ->
@@ -262,11 +258,11 @@ class CacheUpdater(
symbolCanBeExported = true
}
if (symbolCanBeExported) {
idSignatureToFile[signature] = IdSignatureSource(libFile, fileDeserializer.file, symbol)
idSignatureToFile[signature] = IdSignatureSource(libFile, fileSymbolProvider.irFile, symbol)
}
}
val libSrcFile = KotlinSourceFile(fileDeserializer.file)
val libSrcFile = KotlinSourceFile(fileSymbolProvider.irFile)
if (libSrcFile in dirtySrcFiles) {
val metadata = incrementalCache.fetchSourceFileFullMetadata(libSrcFile)
this[libFile, libSrcFile] = DirtyFileMetadata(maybeImportedSignatures, metadata.directDependencies)
@@ -295,16 +291,16 @@ class CacheUpdater(
}
fun rebuildDirtySourceMetadata(
jsIrLinker: JsIrLinker,
loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
loadedIr: LoadedJsIr,
dirtySrcFiles: KotlinSourceFileMap<KotlinSourceFileExports>,
): KotlinSourceFileMap<DirtyFileMetadata> {
val idSignatureToFile = hashMapOf<IdSignature, IdSignatureSource>()
val updatedMetadata = KotlinSourceFileMutableMap<DirtyFileMetadata>()
for ((lib, irModule) in loadedFragments) {
for (lib in loadedIr.loadedFragments.keys) {
val libDirtySrcFiles = dirtySrcFiles[lib]?.keys ?: emptySet()
idSignatureToFile += updatedMetadata.getExportedSignaturesAndAddMetadata(jsIrLinker, irModule, lib, libDirtySrcFiles)
val symbolProviders = loadedIr.getSignatureProvidersForLib(lib)
idSignatureToFile += updatedMetadata.getExportedSignaturesAndAddMetadata(symbolProviders, lib, libDirtySrcFiles)
}
signatureHashCalculator.addAllSignatureSymbols(idSignatureToFile)
@@ -488,20 +484,17 @@ class CacheUpdater(
}
fun updateStdlibIntrinsicDependencies(
linker: JsIrLinker,
loadedIr: LoadedJsIr,
mainModule: IrModuleFragment,
loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
dirtyFiles: Map<KotlinLibraryFile, Set<KotlinSourceFile>>
) {
val stdlibDescriptor = mainModule.descriptor.builtIns.builtInsModule
val (stdlibFile, stdlibIr) = loadedFragments.entries.find {
it.value.descriptor === stdlibDescriptor
} ?: notFoundIcError("stdlib loaded fragment")
val (stdlibFile, _) = findStdlib(mainModule, loadedIr.loadedFragments)
val stdlibDirtyFiles = dirtyFiles[stdlibFile] ?: return
val stdlibSymbolProviders = loadedIr.getSignatureProvidersForLib(stdlibFile)
val updatedMetadata = KotlinSourceFileMutableMap<DirtyFileMetadata>()
val idSignatureToFile = updatedMetadata.getExportedSignaturesAndAddMetadata(linker, stdlibIr, stdlibFile, stdlibDirtyFiles)
val idSignatureToFile = updatedMetadata.getExportedSignaturesAndAddMetadata(stdlibSymbolProviders, stdlibFile, stdlibDirtyFiles)
signatureHashCalculator.addAllSignatureSymbols(idSignatureToFile)
@@ -550,10 +543,7 @@ class CacheUpdater(
}
}
fun buildAndCommitCacheArtifacts(
jsIrLinker: JsIrLinker,
loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>
): Map<KotlinLibraryFile, IncrementalCacheArtifact> {
fun buildAndCommitCacheArtifacts(loadedIr: LoadedJsIr): Map<KotlinLibraryFile, IncrementalCacheArtifact> {
removedIncrementalCaches.forEach {
if (!it.cacheDir.deleteRecursively()) {
icError("can not delete cache directory ${it.cacheDir.absolutePath}")
@@ -562,13 +552,21 @@ class CacheUpdater(
return libraryDependencies.keys.associate { library ->
val libFile = KotlinLibraryFile(library)
val incrementalCache = getLibIncrementalCache(libFile)
val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile)
val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor)
val providers = loadedIr.getSignatureProvidersForLib(libFile)
val signatureToIndexMapping = providers.associate { KotlinSourceFile(it.irFile) to it.getSignatureToIndexMapping() }
val signatureToIndexMapping = moduleDeserializer.fileDeserializers().associate {
KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
val cacheArtifact = incrementalCache.buildIncrementalCacheArtifact(signatureToIndexMapping)
val libFragment = loadedIr.loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile)
val sourceFilesFromCache = cacheArtifact.getSourceFiles()
for (irFile in libFragment.files) {
if (KotlinSourceFile(irFile) !in sourceFilesFromCache) {
// IC doesn't support cases when extra IrFiles (which don't exist in klib) are added into IrModuleFragment
icError("file ${irFile.fileEntry.name} is absent in incremental cache and klib", libFile)
}
}
libFile to incrementalCache.buildIncrementalCacheArtifact(signatureToIndexMapping)
libFile to cacheArtifact
}
}
}
@@ -642,7 +640,7 @@ class CacheUpdater(
while (true) {
stopwatch.startNext("Dependencies ($iterations) - updating a dependency graph")
val dirtyMetadata = updater.rebuildDirtySourceMetadata(loadedIr.linker, loadedIr.loadedFragments, lastDirtyFiles)
val dirtyMetadata = updater.rebuildDirtySourceMetadata(loadedIr, lastDirtyFiles)
stopwatch.startNext("Dependencies ($iterations) - collecting files with updated exports and imports")
val filesWithModifiedExportsOrImports = updater.collectFilesWithModifiedExportsAndImports(dirtyMetadata)
@@ -672,15 +670,15 @@ class CacheUpdater(
val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModuleFragment, compilerConfiguration)
// Load declarations referenced during `context` initialization
loadedIr.linker.loadUnboundSymbols()
loadedIr.loadUnboundSymbols()
val dirtyFiles = dirtyFileExports.entries.associateTo(HashMap(dirtyFileExports.size)) { it.key to HashSet(it.value.keys) }
stopwatch.startNext("Processing IR - updating intrinsics and builtins dependencies")
updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModuleFragment, loadedIr.loadedFragments, dirtyFiles)
updater.updateStdlibIntrinsicDependencies(loadedIr, mainModuleFragment, dirtyFiles)
stopwatch.startNext("Incremental cache - building artifacts")
val incrementalCachesArtifacts = updater.buildAndCommitCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments)
val incrementalCachesArtifacts = updater.buildAndCommitCacheArtifacts(loadedIr)
stopwatch.stop()
return IrForDirtyFilesAndCompiler(incrementalCachesArtifacts, loadedIr.loadedFragments, dirtyFiles, compilerForIC)
@@ -746,9 +744,9 @@ fun rebuildCacheForDirtyFiles(
val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata })
val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, emptyList(), irFactory)
val (jsIrLinker, irModules) = jsIrLoader.loadIr(KotlinSourceFileMap<KotlinSourceFileExports>(modifiedFiles), true)
val loadedIr = jsIrLoader.loadIr(KotlinSourceFileMap<KotlinSourceFileExports>(modifiedFiles), true)
val currentIrModule = irModules[libFile] ?: notFoundIcError("loaded fragment", libFile)
val currentIrModule = loadedIr.loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile)
val dirtyIrFiles = dirtyFiles?.let {
val files = it.toSet()
currentIrModule.files.filter { irFile -> irFile.fileEntry.name in files }
@@ -764,9 +762,9 @@ fun rebuildCacheForDirtyFiles(
)
// Load declarations referenced during `context` initialization
jsIrLinker.loadUnboundSymbols()
loadedIr.loadUnboundSymbols()
val fragments = compilerWithIC.compile(irModules.values, dirtyIrFiles, mainArguments).map { it() }
val fragments = compilerWithIC.compile(loadedIr.loadedFragments.values, dirtyIrFiles, mainArguments).map { it() }
return currentIrModule to dirtyIrFiles.zip(fragments)
}
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.protobuf.CodedInputStream
import org.jetbrains.kotlin.protobuf.CodedOutputStream
import java.io.File
@@ -47,6 +48,17 @@ internal inline fun <K, V> buildMapUntil(to: Int, builderAction: MutableMap<K, V
return HashMap<K, V>(to).apply { repeat(to) { builderAction(it) } }
}
internal fun findStdlib(
mainFragment: IrModuleFragment,
allFragments: Map<KotlinLibraryFile, IrModuleFragment>
): Pair<KotlinLibraryFile, IrModuleFragment> {
val stdlibDescriptor = mainFragment.descriptor.builtIns.builtInsModule
val (stdlibFile, stdlibIr) = allFragments.entries.find {
it.value.descriptor === stdlibDescriptor
} ?: notFoundIcError("stdlib fragment")
return stdlibFile to stdlibIr
}
internal class StopwatchIC {
private var lapStart: Long = 0
private var lapDescription: String? = null
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.serialization.IrFileDeserializer
import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.symbols.IrSymbol
@@ -57,7 +58,7 @@ internal fun resolveFakeOverrideFunction(symbol: IrSymbol): IdSignature? {
}
}
internal fun collectImplementedSymbol(deserializedSymbols: Map<IdSignature, IrSymbol>): Map<IdSignature, IrSymbol> {
private fun collectImplementedSymbol(deserializedSymbols: Map<IdSignature, IrSymbol>): Map<IdSignature, IrSymbol> {
return HashMap<IdSignature, IrSymbol>(deserializedSymbols.size).apply {
for ((signature, symbol) in deserializedSymbols) {
put(signature, symbol)
@@ -98,3 +99,39 @@ internal fun collectImplementedSymbol(deserializedSymbols: Map<IdSignature, IrSy
}
}
}
internal sealed class FileSignatureProvider(val irFile: IrFile) {
abstract fun getSignatureToIndexMapping(): Map<IdSignature, Int>
abstract fun getReachableSignatures(): Set<IdSignature>
abstract fun getImplementedSymbols(): Map<IdSignature, IrSymbol>
class DeserializedFromKlib(private val fileDeserializer: IrFileDeserializer) : FileSignatureProvider(fileDeserializer.file) {
override fun getSignatureToIndexMapping(): Map<IdSignature, Int> {
return fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
}
override fun getReachableSignatures(): Set<IdSignature> {
return getSignatureToIndexMapping().keys
}
override fun getImplementedSymbols(): Map<IdSignature, IrSymbol> {
return collectImplementedSymbol(fileDeserializer.symbolDeserializer.deserializedSymbols)
}
}
class GeneratedFunctionTypeInterface(file: IrFile) : FileSignatureProvider(file) {
private val allSignatures = run {
val topLevelSymbols = buildMap {
for (declaration in irFile.declarations) {
val signature = declaration.symbol.signature ?: continue
put(signature, declaration.symbol)
}
}
collectImplementedSymbol(topLevelSymbols)
}
override fun getSignatureToIndexMapping(): Map<IdSignature, Int> = emptyMap()
override fun getReachableSignatures(): Set<IdSignature> = allSignatures.keys
override fun getImplementedSymbols(): Map<IdSignature, IrSymbol> = allSignatures
}
}
@@ -98,7 +98,7 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca
}
val fileArtifacts = klibSrcFiles.map { srcFile ->
commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
commitSourceFileMetadata(srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
}
return IncrementalCacheArtifact(cacheDir, removedSrcFiles.isNotEmpty(), fileArtifacts, library.jsOutputName)
}
@@ -200,10 +200,10 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca
}
private fun commitSourceFileMetadata(
binaryAstFile: File,
srcFile: KotlinSourceFile,
signatureToIndexMapping: Map<IdSignature, Int>
): SourceFileCacheArtifact {
val binaryAstFile = srcFile.getCacheFile(BINARY_AST_SUFFIX)
val headerCacheFile = srcFile.getCacheFile(METADATA_SUFFIX)
val sourceFileMetadata = kotlinLibrarySourceFileMetadata[srcFile]
?: return SourceFileCacheArtifact.DoNotChangeMetadata(srcFile, binaryAstFile)
@@ -54,6 +54,8 @@ internal class IncrementalCacheArtifact(
private val srcCacheActions: List<SourceFileCacheArtifact>,
private val externalModuleName: String?
) {
fun getSourceFiles() = srcCacheActions.mapTo(HashSet(srcCacheActions.size)) { it.srcFile }
fun buildModuleArtifactAndCommitCache(
moduleName: String,
rebuiltFileFragments: Map<KotlinSourceFile, JsIrProgramFragment>,
@@ -6,6 +6,8 @@
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy
import org.jetbrains.kotlin.backend.common.serialization.checkIsFunctionInterface
import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData
import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnboundSymbols
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
import org.jetbrains.kotlin.config.CompilerConfiguration
@@ -14,11 +16,13 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
import org.jetbrains.kotlin.ir.backend.js.FunctionTypeInterfacePackages
import org.jetbrains.kotlin.ir.backend.js.JsFactories
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
import org.jetbrains.kotlin.ir.declarations.IrFactory
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.descriptors.IrDescriptorBasedFunctionFactory
import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
import org.jetbrains.kotlin.ir.util.SymbolTable
import org.jetbrains.kotlin.ir.util.irMessageLogger
@@ -30,10 +34,43 @@ import org.jetbrains.kotlin.psi2ir.descriptors.IrBuiltInsOverDescriptors
import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
import org.jetbrains.kotlin.storage.LockBasedStorageManager
internal fun JsIrLinker.loadUnboundSymbols() {
ExternalDependenciesGenerator(symbolTable, listOf(this)).generateUnboundSymbolsAsDependencies()
postProcess()
checkNoUnboundSymbols(symbolTable, "at the end of IR linkage process")
internal data class LoadedJsIr(
val loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
private val linker: JsIrLinker,
private val functionTypeInterfacePackages: FunctionTypeInterfacePackages
) {
private val signatureProvidersImpl = hashMapOf<KotlinLibraryFile, List<FileSignatureProvider>>()
private fun collectSignatureProviders(irModule: IrModuleFragment): List<FileSignatureProvider> {
val moduleDeserializer = linker.moduleDeserializer(irModule.descriptor)
val deserializers = moduleDeserializer.fileDeserializers()
val providers = ArrayList<FileSignatureProvider>(deserializers.size)
for (fileDeserializer in deserializers) {
val irFile = fileDeserializer.file
if (functionTypeInterfacePackages.isFunctionTypeInterfacePackageFile(irFile)) {
providers += FileSignatureProvider.GeneratedFunctionTypeInterface(irFile)
} else {
providers += FileSignatureProvider.DeserializedFromKlib(fileDeserializer)
}
}
return providers
}
fun getSignatureProvidersForLib(lib: KotlinLibraryFile): List<FileSignatureProvider> {
return signatureProvidersImpl.getOrPut(lib) {
val irFragment = loadedFragments[lib] ?: notFoundIcError("loaded fragment", lib)
collectSignatureProviders(irFragment)
}
}
fun loadUnboundSymbols() {
signatureProvidersImpl.clear()
ExternalDependenciesGenerator(linker.symbolTable, listOf(linker)).generateUnboundSymbolsAsDependencies()
linker.postProcess()
linker.checkNoUnboundSymbols(linker.symbolTable, "at the end of IR linkage process")
}
}
internal class JsIrLinkerLoader(
@@ -45,14 +82,34 @@ internal class JsIrLinkerLoader(
private val mainLibrary = dependencyGraph.keys.lastOrNull() ?: notFoundIcError("main library")
@OptIn(ObsoleteDescriptorBasedAPI::class)
private fun createLinker(loadedModules: Map<ModuleDescriptor, KotlinLibrary>): JsIrLinker {
private class LinkerContext(
val symbolTable: SymbolTable,
val typeTranslator: TypeTranslatorImpl,
val irBuiltIns: IrBuiltInsOverDescriptors,
val linker: JsIrLinker
) {
val functionTypeInterfacePackages = FunctionTypeInterfacePackages()
fun loadFunctionInterfacesIntoStdlib(stdlibModule: IrModuleFragment) {
irBuiltIns.functionFactory = IrDescriptorBasedFunctionFactory(
irBuiltIns,
symbolTable,
typeTranslator,
functionTypeInterfacePackages.makePackageAccessor(stdlibModule),
true
)
}
}
@OptIn(ObsoleteDescriptorBasedAPI::class)
private fun createLinker(loadedModules: Map<ModuleDescriptor, KotlinLibrary>): LinkerContext {
val signaturer = IdSignatureDescriptor(JsManglerDesc)
val symbolTable = SymbolTable(signaturer, irFactory)
val moduleDescriptor = loadedModules.keys.last()
val typeTranslator = TypeTranslatorImpl(symbolTable, compilerConfiguration.languageVersionSettings, moduleDescriptor)
val irBuiltIns = IrBuiltInsOverDescriptors(moduleDescriptor.builtIns, typeTranslator, symbolTable)
val partialLinkageEnabled = compilerConfiguration[JSConfigurationKeys.PARTIAL_LINKAGE] ?: false
return JsIrLinker(
val linker = JsIrLinker(
currentModule = null,
messageLogger = compilerConfiguration.irMessageLogger,
builtIns = irBuiltIns,
@@ -61,6 +118,7 @@ internal class JsIrLinkerLoader(
translationPluginContext = null,
friendModules = mapOf(mainLibrary.uniqueName to mainModuleFriends.map { it.uniqueName })
)
return LinkerContext(symbolTable, typeTranslator, irBuiltIns, linker)
}
private fun loadModules(): Map<ModuleDescriptor, KotlinLibrary> {
@@ -97,11 +155,9 @@ internal class JsIrLinkerLoader(
.toMap()
}
data class LoadedJsIr(val linker: JsIrLinker, val loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>)
fun loadIr(modifiedFiles: KotlinSourceFileMap<KotlinSourceFileExports>, loadAllIr: Boolean = false): LoadedJsIr {
val loadedModules = loadModules()
val jsIrLinker = createLinker(loadedModules)
val linkerContext = createLinker(loadedModules)
val irModules = loadedModules.entries.associate { (descriptor, module) ->
val libraryFile = KotlinLibraryFile(module)
@@ -111,7 +167,7 @@ internal class JsIrLinkerLoader(
else -> DeserializationStrategy.EXPLICITLY_EXPORTED
}
val modified = modifiedFiles[libraryFile] ?: emptyMap()
libraryFile to jsIrLinker.deserializeIrModuleHeader(descriptor, module, {
libraryFile to linkerContext.linker.deserializeIrModuleHeader(descriptor, module, {
when (KotlinSourceFile(it)) {
in modified -> modifiedStrategy
else -> DeserializationStrategy.WITH_INLINE_BODIES
@@ -119,15 +175,22 @@ internal class JsIrLinkerLoader(
})
}
jsIrLinker.init(null, emptyList())
val mainLibraryFile = KotlinLibraryFile(mainLibrary)
val mainFragment = irModules[mainLibraryFile] ?: notFoundIcError("main module fragment", mainLibraryFile)
val (_, stdlibFragment) = findStdlib(mainFragment, irModules)
linkerContext.loadFunctionInterfacesIntoStdlib(stdlibFragment)
linkerContext.linker.init(null, emptyList())
if (!loadAllIr) {
for ((loadingLibFile, loadingSrcFiles) in modifiedFiles) {
val loadingIrModule = irModules[loadingLibFile] ?: notFoundIcError("loading fragment", loadingLibFile)
val moduleDeserializer = jsIrLinker.moduleDeserializer(loadingIrModule.descriptor)
val moduleDeserializer = linkerContext.linker.moduleDeserializer(loadingIrModule.descriptor)
for (loadingSrcFileSignatures in loadingSrcFiles.values) {
for (loadingSignature in loadingSrcFileSignatures.getExportedSignatures()) {
if (loadingSignature in moduleDeserializer) {
if (checkIsFunctionInterface(loadingSignature)) {
moduleDeserializer.tryDeserializeIrSymbol(loadingSignature, BinarySymbolData.SymbolKind.CLASS_SYMBOL)
} else if (loadingSignature in moduleDeserializer) {
moduleDeserializer.addModuleReachableTopLevel(loadingSignature)
}
}
@@ -135,7 +198,8 @@ internal class JsIrLinkerLoader(
}
}
jsIrLinker.loadUnboundSymbols()
return LoadedJsIr(jsIrLinker, irModules)
val loadedIr = LoadedJsIr(irModules, linkerContext.linker, linkerContext.functionTypeInterfacePackages)
loadedIr.loadUnboundSymbols()
return loadedIr
}
}
@@ -107,22 +107,21 @@ internal object KotlinSourceFileMetadataNotExist : KotlinSourceFileMetadata() {
override val directDependencies = KotlinSourceFileMap<Map<IdSignature, ICHash>>(emptyMap())
}
internal class DirtyFileExports(
override val inverseDependencies: KotlinSourceFileMutableMap<Set<IdSignature>> = KotlinSourceFileMutableMap()
) : KotlinSourceFileExports() {
override fun getExportedSignatures(): Set<IdSignature> = allExportedSignatures
internal class DirtyFileExports : KotlinSourceFileExports() {
val allExportedSignatures = hashSetOf<IdSignature>()
override val inverseDependencies: KotlinSourceFileMutableMap<Set<IdSignature>> = KotlinSourceFileMutableMap()
override fun getExportedSignatures(): Set<IdSignature> = allExportedSignatures
}
internal class DirtyFileMetadata(
val maybeImportedSignatures: Collection<IdSignature>,
val oldDirectDependencies: KotlinSourceFileMap<*>,
override val inverseDependencies: KotlinSourceFileMutableMap<MutableSet<IdSignature>> = KotlinSourceFileMutableMap(),
override val directDependencies: KotlinSourceFileMutableMap<MutableMap<IdSignature, ICHash>> = KotlinSourceFileMutableMap(),
val oldDirectDependencies: KotlinSourceFileMap<*>
) : KotlinSourceFileMetadata() {
override val inverseDependencies: KotlinSourceFileMutableMap<MutableSet<IdSignature>> = KotlinSourceFileMutableMap()
override val directDependencies: KotlinSourceFileMutableMap<MutableMap<IdSignature, ICHash>> = KotlinSourceFileMutableMap()
fun addInverseDependency(lib: KotlinLibraryFile, src: KotlinSourceFile, signature: IdSignature) =
when (val signatures = inverseDependencies[lib, src]) {
null -> inverseDependencies[lib, src] = hashSetOf(signature)
@@ -10,12 +10,14 @@ import java.util.regex.Pattern
internal val functionPattern = Pattern.compile("^K?(Suspend)?Function\\d+$")
internal val functionalPackages = listOf("kotlin", "kotlin.coroutines", "kotlin.reflect")
internal val functionTypeInterfacePackages = listOf("kotlin", "kotlin.coroutines", "kotlin.reflect")
fun checkIsFunctionTypeInterfacePackageFqName(fqName: String) = fqName in functionTypeInterfacePackages
fun checkIsFunctionInterface(idSig: IdSignature?): Boolean {
val publicSig = idSig?.asPublic()
return publicSig != null &&
publicSig.packageFqName in functionalPackages &&
checkIsFunctionTypeInterfacePackageFqName(publicSig.packageFqName) &&
publicSig.declarationFqName.isNotEmpty() &&
functionPattern.matcher(publicSig.firstNameSegment).find()
}
@@ -0,0 +1,46 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js
import org.jetbrains.kotlin.backend.common.serialization.checkIsFunctionTypeInterfacePackageFqName
import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
class FunctionTypeInterfacePackages {
companion object {
private const val FUNCTION_TYPE_INTERFACE_DIR = "function-type-interface"
}
private val functionTypeInterfacePackageFiles = hashSetOf<IrFile>()
// Function type interfaces are not declared in a standard library, and they are generated on flight during a klib deserialization.
// The accessor allows finding a package for storing the generated function type interfaces.
// The optimization reduces the size of a standard library klib.
// Here are some numbers:
// 255 interfaces in one package increase the size of uncompressed klib up to 3.5+MB;
// 512 interfaces in one package increase the size of uncompressed klib up to 14.5+MB!
// We need 3 packages.
fun makePackageAccessor(stdlibModule: IrModuleFragment) = { packageFragmentDescriptor: PackageFragmentDescriptor ->
val packageFqName = packageFragmentDescriptor.fqName.toString()
check(checkIsFunctionTypeInterfacePackageFqName(packageFqName)) { "unexpected function type interface package $packageFqName" }
val fileWithRequiredPackage = "${packageFqName.replace('.', '-')}-package.kt"
val packageFile = stdlibModule.files.singleOrNull {
// Do not check by name "$FUNCTION_TYPE_INTERFACE_DIR/$fileWithRequiredPackage" because the path separator depends on OS
it.fileEntry.name.endsWith(fileWithRequiredPackage) && it.fileEntry.name.contains(FUNCTION_TYPE_INTERFACE_DIR)
} ?: error("can not find a functional interface file for $packageFqName package")
check(packageFragmentDescriptor.fqName == packageFile.fqName) {
"unexpected package in file ${packageFile.fileEntry.name}; expected $packageFqName, got ${packageFile.fqName}"
}
functionTypeInterfacePackageFiles += packageFile
packageFile
}
fun isFunctionTypeInterfacePackageFile(file: IrFile) = file in functionTypeInterfacePackageFiles
}
@@ -29,7 +29,6 @@ import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
@@ -38,7 +37,6 @@ import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.*
import org.jetbrains.kotlin.ir.declarations.IrFactory
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.declarations.impl.IrFileImpl
import org.jetbrains.kotlin.ir.descriptors.IrDescriptorBasedFunctionFactory
import org.jetbrains.kotlin.ir.linkage.IrDeserializer
import org.jetbrains.kotlin.ir.symbols.IrSymbol
@@ -71,6 +69,7 @@ import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.util.DummyLogger
import org.jetbrains.kotlin.util.Logger
import org.jetbrains.kotlin.utils.DFS
import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
import java.io.File
val KotlinLibrary.moduleName: String
@@ -184,14 +183,6 @@ fun deserializeDependencies(
}
}
fun getFunctionFactoryCallback(stdlibModule: IrModuleFragment) = { packageFragmentDescriptor: PackageFragmentDescriptor ->
IrFileImpl(
NaiveSourceBasedFileEntryImpl("${packageFragmentDescriptor.fqName}-[K][Suspend]Functions"),
packageFragmentDescriptor,
stdlibModule
).also { stdlibModule.files += it }
}
fun loadIr(
depsDescriptors: ModulesStructure,
irFactory: IrFactory,
@@ -287,7 +278,9 @@ fun getIrModuleInfoForKlib(
irBuiltIns,
symbolTable,
typeTranslator,
if (loadFunctionInterfacesIntoStdlib) getFunctionFactoryCallback(deserializedModuleFragments.first()) else null,
loadFunctionInterfacesIntoStdlib.ifTrue {
FunctionTypeInterfacePackages().makePackageAccessor(deserializedModuleFragments.first())
},
true
)
@@ -345,7 +338,9 @@ fun getIrModuleInfoForSourceFiles(
irBuiltIns,
symbolTable,
psi2IrContext.typeTranslator,
if (loadFunctionInterfacesIntoStdlib) getFunctionFactoryCallback(deserializedModuleFragments.first()) else null,
loadFunctionInterfacesIntoStdlib.ifTrue {
FunctionTypeInterfacePackages().makePackageAccessor(deserializedModuleFragments.first())
},
true
)
@@ -170,6 +170,11 @@ public class JsIrES6InvalidationTestGenerated extends AbstractJsIrES6Invalidatio
runTest("js/js.translator/testData/incremental/invalidation/functionSignature/");
}
@TestMetadata("functionalInterface")
public void testFunctionalInterface() throws Exception {
runTest("js/js.translator/testData/incremental/invalidation/functionalInterface/");
}
@TestMetadata("genericFunctions")
public void testGenericFunctions() throws Exception {
runTest("js/js.translator/testData/incremental/invalidation/genericFunctions/");
@@ -170,6 +170,11 @@ public class JsIrInvalidationTestGenerated extends AbstractJsIrInvalidationTest
runTest("js/js.translator/testData/incremental/invalidation/functionSignature/");
}
@TestMetadata("functionalInterface")
public void testFunctionalInterface() throws Exception {
runTest("js/js.translator/testData/incremental/invalidation/functionalInterface/");
}
@TestMetadata("genericFunctions")
public void testGenericFunctions() throws Exception {
runTest("js/js.translator/testData/incremental/invalidation/genericFunctions/");
@@ -861,6 +861,12 @@ public class BoxJsTestGenerated extends AbstractBoxJsTest {
public void testSuspendFunctionIsAs() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionIsAs.kt");
}
@Test
@TestMetadata("suspendFunctionalInterface.kt")
public void testSuspendFunctionalInterface() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionalInterface.kt");
}
}
@Nested
@@ -925,6 +925,12 @@ public class FirJsBoxTestGenerated extends AbstractFirJsBoxTest {
public void testSuspendFunctionIsAs() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionIsAs.kt");
}
@Test
@TestMetadata("suspendFunctionalInterface.kt")
public void testSuspendFunctionalInterface() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionalInterface.kt");
}
}
@Nested
@@ -925,6 +925,12 @@ public class IrBoxJsES6TestGenerated extends AbstractIrBoxJsES6Test {
public void testSuspendFunctionIsAs() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionIsAs.kt");
}
@Test
@TestMetadata("suspendFunctionalInterface.kt")
public void testSuspendFunctionalInterface() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionalInterface.kt");
}
}
@Nested
@@ -925,6 +925,12 @@ public class IrBoxJsTestGenerated extends AbstractIrBoxJsTest {
public void testSuspendFunctionIsAs() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionIsAs.kt");
}
@Test
@TestMetadata("suspendFunctionalInterface.kt")
public void testSuspendFunctionalInterface() throws Exception {
runTest("js/js.translator/testData/box/coroutines/suspendFunctionalInterface.kt");
}
}
@Nested
@@ -0,0 +1,30 @@
// IGNORE_BACKEND: JS
inline fun <reified T> handle(s: T): String {
return "${T::class}"
}
fun check(got: String, expected: String): String? {
if (got != expected) {
return "Failed; expected $expected, got $got"
}
return null
}
fun box(): String {
val s0: suspend () -> Unit = {}
check(handle(s0), "class SuspendFunction0")?.let { return it }
val s1: suspend (String) -> Unit = {}
check(handle(s1), "class SuspendFunction1")?.let { return it }
val s7: suspend (Any, Any, Any, Any, Any, Any, Any) -> Unit = { _, _, _, _, _, _, _ -> }
check(handle(s7), "class SuspendFunction7")?.let { return it }
val s15: suspend (Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) -> Unit
= { _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ -> }
check(handle(s15), "class SuspendFunction15")?.let { return it }
return "OK"
}
@@ -0,0 +1,3 @@
inline fun <reified T> getTypeName(s: T): String {
return "${T::class}"
}
@@ -0,0 +1,3 @@
inline fun <reified T> getTypeName(s: T): String {
return "_"
}
@@ -0,0 +1,3 @@
inline fun <reified T> getTypeName(s: T): String {
return "_${T::class}"
}
@@ -0,0 +1,17 @@
STEP 0:
modifications:
U : l1.0.kt -> l1.kt
added file: l1.kt
STEP 1..8:
STEP 9:
modifications:
U : l1.9.kt -> l1.kt
modified ir: l1.kt
STEP 10:
STEP 11:
modifications:
U : l1.11.kt -> l1.kt
modified ir: l1.kt
STEP 12:
updated exports: l1.kt
STEP 13..15:
@@ -0,0 +1,4 @@
fun getString(): String {
val s: suspend () -> Unit = {}
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: suspend (Any) -> Unit = { _ -> }
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: (Int, Int, Int, Int, Int, Int, Int, Int, Int) -> Unit = { _, _, _, _, _, _, _, _, _ -> }
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: (Int, Int, Int, Int, Int, Int, Int, Int, Int) -> Unit = { _, _, _, _, _, _, _, _, _ -> }
return "${s::class}"
}
@@ -0,0 +1,5 @@
fun getString(): String {
val s1: (Int, Int, Int, Int, Int, Int, Int, Int, Int) -> Unit = { _, _, _, _, _, _, _, _, _ -> }
val s2: (Int, Int, Int, Int, Int, Int, Int, Int, Int) -> Unit = { _, _, _, _, _, _, _, _, _ -> }
return "${s1::class.hashCode() == s2::class.hashCode()}"
}
@@ -0,0 +1,5 @@
fun getString(): String {
val s1: (Int, Int, Int, Int, Int, Int, Int, Int, Int) -> Unit = { _, _, _, _, _, _, _, _, _ -> }
val s2: (Int, Int, Int, Int) -> Unit = { _, _, _, _ -> }
return "${s1::class == s2::class}"
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: suspend (Any, Any, Any, Any, Any) -> Unit = { _, _, _, _, _ -> }
return "${s::class}"
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: suspend (Any, Any, Any, Any, Any, Any, Any) -> Unit = { _, _, _, _, _, _, _ -> }
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: (Any, Any, Any) -> Unit = { _, _, _ -> }
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: (Any, Any, Any, Any, Any, Any) -> Unit = { _, _, _, _, _, _ -> }
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: suspend (Any, Any, Any, Any, Any, Any) -> Unit = { _, _, _, _, _, _ -> }
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: Any = 123
return getTypeName(s)
}
@@ -0,0 +1,4 @@
fun getString(): String {
val s: suspend (Int, Int, Int, Int, Int, Int, Int, Int) -> Unit = { _, _, _, _, _, _, _, _ -> }
return getTypeName(s)
}
@@ -0,0 +1,3 @@
fun doTest(): String {
return getString()
}
@@ -0,0 +1,3 @@
fun doTest(): String {
return getString() + "_"
}
@@ -0,0 +1,77 @@
STEP 0:
dependencies: lib1
modifications:
U : l2a.0.kt -> l2a.kt
U : l2b.0.kt -> l2b.kt
added file: l2a.kt, l2b.kt
STEP 1:
dependencies: lib1
modifications:
U : l2a.1.kt -> l2a.kt
modified ir: l2a.kt
STEP 2:
dependencies: lib1
modifications:
U : l2a.2.kt -> l2a.kt
modified ir: l2a.kt
STEP 3:
dependencies: lib1
modifications:
U : l2a.3.kt -> l2a.kt
modified ir: l2a.kt
STEP 4:
dependencies: lib1
modifications:
U : l2a.4.kt -> l2a.kt
modified ir: l2a.kt
STEP 5:
dependencies: lib1
modifications:
U : l2b.5.kt -> l2b.kt
modified ir: l2b.kt
STEP 6:
dependencies: lib1
modifications:
U : l2a.6.kt -> l2a.kt
modified ir: l2a.kt
STEP 7:
dependencies: lib1
modifications:
U : l2a.7.kt -> l2a.kt
modified ir: l2a.kt
STEP 8:
dependencies: lib1
modifications:
U : l2a.8.kt -> l2a.kt
modified ir: l2a.kt
STEP 9:
dependencies: lib1
updated imports: l2a.kt
STEP 10:
dependencies: lib1
modifications:
U : l2a.10.kt -> l2a.kt
modified ir: l2a.kt
STEP 11:
dependencies: lib1
updated imports: l2a.kt
STEP 12:
dependencies: lib1
modifications:
U : l2a.12.kt -> l2a.kt
modified ir: l2a.kt
STEP 13:
dependencies: lib1
modifications:
U : l2a.13.kt -> l2a.kt
modified ir: l2a.kt
STEP 14:
dependencies: lib1
modifications:
U : l2a.14.kt -> l2a.kt
modified ir: l2a.kt
STEP 15:
dependencies: lib1
modifications:
U : l2a.15.kt -> l2a.kt
modified ir: l2a.kt
@@ -0,0 +1,26 @@
fun box(stepId: Int): String {
val s = doTest()
when (stepId) {
0 -> if (s == "class SuspendFunction0") return "OK"
1 -> if (s == "class SuspendFunction1") return "OK"
2 -> if (s == "class SuspendFunction7") return "OK"
3 -> if (s == "class Function3") return "OK"
4 -> if (s == "class Function6") return "OK"
5 -> if (s == "class Function6_") return "OK"
6 -> if (s == "class SuspendFunction6_") return "OK"
7 -> if (s == "class Any_") return "OK"
8 -> if (s == "class SuspendFunction8_") return "OK"
9 -> if (s == "_class SuspendFunction8_") return "OK"
10 -> if (s == "_class Function9_") return "OK"
11 -> if (s == "__") return "OK"
12 -> if (s == "class Function9_") return "OK"
13 -> if (s == "true_") return "OK"
14 -> if (s == "false_") return "OK"
// TODO: I would expect, that it should be SuspendFunction5_,
// but it seems a feature of Kotlin/JS runtime.
// In Kotlin/JVM it also gives different strings.
15 -> if (s == "class Function6_") return "OK"
else -> return "Unknown"
}
return "Fail; got $s"
}
@@ -0,0 +1,5 @@
STEP 0:
dependencies: lib1, lib2
added file: m.kt
STEP 1..15:
dependencies: lib1, lib2
@@ -0,0 +1,20 @@
MODULES: lib1, lib2, main
STEP 0:
libs: lib1, lib2, main
dirty js: lib1, lib2, main
STEP 1..8:
libs: lib1, lib2, main
dirty js: lib2
STEP 9:
libs: lib1, lib2, main
dirty js: lib1, lib2
STEP 10:
libs: lib1, lib2, main
dirty js: lib2
STEP 11..12:
libs: lib1, lib2, main
dirty js: lib1, lib2
STEP 13..15:
libs: lib1, lib2, main
dirty js: lib2
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin.coroutines
// Do not remove this file.
// During klib deserialization the compiler uses this file as a place for declaring function type interfaces.
// See FunctionTypeInterfacePackages.kt
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin
// Do not remove this file.
// During klib deserialization the compiler uses this file as a place for declaring function type interfaces.
// See FunctionTypeInterfacePackages.kt
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin.reflect
// Do not remove this file.
// During klib deserialization the compiler uses this file as a place for declaring function type interfaces.
// See FunctionTypeInterfacePackages.kt
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin.coroutines
// Do not remove this file.
// During klib deserialization the compiler uses this file as a place for declaring function type interfaces.
// See FunctionTypeInterfacePackages.kt
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin
// Do not remove this file.
// During klib deserialization the compiler uses this file as a place for declaring function type interfaces.
// See FunctionTypeInterfacePackages.kt
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin.reflect
// Do not remove this file.
// During klib deserialization the compiler uses this file as a place for declaring function type interfaces.
// See FunctionTypeInterfacePackages.kt