diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt index d196dbf7853..78be8c451c6 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt @@ -205,14 +205,6 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { var generateDebugTrampolineString: String? = null - @Argument( - value = MAKE_CACHE, - valueDescription = "", - description = "Path of the library to be compiled to cache", - delimiter = "" - ) - var librariesToCache: Array? = null - @Argument( value = ADD_CACHE, valueDescription = "", @@ -221,6 +213,14 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { ) var libraryToAddToCache: String? = null + @Argument( + value = "-Xfile-to-cache", + valueDescription = "", + description = "Path to file to cache", + delimiter = "" + ) + var fileToCache: String? = null + @Argument(value = "-Xexport-kdoc", description = "Export KDoc in framework header") var exportKDoc: Boolean = false @@ -415,7 +415,6 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { const val STATIC_FRAMEWORK_FLAG = "-Xstatic-framework" const val INCLUDE_ARG = "-Xinclude" const val CACHED_LIBRARY = "-Xcached-library" - const val MAKE_CACHE = "-Xmake-cache" const val ADD_CACHE = "-Xadd-cache" const val SHORT_MODULE_NAME_ARG = "-Xshort-module-name" } diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/overrides/FakeOverrides.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/overrides/FakeOverrides.kt index 0cc70e3a26a..79b2a473ebf 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/overrides/FakeOverrides.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/overrides/FakeOverrides.kt @@ -87,9 +87,8 @@ class FakeOverrideBuilder( fakeOverrideCandidates[clazz] = compatibilityMode } - private fun buildFakeOverrideChainsForClass(clazz: IrClass, compatibilityMode: CompatibilityMode) { - if (haveFakeOverrides.contains(clazz)) return - if (!platformSpecificClassFilter.needToConstructFakeOverrides(clazz)) return + private fun buildFakeOverrideChainsForClass(clazz: IrClass, compatibilityMode: CompatibilityMode): Boolean { + if (haveFakeOverrides.contains(clazz)) return true val superTypes = clazz.superTypes @@ -99,15 +98,18 @@ class FakeOverrideBuilder( superClasses.forEach { superClass -> val mode = fakeOverrideCandidates[superClass] ?: compatibilityMode - buildFakeOverrideChainsForClass(superClass, mode) - haveFakeOverrides.add(superClass) + if (buildFakeOverrideChainsForClass(superClass, mode)) + haveFakeOverrides.add(superClass) } + if (!platformSpecificClassFilter.needToConstructFakeOverrides(clazz)) return false + fakeOverrideDeclarationTable.run { inFile(clazz.fileOrNull) { irOverridingUtil.buildFakeOverridesForClass(clazz, compatibilityMode.oldSignatures) } } + return true } override fun linkFunctionFakeOverride(declaration: IrFakeOverrideFunction, compatibilityMode: Boolean) { diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt index 54a72ca2f89..b45ace607a0 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt @@ -183,7 +183,7 @@ class IrKlibBytesSource(private val klib: IrLibrary, private val fileIndex: Int) override fun debugInfo(index: Int): ByteArray? = klib.debugInfo(index, fileIndex) } -internal fun IrLibraryFile.deserializeFqName(fqn: List): String = +fun IrLibraryFile.deserializeFqName(fqn: List): String = fqn.joinToString(".", transform = ::string) fun IrLibraryFile.createFile(module: IrModuleFragment, fileProto: ProtoFile): IrFile { diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 002e3565816..638599feae9 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -117,7 +117,7 @@ class K2Native : CLICompiler() { val K2NativeCompilerArguments.isUsefulWithoutFreeArgs: Boolean get() = listTargets || listPhases || checkDependencies || !includes.isNullOrEmpty() || - !librariesToCache.isNullOrEmpty() || libraryToAddToCache != null || !exportedLibraries.isNullOrEmpty() + libraryToAddToCache != null || !exportedLibraries.isNullOrEmpty() fun Array?.toNonNullList(): List { return this?.asList() ?: listOf() @@ -297,7 +297,6 @@ class K2Native : CLICompiler() { put(OBJC_GENERICS, !arguments.noObjcGenerics) put(DEBUG_PREFIX_MAP, parseDebugPrefixMap(arguments, configuration)) - put(LIBRARIES_TO_CACHE, parseLibrariesToCache(arguments, configuration, outputKind)) val libraryToAddToCache = parseLibraryToAddToCache(arguments, configuration, outputKind) if (libraryToAddToCache != null && !arguments.outputName.isNullOrEmpty()) configuration.report(ERROR, "${K2NativeCompilerArguments.ADD_CACHE} already implicitly sets output file name") @@ -305,6 +304,10 @@ class K2Native : CLICompiler() { libraryToAddToCache?.let { put(LIBRARY_TO_ADD_TO_CACHE, it) } put(CACHE_DIRECTORIES, cacheDirectories) put(CACHED_LIBRARIES, parseCachedLibraries(arguments, configuration)) + val fileToCache = arguments.fileToCache + if (outputKind == CompilerOutputKind.PRELIMINARY_CACHE && fileToCache == null) + configuration.report(ERROR, "preliminary_cache only supported for per-file caches") + fileToCache?.let { put(FILE_TO_CACHE, it) } parseShortModuleName(arguments, configuration, outputKind)?.let { put(SHORT_MODULE_NAME, it) @@ -521,27 +524,6 @@ private fun parseCachedLibraries( } }.toMap() -private fun parseLibrariesToCache( - arguments: K2NativeCompilerArguments, - configuration: CompilerConfiguration, - outputKind: CompilerOutputKind -): List { - val input = arguments.librariesToCache?.asList().orEmpty() - - return if (input.isNotEmpty() && !outputKind.isCache) { - configuration.report(ERROR, "${K2NativeCompilerArguments.MAKE_CACHE} can't be used when not producing cache") - emptyList() - } else if (input.isNotEmpty() && !arguments.libraryToAddToCache.isNullOrEmpty()) { - configuration.report( - ERROR, - "supplied both ${K2NativeCompilerArguments.MAKE_CACHE} and ${K2NativeCompilerArguments.ADD_CACHE} options" - ) - emptyList() - } else { - input - } -} - private fun parseLibraryToAddToCache( arguments: K2NativeCompilerArguments, configuration: CompilerConfiguration, diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt index da4fda82fa0..0beef4abafe 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt @@ -43,12 +43,6 @@ const val INCLUDE_ARG = MovedK2NativeCompilerArguments.INCLUDE_ARG ) const val CACHED_LIBRARY = MovedK2NativeCompilerArguments.CACHED_LIBRARY -@Deprecated( - "Moved to new 'org.jetbrains.kotlin.cli.common.arguments' package", - ReplaceWith("MAKE_CACHE", "org.jetbrains.kotlin.cli.common.arguments.MAKE_CACHE") -) -const val MAKE_CACHE = MovedK2NativeCompilerArguments.MAKE_CACHE - @Deprecated( "Moved to new 'org.jetbrains.kotlin.cli.common.arguments' package", ReplaceWith("ADD_CACHE", "org.jetbrains.kotlin.cli.common.arguments.ADD_CACHE") diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt index 2795fa9e6e8..d19146c93bd 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt @@ -173,7 +173,7 @@ internal fun initializeCachedBoxes(context: Context) { val rangeStart = "${cache.name}_RANGE_FROM" val rangeEnd = "${cache.name}_RANGE_TO" initCache(cache, context, cacheName, rangeStart, rangeEnd, - declareOnly = !context.producedLlvmModuleContainsStdlib + declareOnly = !context.shouldDefineCachedBoxes ).also { context.llvm.boxCacheGlobals[cache] = it } } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt index 7087a09f007..17adb0b0634 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt @@ -5,6 +5,10 @@ package org.jetbrains.kotlin.backend.konan +import org.jetbrains.kotlin.backend.common.serialization.IrKlibBytesSource +import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes +import org.jetbrains.kotlin.backend.common.serialization.codedInputStream +import org.jetbrains.kotlin.backend.common.serialization.deserializeFqName import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.konan.file.File @@ -12,16 +16,31 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite +import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile sealed class CacheDeserializationStrategy { abstract fun contains(filePath: String): Boolean + abstract fun contains(fqName: FqName, fileName: String): Boolean object Nothing : CacheDeserializationStrategy() { override fun contains(filePath: String) = false + override fun contains(fqName: FqName, fileName: String) = false } object WholeModule : CacheDeserializationStrategy() { override fun contains(filePath: String) = true + override fun contains(fqName: FqName, fileName: String) = true + } + + class SingleFile(val filePath: String) : CacheDeserializationStrategy() { + override fun contains(filePath: String) = filePath == this.filePath + + lateinit var fqName: String + + override fun contains(fqName: FqName, fileName: String) = + fqName.asString() == this.fqName && File(filePath).name == fileName } } @@ -32,7 +51,7 @@ class CacheSupport( resolvedLibraries: KotlinLibraryResolveResult, ignoreCacheReason: String?, target: KonanTarget, - produce: CompilerOutputKind + val produce: CompilerOutputKind ) { private val allLibraries = resolvedLibraries.getFullList() @@ -46,14 +65,34 @@ class CacheSupport( } internal fun tryGetImplicitOutput(): String? { - val libraryToAddToCache = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE) ?: return null + val libraryToCache = libraryToCache ?: return null // Put the resulting library in the first cache directory. val cacheDirectory = implicitCacheDirectories.firstOrNull() ?: return null - val libraryToAddToCacheFile = File(libraryToAddToCache) - val library = allLibraries.single { it.libraryFile == libraryToAddToCacheFile } - return cacheDirectory.child(CachedLibraries.getCachedLibraryName(library)).absolutePath - } + val singleFileStrategy = libraryToCache.strategy as? CacheDeserializationStrategy.SingleFile + val baseLibraryCacheDirectory = cacheDirectory.child( + if (singleFileStrategy == null) + CachedLibraries.getCachedLibraryName(libraryToCache.klib) + else + CachedLibraries.getPerFileCachedLibraryName(libraryToCache.klib) + ) + if (singleFileStrategy == null) + return baseLibraryCacheDirectory.absolutePath + val fileToCache = singleFileStrategy.filePath + val fileProtos = Array(libraryToCache.klib.fileCount()) { + ProtoFile.parseFrom(libraryToCache.klib.file(it).codedInputStream, ExtensionRegistryLite.newInstance()) + } + val fileIndex = fileProtos.indexOfFirst { it.fileEntry.name == fileToCache } + require(fileIndex >= 0) { "No file found in klib with path $fileToCache" } + val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(libraryToCache.klib, fileIndex)) + val fqName = fileReader.deserializeFqName(fileProtos[fileIndex].fqNameList) + singleFileStrategy.fqName = fqName + val fileCacheDirectory = baseLibraryCacheDirectory.child(cacheFileId(fqName, fileToCache)) + val contentDirName = if (produce == CompilerOutputKind.PRELIMINARY_CACHE) + CachedLibraries.PER_FILE_CACHE_IR_LEVEL_DIR_NAME + else CachedLibraries.PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME + return fileCacheDirectory.child(contentDirName).absolutePath + } internal val cachedLibraries: CachedLibraries = run { val explicitCacheFiles = configuration.get(KonanConfigKeys.CACHED_LIBRARIES)!! @@ -86,39 +125,33 @@ class CacheSupport( "not found among resolved libraries:\n " + allLibraries.joinToString("\n ") { it.libraryFile.absolutePath }) - internal val librariesToCache: Set = run { - val libraryToAddToCachePath = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE) - if (libraryToAddToCachePath.isNullOrEmpty()) { - configuration.get(KonanConfigKeys.LIBRARIES_TO_CACHE)!! - .map { getLibrary(File(it)) } - .toSet() - .also { if (!produce.isCache) check(it.isEmpty()) } - } else { - val libraryToAddToCacheFile = File(libraryToAddToCachePath) - val libraryToAddToCache = getLibrary(libraryToAddToCacheFile) - val libraryCache = cachedLibraries.getLibraryCache(libraryToAddToCache) - if (libraryCache == null) - setOf(libraryToAddToCache) - else - emptySet() + internal val libraryToCache = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { + val libraryToAddToCacheFile = File(it) + val libraryToAddToCache = getLibrary(libraryToAddToCacheFile) + val libraryCache = cachedLibraries.getLibraryCache(libraryToAddToCache) + if (libraryCache != null && libraryCache.granularity == CachedLibraries.Granularity.MODULE) + null + else { + val fileToCache = configuration.get(KonanConfigKeys.FILE_TO_CACHE) + PartialCacheInfo(libraryToAddToCache, if (fileToCache == null) CacheDeserializationStrategy.WholeModule else CacheDeserializationStrategy.SingleFile(fileToCache)) } } - internal val libraryToCache = librariesToCache.singleOrNull()?.let { - PartialCacheInfo(it, CacheDeserializationStrategy.WholeModule) - } - internal val preLinkCaches: Boolean = configuration.get(KonanConfigKeys.PRE_LINK_CACHES, false) + companion object { + fun cacheFileId(fqName: String, filePath: String) = "${if (fqName == "") "ROOT" else fqName}.${filePath.hashCode().toString(Character.MAX_RADIX)}" + } + init { // Ensure dependencies of every cached library are cached too: resolvedLibraries.getFullList { libraries -> libraries.map { library -> val cache = cachedLibraries.getLibraryCache(library.library) - if (cache != null || library.library in librariesToCache) { + if (cache != null || library.library == libraryToCache?.klib) { library.resolvedDependencies.forEach { - if (!cachedLibraries.isLibraryCached(it.library) && it.library !in librariesToCache) { + if (!cachedLibraries.isLibraryCached(it.library) && it.library != libraryToCache?.klib) { val description = if (cache != null) { "cached (in ${cache.path})" } else { @@ -137,15 +170,15 @@ class CacheSupport( } // Ensure not making cache for libraries that are already cached: - librariesToCache.forEach { + libraryToCache?.klib?.let { val cache = cachedLibraries.getLibraryCache(it) - if (cache != null) { - configuration.reportCompilationError("Can't cache library '${it.libraryName}' " + + if (cache != null && cache.granularity == CachedLibraries.Granularity.MODULE) { + configuration.reportCompilationError("can't cache library '${it.libraryName}' " + "that is already cached in '${cache.path}'") } } - if ((librariesToCache.isNotEmpty() || cachedLibraries.hasDynamicCaches || cachedLibraries.hasStaticCaches) + if ((libraryToCache != null || cachedLibraries.hasDynamicCaches || cachedLibraries.hasStaticCaches) && configuration.getBoolean(KonanConfigKeys.OPTIMIZATION)) { configuration.reportCompilationError("Cache cannot be used in optimized compilation") } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt index 4411ef3af78..f9e6a119e70 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.backend.konan.serialization.ClassFieldsSerializer import org.jetbrains.kotlin.backend.konan.serialization.InlineFunctionBodyReferenceSerializer +import org.jetbrains.kotlin.backend.konan.serialization.SerializedClassFields +import org.jetbrains.kotlin.backend.konan.serialization.SerializedInlineFunctionReference import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.KonanTarget @@ -19,25 +21,69 @@ class CachedLibraries( explicitCaches: Map, implicitCacheDirectories: List ) { + enum class Kind { DYNAMIC, STATIC } + enum class Granularity { MODULE, FILE } - class Cache(val kind: Kind, val path: String) { - enum class Kind { DYNAMIC, STATIC } + private fun Kind.toCompilerOutputKind(): CompilerOutputKind = when (this) { + Kind.DYNAMIC -> CompilerOutputKind.DYNAMIC_CACHE + Kind.STATIC -> CompilerOutputKind.STATIC_CACHE + } + + inner class Cache(val kind: Kind, val granularity: Granularity, val path: String) { + val fileDirs by lazy { File(path).listFiles.filter { it.isDirectory }.sortedBy { it.name } } val bitcodeDependencies by lazy { - val directory = File(path).absoluteFile.parent - File(directory, BITCODE_DEPENDENCIES_FILE_NAME).readStrings() + when (granularity) { + Granularity.MODULE -> File(path).parentFile.child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings() + Granularity.FILE -> fileDirs.flatMap { + it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings() + }.distinct() + } + } + + val binariesPaths by lazy { + when (granularity) { + Granularity.MODULE -> listOf(path) + Granularity.FILE -> fileDirs.map { + it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(getArtifactName(it.name, kind.toCompilerOutputKind())).absolutePath + } + } } val serializedInlineFunctionBodies by lazy { - val directory = File(path).absoluteFile.parent - val data = File(directory, INLINE_FUNCTION_BODIES_FILE_NAME).readBytes() - InlineFunctionBodyReferenceSerializer.deserialize(data) + val result = mutableListOf() + when (granularity) { + Granularity.MODULE -> { + val directory = File(path).absoluteFile.parent + val data = File(directory, INLINE_FUNCTION_BODIES_FILE_NAME).readBytes() + InlineFunctionBodyReferenceSerializer.deserializeTo(data, result) + } + Granularity.FILE -> { + fileDirs.forEach { fileDir -> + val data = fileDir.child(PER_FILE_CACHE_IR_LEVEL_DIR_NAME).child(INLINE_FUNCTION_BODIES_FILE_NAME).readBytes() + InlineFunctionBodyReferenceSerializer.deserializeTo(data, result) + } + } + } + result } val serializedClassFields by lazy { - val directory = File(path).absoluteFile.parent - val data = File(directory, CLASS_FIELDS_FILE_NAME).readBytes() - ClassFieldsSerializer.deserialize(data) + val result = mutableListOf() + when (granularity) { + Granularity.MODULE -> { + val directory = File(path).absoluteFile.parent + val data = File(directory, CLASS_FIELDS_FILE_NAME).readBytes() + ClassFieldsSerializer.deserializeTo(data, result) + } + Granularity.FILE -> { + fileDirs.forEach { fileDir -> + val data = fileDir.child(PER_FILE_CACHE_IR_LEVEL_DIR_NAME).child(CLASS_FIELDS_FILE_NAME).readBytes() + ClassFieldsSerializer.deserializeTo(data, result) + } + } + } + result } } @@ -57,9 +103,9 @@ class CachedLibraries( error("Both dynamic and static caches files cannot be in the same directory." + " Library: ${library.libraryName}, path to cache: ${cacheDir.absolutePath}") return when { - dynamicFile.absolutePath in cacheDirContents -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath) - staticFile.absolutePath in cacheDirContents -> Cache(Cache.Kind.STATIC, staticFile.absolutePath) - else -> error("No cache found for library ${library.libraryName} at ${cacheDir.absolutePath}") + dynamicFile.absolutePath in cacheDirContents -> Cache(Kind.DYNAMIC, Granularity.MODULE, dynamicFile.absolutePath) + staticFile.absolutePath in cacheDirContents -> Cache(Kind.STATIC, Granularity.MODULE, staticFile.absolutePath) + else -> Cache(Kind.STATIC, Granularity.FILE, cacheDir.absolutePath) } } @@ -71,7 +117,8 @@ class CachedLibraries( ?: error("No cache found for library ${library.libraryName} at $explicitPath") } else { implicitCacheDirectories.firstNotNullOfOrNull { dir -> - selectCache(library, dir.child(getCachedLibraryName(library))) + selectCache(library, dir.child(getPerFileCachedLibraryName(library))) + ?: selectCache(library, dir.child(getCachedLibraryName(library))) } } @@ -89,22 +136,26 @@ class CachedLibraries( val hasStaticCaches = allCaches.values.any { when (it.kind) { - Cache.Kind.STATIC -> true - Cache.Kind.DYNAMIC -> false + Kind.STATIC -> true + Kind.DYNAMIC -> false } } val hasDynamicCaches = allCaches.values.any { when (it.kind) { - Cache.Kind.STATIC -> false - Cache.Kind.DYNAMIC -> true + Kind.STATIC -> false + Kind.DYNAMIC -> true } } companion object { + fun getPerFileCachedLibraryName(library: KotlinLibrary): String = "${library.uniqueName}-per-file-cache" fun getCachedLibraryName(library: KotlinLibrary): String = getCachedLibraryName(library.uniqueName) fun getCachedLibraryName(libraryName: String): String = "$libraryName-cache" + const val PER_FILE_CACHE_IR_LEVEL_DIR_NAME = "ir" + const val PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME = "bin" + const val BITCODE_DEPENDENCIES_FILE_NAME = "bitcode_deps" const val INLINE_FUNCTION_BODIES_FILE_NAME = "inline_bodies" const val CLASS_FIELDS_FILE_NAME = "class_fields" diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt index 48b6b49268a..c1012c9e1e7 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt @@ -31,10 +31,9 @@ import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty val KonanConfig.isFinalBinary: Boolean get() = when (this.produce) { CompilerOutputKind.PROGRAM, CompilerOutputKind.DYNAMIC, CompilerOutputKind.STATIC -> true - CompilerOutputKind.DYNAMIC_CACHE, CompilerOutputKind.STATIC_CACHE, + CompilerOutputKind.DYNAMIC_CACHE, CompilerOutputKind.STATIC_CACHE, CompilerOutputKind.PRELIMINARY_CACHE, CompilerOutputKind.LIBRARY, CompilerOutputKind.BITCODE -> false CompilerOutputKind.FRAMEWORK -> !omitFrameworkBinary - else -> error("Unknown compiler output kind") } val CompilerOutputKind.involvesBitcodeGeneration: Boolean @@ -43,18 +42,30 @@ val CompilerOutputKind.involvesBitcodeGeneration: Boolean internal val Context.producedLlvmModuleContainsStdlib: Boolean get() = this.llvmModuleSpecification.containsModule(this.stdlibModule) +internal val Context.shouldDefineFunctionClasses: Boolean + get() = producedLlvmModuleContainsStdlib && + config.libraryToCache?.strategy?.contains(KonanFqNames.internalPackageName, "KFunctionImpl.kt") != false + +internal val Context.shouldDefineCachedBoxes: Boolean + get() = producedLlvmModuleContainsStdlib && + config.libraryToCache?.strategy?.contains(KonanFqNames.internalPackageName, "Boxing.kt") != false + +internal val Context.shouldLinkRuntimeNativeLibraries: Boolean + get() = producedLlvmModuleContainsStdlib && + config.libraryToCache?.strategy?.contains(KonanFqNames.packageName, "Runtime.kt") != false + val KonanConfig.involvesLinkStage: Boolean get() = when (this.produce) { CompilerOutputKind.PROGRAM, CompilerOutputKind.DYNAMIC, CompilerOutputKind.DYNAMIC_CACHE, CompilerOutputKind.STATIC_CACHE, - CompilerOutputKind.STATIC-> true - CompilerOutputKind.LIBRARY, CompilerOutputKind.BITCODE -> false + CompilerOutputKind.STATIC -> true + CompilerOutputKind.LIBRARY, CompilerOutputKind.BITCODE, CompilerOutputKind.PRELIMINARY_CACHE -> false CompilerOutputKind.FRAMEWORK -> !omitFrameworkBinary - else -> error("Unknown compiler output kind") } val CompilerOutputKind.isCache: Boolean - get() = (this == CompilerOutputKind.STATIC_CACHE || this == CompilerOutputKind.DYNAMIC_CACHE) + get() = this == CompilerOutputKind.STATIC_CACHE || this == CompilerOutputKind.DYNAMIC_CACHE + || this == CompilerOutputKind.PRELIMINARY_CACHE internal fun llvmIrDumpCallback(state: ActionState, module: IrModuleFragment, context: Context) { module.let{} @@ -117,7 +128,6 @@ private fun collectLlvmModules(context: Context, generatedBitcodeFiles: List): List = files.map { bitcodeFile -> @@ -128,7 +138,10 @@ private fun collectLlvmModules(context: Context, generatedBitcodeFiles: List {} - else -> error("Unknown compiler output kind") + CompilerOutputKind.PRELIMINARY_CACHE -> {} } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index 71db05ace19..877a30e55b0 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -496,7 +496,7 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { val llvmModuleSpecification: LlvmModuleSpecification by lazy { when { config.produce.isCache -> - CacheLlvmModuleSpecification(config.cachedLibraries, config.librariesToCache) + CacheLlvmModuleSpecification(config.cachedLibraries, config.libraryToCache!!.klib) else -> DefaultLlvmModuleSpecification(config.cachedLibraries) } } @@ -514,6 +514,9 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { val classFields = mutableListOf() + val calledFromExportedInlineFunctions = mutableSetOf() + val constructedFromExportedInlineFunctions = mutableSetOf() + val targetAbiInfo: TargetAbiInfo by lazy { when { config.target == KonanTarget.MINGW_X64 -> { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InternalAbi.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InternalAbi.kt index 21dd99bc271..7d6fb1f3759 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InternalAbi.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InternalAbi.kt @@ -7,14 +7,17 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.backend.konan.descriptors.synthesizedName import org.jetbrains.kotlin.backend.konan.llvm.llvmSymbolOrigin import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl import org.jetbrains.kotlin.ir.declarations.* -import org.jetbrains.kotlin.ir.declarations.impl.IrModuleFragmentImpl +import org.jetbrains.kotlin.ir.declarations.impl.IrExternalPackageFragmentImpl +import org.jetbrains.kotlin.ir.symbols.impl.IrExternalPackageFragmentSymbolImpl import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl import org.jetbrains.kotlin.ir.util.addChild import org.jetbrains.kotlin.ir.util.addFile import org.jetbrains.kotlin.ir.util.fqNameForIrSerialization import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.resolve.scopes.MemberScope /** * Sometimes we need to reference symbols that are not declared in metadata. @@ -38,14 +41,21 @@ internal class InternalAbi(private val context: Context) { /** * Representation of ABI files from external modules. */ - private val externalAbiFiles = mutableMapOf() + private val externalAbiFiles = mutableMapOf() fun init(modules: List) { internalAbiFiles = modules.associate { it.descriptor to createAbiFile(it) } } private fun createAbiFile(module: IrModuleFragment): IrFile = - module.addFile(NaiveSourceBasedFileEntryImpl("internal"), FqName("kotlin.native.caches.abi")) + module.addFile(NaiveSourceBasedFileEntryImpl("internal"), KonanFqNames.cachesInternalAbi) + + private fun createExternalAbiFile(module: ModuleDescriptor, fqName: FqName): IrExternalPackageFragment { + val packageFragmentDescriptor = object : PackageFragmentDescriptorImpl(module, fqName) { + override fun getMemberScope(): MemberScope = MemberScope.Empty + } + return IrExternalPackageFragmentImpl(IrExternalPackageFragmentSymbolImpl(packageFragmentDescriptor), fqName) + } /** * Adds external [function] from [module] to a list of external references. @@ -53,9 +63,8 @@ internal class InternalAbi(private val context: Context) { fun reference(function: IrFunction, module: ModuleDescriptor) { assert(function.isExternal) { "Function that represents external ABI should be marked as external" } context.llvmImports.add(module.llvmSymbolOrigin) - externalAbiFiles.getOrPut(module) { - createAbiFile(IrModuleFragmentImpl(module, context.irBuiltIns)) - }.addChild(function) + val externalAbiFile = externalAbiFiles.getOrPut(module) { createExternalAbiFile(module, KonanFqNames.cachesInternalAbi) } + externalAbiFile.addChild(function) } /** diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InteropUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InteropUtils.kt index a759f968f86..70cd0172329 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InteropUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InteropUtils.kt @@ -91,6 +91,14 @@ internal class InteropBuiltIns(builtIns: KonanBuiltIns) { val objCObjectBase = packageScope.getContributedClass("ObjCObjectBase") + val objCObjectBaseMeta = packageScope.getContributedClass("ObjCObjectBaseMeta") + + val objCClass = packageScope.getContributedClass("ObjCClass") + + val objCClassOf = packageScope.getContributedClass("ObjCClassOf") + + val objCProtocol = packageScope.getContributedClass("ObjCProtocol") + val allocObjCObject = packageScope.getContributedFunctions("allocObjCObject").single() val getObjCClass = packageScope.getContributedFunctions("getObjCClass").single() diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 898b31c80e6..c5cd930dbe7 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -394,27 +394,27 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val cachedLibraries: CachedLibraries get() = cacheSupport.cachedLibraries - internal val librariesToCache: Set - get() = cacheSupport.librariesToCache - internal val libraryToCache: PartialCacheInfo? get() = cacheSupport.libraryToCache + internal val producePerFileCache = libraryToCache?.strategy is CacheDeserializationStrategy.SingleFile + val outputFiles = OutputFiles(configuration.get(KonanConfigKeys.OUTPUT) ?: cacheSupport.tryGetImplicitOutput(), - target, produce) + target, produce, producePerFileCache) val tempFiles = TempFiles(outputFiles.outputName, configuration.get(KonanConfigKeys.TEMPORARY_FILES_DIR)) - val outputFile get() = outputFiles.mainFile + val outputFile get() = outputFiles.mainFileName private val implicitModuleName: String - get() = File(outputFiles.outputName).name + get() = if (produce.isCache) outputFiles.cacheFileName else File(outputFiles.outputName).name - val infoArgsOnly = configuration.kotlinSourceRoots.isEmpty() + val infoArgsOnly = (configuration.kotlinSourceRoots.isEmpty() && configuration[KonanConfigKeys.INCLUDED_LIBRARIES].isNullOrEmpty() - && librariesToCache.isEmpty() && configuration[KonanConfigKeys.EXPORTED_LIBRARIES].isNullOrEmpty() + && libraryToCache == null) + || (producePerFileCache && outputFiles.mainFile.exists) /** * Do not compile binary when compiling framework. diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index 12d805814c6..14c0f6ae736 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -44,6 +44,8 @@ class KonanConfigKeys { = CompilerConfigurationKey.create>("paths to directories containing caches") val CACHED_LIBRARIES: CompilerConfigurationKey> = CompilerConfigurationKey.create>("mapping from library paths to cache paths") + val FILE_TO_CACHE: CompilerConfigurationKey + = CompilerConfigurationKey.create("which file should be compiled to cache") val FRAMEWORK_IMPORT_HEADERS: CompilerConfigurationKey> = CompilerConfigurationKey.create>("headers imported to framework header") val FRIEND_MODULES: CompilerConfigurationKey> diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt index 68e970389da..0cf6064275b 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt @@ -39,4 +39,5 @@ object KonanFqNames { val noReorderFields = FqName("kotlin.native.internal.NoReorderFields") val objCName = FqName("kotlin.native.ObjCName") val reflectionPackageName = FqName("kotlin.native.internal.ReflectionPackageName") + val cachesInternalAbi = FqName("kotlin.native.caches.abi") } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt index 781390c8bb7..02025e0406a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLibrariesResolveSupport.kt @@ -26,11 +26,8 @@ class KonanLibrariesResolveSupport( private val includedLibraryFiles = configuration.getList(KonanConfigKeys.INCLUDED_LIBRARIES).map { File(it) } - private val librariesToCacheFiles = - configuration.getList(KonanConfigKeys.LIBRARIES_TO_CACHE).map { File(it) } + - configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE).let { - if (it.isNullOrEmpty()) emptyList() else listOf(File(it)) - } + private val libraryToCacheFile = + configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { File(it) } private val libraryNames = configuration.getList(KonanConfigKeys.LIBRARY_FILES) @@ -62,7 +59,7 @@ class KonanLibrariesResolveSupport( // But currently the resolver is in the middle of a complex refactoring so it was decided to avoid changes in its logic. // TODO: Handle included libraries in KonanLibraryResolver when it's refactored and moved into the big Kotlin repo. internal val resolvedLibraries = run { - val additionalLibraryFiles = includedLibraryFiles + librariesToCacheFiles + val additionalLibraryFiles = includedLibraryFiles + listOfNotNull(libraryToCacheFile) resolver.resolveWithDependencies( unresolvedLibraries + additionalLibraryFiles.map { UnresolvedLibrary(it.absolutePath, null) }, noStdLib = configuration.getBoolean(KonanConfigKeys.NOSTDLIB), diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt index 61838fc8ceb..2e1a65c45ed 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt @@ -34,6 +34,50 @@ internal fun determineLinkerOutput(context: Context): LinkerOutputKind = else -> TODO("${context.config.produce} should not reach native linker stage") } +internal class CacheStorage(val context: Context) { + private val isPreliminaryCache = context.config.produce == CompilerOutputKind.PRELIMINARY_CACHE + + fun renameOutput() { + val outputFiles = context.config.outputFiles + // For caches the output file is a directory. It might be created by someone else, + // we have to delete it in order for the next renaming operation to succeed. + outputFiles.mainFile.delete() + if (!outputFiles.tempCacheDirectory!!.renameTo(outputFiles.mainFile)) + outputFiles.tempCacheDirectory.deleteRecursively() + } + + fun saveAdditionalCacheInfo() { + context.config.outputFiles.tempCacheDirectory!!.mkdirs() + if (!isPreliminaryCache) + saveCacheBitcodeDependencies() + if (isPreliminaryCache || context.configuration.get(KonanConfigKeys.FILE_TO_CACHE) == null) { + saveInlineFunctionBodies() + saveClassFields() + } + } + + private fun saveCacheBitcodeDependencies() { + val bitcodeDependencies = context.config.resolvedLibraries + .getFullList(TopologicalLibraryOrder) + .filter { + require(it is KonanLibrary) + context.llvmImports.bitcodeIsUsed(it) + && it != context.config.cacheSupport.libraryToCache?.klib // Skip loops. + }.cast>() + context.config.outputFiles.bitcodeDependenciesFile!!.writeLines(bitcodeDependencies.map { it.uniqueName }) + } + + private fun saveInlineFunctionBodies() { + context.config.outputFiles.inlineFunctionBodiesFile!!.writeBytes( + InlineFunctionBodyReferenceSerializer.serialize(context.inlineFunctionBodies)) + } + + private fun saveClassFields() { + context.config.outputFiles.classFieldsFile!!.writeBytes( + ClassFieldsSerializer.serialize(context.classFields)) + } +} + // TODO: We have a Linker.kt file in the shared module. internal class Linker(val context: Context) { @@ -48,65 +92,13 @@ internal class Linker(val context: Context) { fun link(objectFiles: List) { val nativeDependencies = context.llvm.nativeDependenciesToLink - val includedBinariesLibraries = if (context.config.produce.isCache) { - context.config.librariesToCache - } else { - nativeDependencies.filterNot { context.config.cachedLibraries.isLibraryCached(it) } - } + val includedBinariesLibraries = context.config.libraryToCache?.let { listOf(it.klib) } + ?: nativeDependencies.filterNot { context.config.cachedLibraries.isLibraryCached(it) } val includedBinaries = includedBinariesLibraries.map { (it as? KonanLibrary)?.includedPaths.orEmpty() }.flatten() val libraryProvidedLinkerFlags = context.llvm.allNativeDependencies.map { it.linkerOpts }.flatten() - if (context.config.produce.isCache) { - context.config.outputFiles.tempCacheDirectory!!.mkdirs() - saveAdditionalInfoForCache() - } - runLinker(objectFiles, includedBinaries, libraryProvidedLinkerFlags) - - renameOutput() - } - - private fun saveAdditionalInfoForCache() { - saveCacheBitcodeDependencies() - saveInlineFunctionBodies() - saveClassFields() - } - - private fun saveCacheBitcodeDependencies() { - val outputFiles = context.config.outputFiles - val bitcodeDependenciesFile = File(outputFiles.bitcodeDependenciesFile!!) - val bitcodeDependencies = context.config.resolvedLibraries - .getFullList(TopologicalLibraryOrder) - .filter { - require(it is KonanLibrary) - context.llvmImports.bitcodeIsUsed(it) - && it !in context.config.cacheSupport.librariesToCache // Skip loops. - }.cast>() - bitcodeDependenciesFile.writeLines(bitcodeDependencies.map { it.uniqueName }) - } - - private fun saveInlineFunctionBodies() { - val outputFiles = context.config.outputFiles - val inlineFunctionBodiesFile = File(outputFiles.inlineFunctionBodiesFile!!) - inlineFunctionBodiesFile.writeBytes(InlineFunctionBodyReferenceSerializer.serialize(context.inlineFunctionBodies)) - } - - private fun saveClassFields() { - val outputFiles = context.config.outputFiles - val classFieldsFile = File(outputFiles.classFieldsFile!!) - classFieldsFile.writeBytes(ClassFieldsSerializer.serialize(context.classFields)) - } - - private fun renameOutput() { - if (context.config.produce.isCache) { - val outputFiles = context.config.outputFiles - // For caches the output file is a directory. It might be created by someone else, - // We have to delete it in order to the next renaming operation to succeed. - java.io.File(outputFiles.mainFile).delete() - if (!java.io.File(outputFiles.tempCacheDirectory!!.absolutePath).renameTo(java.io.File(outputFiles.mainFile))) - outputFiles.tempCacheDirectory.deleteRecursively() - } } private fun asLinkerArgs(args: List): List { @@ -255,11 +247,11 @@ private fun determineCachesToLink(context: Context): CachesToLink { error("Library ${library.libraryName} is found in both cache and current binary") val list = when (cache.kind) { - CachedLibraries.Cache.Kind.DYNAMIC -> dynamicCaches - CachedLibraries.Cache.Kind.STATIC -> staticCaches + CachedLibraries.Kind.DYNAMIC -> dynamicCaches + CachedLibraries.Kind.STATIC -> staticCaches } - list += cache.path + list += cache.binariesPaths } return CachesToLink(static = staticCaches, dynamic = dynamicCaches) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt index dfae43f4018..6d935f013e0 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt @@ -7,10 +7,8 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.backend.konan.ir.konanLibrary import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.ir.declarations.IrDeclaration -import org.jetbrains.kotlin.ir.declarations.IrFile -import org.jetbrains.kotlin.ir.declarations.IrModuleFragment -import org.jetbrains.kotlin.ir.declarations.IrPackageFragment +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.util.getPackageFragment import org.jetbrains.kotlin.library.KotlinLibrary internal abstract class LlvmModuleSpecificationBase(protected val cachedLibraries: CachedLibraries) : LlvmModuleSpecification { @@ -31,6 +29,8 @@ internal abstract class LlvmModuleSpecificationBase(protected val cachedLibrarie override fun containsDeclaration(declaration: IrDeclaration): Boolean = declaration.konanLibrary.let { it == null || containsLibrary(it) } + // When producing per-file caches, some declarations might be generated by the stub generator. + && declaration.getPackageFragment() !is IrExternalPackageFragment } internal class DefaultLlvmModuleSpecification(cachedLibraries: CachedLibraries) @@ -42,9 +42,9 @@ internal class DefaultLlvmModuleSpecification(cachedLibraries: CachedLibraries) internal class CacheLlvmModuleSpecification( cachedLibraries: CachedLibraries, - private val librariesToCache: Set + private val libraryToCache: KotlinLibrary ) : LlvmModuleSpecificationBase(cachedLibraries) { override val isFinal = false - override fun containsLibrary(library: KotlinLibrary): Boolean = library in librariesToCache + override fun containsLibrary(library: KotlinLibrary): Boolean = library == libraryToCache } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt index dbd04f60484..f55c7012f26 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt @@ -17,7 +17,7 @@ import kotlin.random.Random /** * Creates and stores terminal compiler outputs. */ -class OutputFiles(outputPath: String?, target: KonanTarget, val produce: CompilerOutputKind) { +class OutputFiles(outputPath: String?, target: KonanTarget, val produce: CompilerOutputKind, producePerFileCache: Boolean) { private val prefix = produce.prefix(target) private val suffix = produce.suffix(target) @@ -34,44 +34,41 @@ class OutputFiles(outputPath: String?, target: KonanTarget, val produce: Compile val cAdapterDef by lazy { File("${outputName}.def") } /** - * Main compiler's output file. + * Compiler's main output file. */ - val mainFile = + val mainFileName = if (produce.isCache) outputName else outputName.fullOutputName() - private val cacheFile = File(outputName.fullOutputName()).absoluteFile.name + val mainFile = File(mainFileName) - val dynamicCacheInstallName = File(outputName).child(cacheFile).absolutePath + private val pathToPerFileCache = + if (producePerFileCache) + outputName.substring(0, outputName.lastIndexOf(File.separatorChar) /* skip [PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME]*/) + else null + + val perFileCacheFileName = pathToPerFileCache?.let { File(it).name } + + val cacheFileName = File((pathToPerFileCache ?: outputName).fullOutputName()).absoluteFile.name + + val dynamicCacheInstallName = File(outputName).child(cacheFileName).absolutePath val tempCacheDirectory = if (produce.isCache) File(outputName + Random.nextLong().toString()) else null - val nativeBinaryFile = - if (produce.isCache) - tempCacheDirectory!!.child(cacheFile).absolutePath - else mainFile + val nativeBinaryFile = tempCacheDirectory?.child(cacheFileName)?.absolutePath ?: mainFileName val symbolicInfoFile = "$nativeBinaryFile.dSYM" - val bitcodeDependenciesFile = - if (produce.isCache) - tempCacheDirectory!!.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME).absolutePath - else null + val bitcodeDependenciesFile = tempCacheDirectory?.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME) - val inlineFunctionBodiesFile = - if (produce.isCache) - tempCacheDirectory!!.child(CachedLibraries.INLINE_FUNCTION_BODIES_FILE_NAME).absolutePath - else null + val inlineFunctionBodiesFile = tempCacheDirectory?.child(CachedLibraries.INLINE_FUNCTION_BODIES_FILE_NAME) - val classFieldsFile = - if (produce.isCache) - tempCacheDirectory!!.child(CachedLibraries.CLASS_FIELDS_FILE_NAME).absolutePath - else null + val classFieldsFile = tempCacheDirectory?.child(CachedLibraries.CLASS_FIELDS_FILE_NAME) private fun String.fullOutputName() = prefixBaseNameIfNeeded(prefix).suffixIfNeeded(suffix) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt index 0a398db87f4..31c1ea4bb54 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt @@ -64,10 +64,10 @@ internal fun Context.psiToIr( ) val irBuiltInsOverDescriptors = generatorContext.irBuiltIns as IrBuiltInsOverDescriptors val functionIrClassFactory: KonanIrAbstractDescriptorBasedFunctionFactory = - if (config.lazyIrForCaches && !llvmModuleSpecification.containsModule(this.stdlibModule)) - LazyIrFunctionFactory(symbolTable, stubGenerator, irBuiltInsOverDescriptors, reflectionTypes) - else + if (shouldDefineFunctionClasses || (!config.lazyIrForCaches && !config.producePerFileCache)) BuiltInFictitiousFunctionIrClassFactory(symbolTable, irBuiltInsOverDescriptors, reflectionTypes) + else + LazyIrFunctionFactory(symbolTable, stubGenerator, irBuiltInsOverDescriptors, reflectionTypes) irBuiltInsOverDescriptors.functionFactory = functionIrClassFactory val symbols = KonanSymbols(this, generatorContext.irBuiltIns, symbolTable, symbolTable.lazyWrapper) @@ -144,10 +144,11 @@ internal fun Context.psiToIr( for (dependency in sortDependencies(dependencies).filter { it != moduleDescriptor }) { val kotlinLibrary = (dependency.getCapability(KlibModuleOrigin.CAPABILITY) as? DeserializedKlibModuleOrigin)?.library - val isCachedLibrary = kotlinLibrary != null && config.cachedLibraries.isLibraryCached(kotlinLibrary) - if (isProducingLibrary || (config.lazyIrForCaches && isCachedLibrary)) + val isFullyCachedLibrary = kotlinLibrary != null && + config.cachedLibraries.isLibraryCached(kotlinLibrary) && kotlinLibrary != config.libraryToCache?.klib + if (isProducingLibrary || (config.lazyIrForCaches && isFullyCachedLibrary)) linker.deserializeOnlyHeaderModule(dependency, kotlinLibrary) - else if (isCachedLibrary) + else if (isFullyCachedLibrary) linker.deserializeHeadersWithInlineBodies(dependency, kotlinLibrary!!) else linker.deserializeIrModuleHeader(dependency, kotlinLibrary, dependency.name.asString()) @@ -217,6 +218,11 @@ internal fun Context.psiToIr( // Note: coupled with [shouldLower] below. irModules = modules.filterValues { llvmModuleSpecification.containsModule(it) } + // IR linker deserializes files in the order they lie on the disk, which might be inconvenient, + // so to make the pipeline more deterministic, the files are to be sorted. + // This concerns in the first place global initializers order for the eager initialization strategy, + // where the files are being initialized in order one by one. + irModules.values.forEach { module -> module.files.sortBy { it.fileEntry.name } } if (!isProducingLibrary) irLinker = irDeserializer as KonanIrLinker @@ -226,7 +232,7 @@ internal fun Context.psiToIr( if (!isProducingLibrary) { // TODO: find out what should be done in the new builtins/symbols about it if (this.stdlibModule in modulesWithoutDCE) { - (functionIrClassFactory as BuiltInFictitiousFunctionIrClassFactory).buildAllClasses() + (functionIrClassFactory as? BuiltInFictitiousFunctionIrClassFactory)?.buildAllClasses() } internalAbi.init(irModules.values + irModule!!) (functionIrClassFactory as? BuiltInFictitiousFunctionIrClassFactory)?.module = diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt index 8cf29ee5d51..c401f02ca90 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt @@ -9,6 +9,7 @@ import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer import org.jetbrains.kotlin.backend.konan.descriptors.isFromInteropLibrary import org.jetbrains.kotlin.backend.konan.llvm.* +import org.jetbrains.kotlin.backend.konan.lower.CacheInfoBuilder import org.jetbrains.kotlin.backend.konan.lower.ExpectToActualDefaultValueCopier import org.jetbrains.kotlin.backend.konan.lower.SamSuperTypesChecker import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport @@ -22,15 +23,10 @@ import org.jetbrains.kotlin.ir.builders.declarations.buildFun import org.jetbrains.kotlin.ir.builders.declarations.addValueParameter import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl -import org.jetbrains.kotlin.ir.expressions.IrExpression -import org.jetbrains.kotlin.ir.expressions.IrGetObjectValue -import org.jetbrains.kotlin.ir.expressions.IrGetField +import org.jetbrains.kotlin.ir.expressions.* import org.jetbrains.kotlin.ir.expressions.impl.IrCallImpl import org.jetbrains.kotlin.ir.util.* -import org.jetbrains.kotlin.ir.visitors.IrElementTransformerVoid -import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid -import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid -import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid +import org.jetbrains.kotlin.ir.visitors.* import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name @@ -133,33 +129,7 @@ internal val buildAdditionalCacheInfoPhase = konanUnitPhase( if (moduleDeserializer == null) { require(module.descriptor.isFromInteropLibrary()) { "No module deserializer for ${module.descriptor}" } } else { - val compatibleMode = CompatibilityMode(moduleDeserializer.libraryAbiVersion).oldSignatures - module.acceptChildrenVoid(object : IrElementVisitorVoid { - override fun visitElement(element: IrElement) { - element.acceptChildrenVoid(this) - } - - override fun visitClass(declaration: IrClass) { - declaration.acceptChildrenVoid(this) - - if (!declaration.isInterface && declaration.visibility != DescriptorVisibilities.LOCAL - && declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS) - classFields.add(moduleDeserializer.buildClassFields(declaration, getLayoutBuilder(declaration).getDeclaredFields())) - } - - override fun visitFunction(declaration: IrFunction) { - declaration.acceptChildrenVoid(this) - - if (declaration.isFakeOverride || !declaration.isExportedInlineFunction) return - inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration)) - } - - private val IrClass.isExported - get() = with(KonanManglerIr) { isExported(compatibleMode) } - - private val IrFunction.isExportedInlineFunction - get() = isInline && with(KonanManglerIr) { isExported(compatibleMode) } - }) + CacheInfoBuilder(this, moduleDeserializer).build() } } }, @@ -240,6 +210,12 @@ internal val serializerPhase = konanUnitPhase( description = "Serialize descriptor tree and inline IR bodies" ) +internal val saveAdditionalCacheInfoPhase = konanUnitPhase( + op = { CacheStorage(this).saveAdditionalCacheInfo() }, + name = "SaveAdditionalCacheInfo", + description = "Save additional cache info (inline functions bodies and fields of classes)" +) + internal val objectFilesPhase = konanUnitPhase( op = { compilerOutput = BitcodeCompiler(this).makeObjectFiles(bitcodeFileName) }, name = "ObjectFiles", @@ -252,6 +228,12 @@ internal val linkerPhase = konanUnitPhase( description = "Linker" ) +internal val finalizeCachePhase = konanUnitPhase( + op = { CacheStorage(this).renameOutput() }, + name = "FinalizeCache", + description = "Finalize cache (rename temp to the final dist)" +) + internal val allLoweringsPhase = NamedCompilerPhase( name = "IrLowering", description = "IR Lowering", @@ -407,6 +389,9 @@ internal val exportInternalAbiPhase = makeKonanModuleOpPhase( override fun visitClass(declaration: IrClass) { declaration.acceptChildrenVoid(this) + + if (declaration.isLocal) return + if (declaration.isCompanion) { val function = context.irFactory.buildFun { name = InternalAbi.getCompanionObjectAccessorName(declaration) @@ -595,8 +580,10 @@ val toplevelPhase: CompilerPhase<*, Unit, Unit> = namedUnitPhase( disposeLLVMPhase then unitSink() ) then + saveAdditionalCacheInfoPhase then objectFilesPhase then - linkerPhase + linkerPhase then + finalizeCachePhase ) internal fun PhaseConfig.disableIf(phase: AnyNamedPhase, condition: Boolean) { @@ -621,8 +608,10 @@ internal fun PhaseConfig.konanPhasesConfig(config: KonanConfig) { disableUnless(serializerPhase, config.produce == CompilerOutputKind.LIBRARY) disableUnless(entryPointPhase, config.produce == CompilerOutputKind.PROGRAM) disableUnless(buildAdditionalCacheInfoPhase, config.produce.isCache && config.lazyIrForCaches) + disableUnless(saveAdditionalCacheInfoPhase, config.produce.isCache && config.lazyIrForCaches) + disableUnless(finalizeCachePhase, config.produce.isCache) disableUnless(exportInternalAbiPhase, config.produce.isCache) - disableIf(backendCodegen, config.produce == CompilerOutputKind.LIBRARY || config.omitFrameworkBinary) + disableIf(backendCodegen, config.produce == CompilerOutputKind.LIBRARY || config.omitFrameworkBinary || config.produce == CompilerOutputKind.PRELIMINARY_CACHE) disableUnless(checkExternalCallsPhase, getBoolean(KonanConfigKeys.CHECK_EXTERNAL_CALLS)) disableUnless(rewriteExternalCallsCheckerGlobals, getBoolean(KonanConfigKeys.CHECK_EXTERNAL_CALLS)) disableUnless(stringConcatenationTypeNarrowingPhase, config.optimizationsEnabled) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt index d8002cbcfe4..acfe4fea2c2 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt @@ -279,7 +279,7 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { context.getLayoutBuilder(superClass).vtableEntries } - val methods = irClass.overridableOrOverridingMethods + val methods = overridableOrOverridingMethods val newVtableSlots = mutableListOf() val overridenVtableSlots = mutableMapOf() @@ -349,7 +349,7 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { } fun overridingOf(function: IrSimpleFunction) = - irClass.overridableOrOverridingMethods.firstOrNull { function in it.allOverriddenFunctions }?.let { + overridableOrOverridingMethods.firstOrNull { function in it.allOverriddenFunctions }?.let { OverriddenFunctionInfo(it, function).getImplementation(context) } @@ -468,13 +468,15 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { val outerThisField = if (irClass.isInner) context.specialDeclarationsFactory.getOuterThisField(irClass) else null - if (context.config.lazyIrForCaches && !context.llvmModuleSpecification.containsDeclaration(irClass)) { - val packageFragment = irClass.getPackageFragment() + val packageFragment = irClass.getPackageFragment() + if (packageFragment is IrExternalPackageFragment) { val moduleDescriptor = packageFragment.packageFragmentDescriptor.containingDeclaration - if (moduleDescriptor.isFromInteropLibrary()) - return emptyList() + if (moduleDescriptor.isFromInteropLibrary()) return emptyList() val moduleDeserializer = context.irLinker.moduleDeserializers[moduleDescriptor] ?: error("No module deserializer for ${irClass.render()}") + require(context.config.cachedLibraries.isLibraryCached(moduleDeserializer.klib)) { + "No IR and no cache for ${irClass.render()}" + } return moduleDeserializer.deserializeClassFields(irClass, outerThisField) } @@ -492,8 +494,8 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { } } - private val IrClass.overridableOrOverridingMethods: List - get() = this.simpleFunctions().filter { it.isOverridableOrOverrides && it.bridgeTarget == null } + private val overridableOrOverridingMethods: List + get() = irClass.simpleFunctions().filter { it.isOverridableOrOverrides && it.bridgeTarget == null } private val IrFunction.uniqueName get() = computeFunctionName() } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt index d128d1f9421..dfe99fee5ef 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt @@ -123,6 +123,10 @@ internal class KonanSymbols( val interopCOpaque = symbolTable.referenceClass(context.interopBuiltIns.cOpaque) val interopObjCObject = symbolTable.referenceClass(context.interopBuiltIns.objCObject) val interopObjCObjectBase = symbolTable.referenceClass(context.interopBuiltIns.objCObjectBase) + val interopObjCObjectBaseMeta = symbolTable.referenceClass(context.interopBuiltIns.objCObjectBaseMeta) + val interopObjCClass = symbolTable.referenceClass(context.interopBuiltIns.objCClass) + val interopObjCClassOf = symbolTable.referenceClass(context.interopBuiltIns.objCClassOf) + val interopObjCProtocol = symbolTable.referenceClass(context.interopBuiltIns.objCProtocol) val interopObjCRelease = interopFunction("objc_release") @@ -175,8 +179,14 @@ internal class KonanSymbols( val interopObjCGetSelector = interopFunction("objCGetSelector") + val interopCEnum = interopClass("CEnum") + + val interopCPrimitiveVar = interopClass("CPrimitiveVar") + val interopCEnumVar = interopClass("CEnumVar") + val interopCStructVar = interopClass("CStructVar") + val nativeMemUtils = symbolTable.referenceClass(context.interopBuiltIns.nativeMemUtils) val nativeHeap = symbolTable.referenceClass(context.interopBuiltIns.nativeHeap) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt index 51687a5b42b..143a74104cc 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt @@ -19,7 +19,6 @@ import org.jetbrains.kotlin.backend.konan.serialization.AbstractKonanIrMangler import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.util.findAnnotation import org.jetbrains.kotlin.ir.util.fqNameForIrSerialization -import org.jetbrains.kotlin.ir.util.isSuspend import org.jetbrains.kotlin.ir.util.render import org.jetbrains.kotlin.konan.library.KonanLibrary import org.jetbrains.kotlin.library.uniqueName @@ -37,22 +36,28 @@ object KonanBinaryInterface { val IrFunction.functionName: String get() = mangler.run { signatureString(compatibleMode = true) } - val IrFunction.symbolName: String get() = funSymbolNameImpl() - val IrField.symbolName: String get() = - withPrefix(MangleConstant.FIELD_PREFIX, fieldSymbolNameImpl()) - val IrClass.typeInfoSymbolName: String get() = - withPrefix(MangleConstant.CLASS_PREFIX, typeInfoSymbolNameImpl()) + val IrFunction.symbolName: String + get() { + require(isExported(this)) { "Asked for symbol name for a private function ${render()}" } + + return funSymbolNameImpl(null) + } + + val IrField.symbolName: String get() = withPrefix(MangleConstant.FIELD_PREFIX, fieldSymbolNameImpl()) + + val IrClass.typeInfoSymbolName: String get() = typeInfoSymbolNameImpl(null) + + fun IrFunction.privateSymbolName(containerName: String): String = funSymbolNameImpl(containerName) + + fun IrClass.privateTypeInfoSymbolName(containerName: String): String = typeInfoSymbolNameImpl(containerName) + fun isExported(declaration: IrDeclaration) = exportChecker.run { check(declaration, SpecialDeclarationType.REGULAR) || declaration.isPlatformSpecificExported() } private fun withPrefix(prefix: String, mangle: String) = "$prefix:$mangle" - private fun IrFunction.funSymbolNameImpl(): String { - if (!isExported(this)) { - throw AssertionError(render()) - } - + private fun IrFunction.funSymbolNameImpl(containerName: String?): String { if (isExternal) { this.externalSymbolOrThrow()?.let { return it @@ -64,7 +69,8 @@ object KonanBinaryInterface { return name // no wrapping currently required } - return withPrefix(MangleConstant.FUN_PREFIX, mangler.run { mangleString(compatibleMode = true) }) + val mangle = mangler.run { mangleString(compatibleMode = true) } + return withPrefix(MangleConstant.FUN_PREFIX, containerName?.plus(".$mangle") ?: mangle) } private fun IrField.fieldSymbolNameImpl(): String { @@ -74,8 +80,9 @@ object KonanBinaryInterface { return "$containingDeclarationPart$name" } - private fun IrClass.typeInfoSymbolNameImpl(): String { - return this.fqNameForIrSerialization.toString() + private fun IrClass.typeInfoSymbolNameImpl(containerName: String?): String { + val fqName = fqNameForIrSerialization.toString() + return withPrefix(MangleConstant.CLASS_PREFIX, containerName?.plus(".$fqName") ?: fqName) } } @@ -117,10 +124,14 @@ fun IrFunction.computeFullName() = parent.fqNameForIrSerialization.child(Name.id fun IrFunction.computeSymbolName() = with(KonanBinaryInterface) { symbolName }.replaceSpecialSymbols() +fun IrFunction.computePrivateSymbolName(containerName: String) = with(KonanBinaryInterface) { privateSymbolName(containerName) }.replaceSpecialSymbols() + fun IrField.computeSymbolName() = with(KonanBinaryInterface) { symbolName }.replaceSpecialSymbols() fun IrClass.computeTypeInfoSymbolName() = with(KonanBinaryInterface) { typeInfoSymbolName }.replaceSpecialSymbols() +fun IrClass.computePrivateTypeInfoSymbolName(containerName: String) = with(KonanBinaryInterface) { privateTypeInfoSymbolName(containerName) }.replaceSpecialSymbols() + private fun String.replaceSpecialSymbols() = // '@' is used for symbol versioning in GCC: https://gcc.gnu.org/wiki/SymbolVersioning. this.replace("@", "__at__") diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index daa054f13ea..3a8fbadab76 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -514,8 +514,8 @@ internal abstract class FunctionGenerationContext( init { irFunction?.let { if (!irFunction.isExported()) { - LLVMSetLinkage(function, LLVMLinkage.LLVMInternalLinkage) - // (Cannot do this before the function body is created). + if (!context.config.producePerFileCache || irFunction !in context.calledFromExportedInlineFunctions) + LLVMSetLinkage(function, LLVMLinkage.LLVMInternalLinkage) // (Cannot do this before the function body is created) } } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 88b36871e8c..edf19feef5e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -7,19 +7,18 @@ package org.jetbrains.kotlin.backend.konan.llvm import kotlinx.cinterop.toKString import llvm.* -import org.jetbrains.kotlin.backend.konan.BoxCache -import org.jetbrains.kotlin.backend.konan.CachedLibraries -import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder +import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleConstant +import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.Context -import org.jetbrains.kotlin.backend.konan.TargetAbiInfo +import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.backend.konan.ir.llvmSymbolOrigin +import org.jetbrains.kotlin.backend.konan.llvm.KonanBinaryInterface.functionName import org.jetbrains.kotlin.descriptors.konan.CompiledKlibModuleOrigin import org.jetbrains.kotlin.descriptors.konan.CurrentKlibModuleOrigin import org.jetbrains.kotlin.descriptors.konan.DeserializedKlibModuleOrigin import org.jetbrains.kotlin.ir.declarations.* -import org.jetbrains.kotlin.ir.util.file -import org.jetbrains.kotlin.ir.util.fqNameForIrSerialization -import org.jetbrains.kotlin.ir.util.isReal +import org.jetbrains.kotlin.ir.declarations.lazy.IrLazyFunction +import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.konan.library.KonanLibrary import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.KotlinLibrary @@ -183,7 +182,14 @@ internal interface ContextUtils : RuntimeAware { } return if (isExternal(this)) { runtime.addedLLVMExternalFunctions.getOrPut(this) { - val proto = LlvmFunctionProto(this, this.computeSymbolName(), this@ContextUtils) + val symbolName = if (KonanBinaryInterface.isExported(this)) { + this.computeSymbolName() + } else { + val containerName = parentClassOrNull?.fqNameForIrSerialization?.asString() + ?: context.irLinker.getExternalDeclarationFileName(this) + this.computePrivateSymbolName(containerName) + } + val proto = LlvmFunctionProto(this, symbolName, this@ContextUtils) context.llvm.externalFunction(proto) } } else { @@ -203,7 +209,13 @@ internal interface ContextUtils : RuntimeAware { val IrClass.typeInfoPtr: ConstPointer get() { return if (isExternal(this)) { - constPointer(importGlobal(this.computeTypeInfoSymbolName(), runtime.typeInfoType, + val typeInfoSymbolName = if (KonanBinaryInterface.isExported(this)) { + this.computeTypeInfoSymbolName() + } else { + this.computePrivateTypeInfoSymbolName(context.irLinker.getExternalDeclarationFileName(this)) + } + + constPointer(importGlobal(typeInfoSymbolName, runtime.typeInfoType, origin = this.llvmSymbolOrigin)) } else { context.llvmDeclarations.forClass(this).typeInfo @@ -379,6 +391,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) : Runti } for (library in immediateBitcodeDependencies) { + if (library == context.config.libraryToCache?.klib) continue val cache = context.config.cachedLibraries.getLibraryCache(library) if (cache != null) { result += library @@ -395,7 +408,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) : Runti val allBitcodeDependencies: List by lazy { val allNonCachedDependencies = context.librariesWithDependencies.filter { - context.config.cachedLibraries.getLibraryCache(it) == null + context.config.cachedLibraries.getLibraryCache(it) == null || it == context.config.libraryToCache?.klib } val set = (allNonCachedDependencies + allCachedBitcodeDependencies).toSet() // This list is used in particular to build the libraries' initializers chain. diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 00fa566c00e..09d416dec2e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -36,6 +36,7 @@ import org.jetbrains.kotlin.konan.ForeignExceptionMode import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.Family import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.uniqueName import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.resolve.descriptorUtil.classId @@ -2816,35 +2817,52 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map - val ctorName = if (library != null) { - library.moduleConstructorName - } else { - context.config.moduleId.moduleConstructorName - } + fun fileCtorName(libraryName: String, fileName: String) = "$libraryName:$fileName".moduleConstructorName - val ctorFunction = addLlvmFunctionWithDefaultAttributes( - context, - context.llvmModule!!, - ctorName, - kVoidFuncType - ) - LLVMSetLinkage(ctorFunction, LLVMLinkage.LLVMExternalLinkage) + fun addCtorFunction(ctorName: String) = + addLlvmFunctionWithDefaultAttributes( + context, + context.llvmModule!!, + ctorName, + kVoidFuncType + ).also { LLVMSetLinkage(it, LLVMLinkage.LLVMExternalLinkage) } + val ctorFunctions = libraries.flatMap { library -> val initializers = libraryToInitializers.getValue(library) - if (library == null || context.llvmModuleSpecification.containsLibrary(library)) { - val otherInitializers = - context.llvm.otherStaticInitializers.takeIf { library == null }.orEmpty() + val ctorName = when { + library == null -> context.config.moduleId.moduleConstructorName + library == context.config.libraryToCache?.klib + && context.config.producePerFileCache -> + fileCtorName(library.uniqueName, context.config.outputFiles.perFileCacheFileName!!) + else -> library.moduleConstructorName + } + if (library == null || context.llvmModuleSpecification.containsLibrary(library)) { + val otherInitializers = context.llvm.otherStaticInitializers.takeIf { library == null }.orEmpty() + + val ctorFunction = addCtorFunction(ctorName) appendStaticInitializers(ctorFunction, initializers + otherInitializers) + + listOf(ctorFunction) } else { + // A cached library. check(initializers.isEmpty()) { "found initializer from ${library.libraryFile}, which is not included into compilation" } - } - ctorFunction + val cache = context.config.cachedLibraries.getLibraryCache(library) + ?: error("Library $library is expected to be cached") + + when (cache.granularity) { + CachedLibraries.Granularity.MODULE -> listOf(addCtorFunction(ctorName)) + CachedLibraries.Granularity.FILE -> { + context.irLinker.klibToModuleDeserializerMap[library]!!.sortedFileIds.map { + addCtorFunction(fileCtorName(library.uniqueName, it)) + } + } + } + } } appendGlobalCtors(ctorFunctions) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt index 6b16c93290c..e4d28f7ed33 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt @@ -7,12 +7,17 @@ package org.jetbrains.kotlin.backend.konan.llvm import kotlinx.cinterop.toCValues import llvm.* +import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleConstant import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic import org.jetbrains.kotlin.backend.konan.ir.* +import org.jetbrains.kotlin.backend.konan.llvm.KonanBinaryInterface.functionName import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.expressions.IrBlockBody +import org.jetbrains.kotlin.ir.expressions.IrConst +import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid @@ -170,7 +175,12 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : val typeInfoSymbolName = if (declaration.isExported()) { declaration.computeTypeInfoSymbolName() } else { - "ktype:$internalName" + if (!context.config.producePerFileCache) + "${MangleConstant.CLASS_PREFIX}:$internalName" + else { + val containerName = (context.config.libraryToCache!!.strategy as CacheDeserializationStrategy.SingleFile).filePath + declaration.computePrivateTypeInfoSymbolName(containerName) + } } if (declaration.typeInfoHasVtableAttached) { @@ -196,7 +206,8 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : throw IllegalArgumentException("Global '$typeInfoSymbolName' already exists") } } else { - LLVMSetLinkage(llvmTypeInfoPtr, LLVMLinkage.LLVMInternalLinkage) + if (!context.config.producePerFileCache || declaration !in context.constructedFromExportedInlineFunctions) + LLVMSetLinkage(llvmTypeInfoPtr, LLVMLinkage.LLVMInternalLinkage) } typeInfoPtr = constPointer(llvmTypeInfoPtr) @@ -365,8 +376,15 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : } } } else { - "kfun:" + qualifyInternalName(declaration) + if (!context.config.producePerFileCache) + "${MangleConstant.FUN_PREFIX}:${qualifyInternalName(declaration)}" + else { + val containerName = declaration.parentClassOrNull?.fqNameForIrSerialization?.asString() + ?: (context.config.libraryToCache!!.strategy as CacheDeserializationStrategy.SingleFile).filePath + declaration.computePrivateSymbolName(containerName) + } } + val proto = LlvmFunctionProto(declaration, symbolName, this) val llvmFunction = addLlvmFunctionWithDefaultAttributes( context, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objcexport/ObjCExportCodeGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objcexport/ObjCExportCodeGenerator.kt index b5171db6239..2c8f9ab2b6b 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objcexport/ObjCExportCodeGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objcexport/ObjCExportCodeGenerator.kt @@ -287,7 +287,7 @@ internal class ObjCExportBlockCodeGenerator(codegen: CodeGenerator) : ObjCExport // 1. Enumerates [BuiltInFictitiousFunctionIrClassFactory] built classes, which may be incomplete otherwise. // 2. Modifies stdlib global initializers. // 3. Defines runtime-declared globals. - require(context.producedLlvmModuleContainsStdlib) + require(context.shouldDefineFunctionClasses) } fun generate() { @@ -582,6 +582,8 @@ internal class ObjCExportCodeGenerator( unreachable() } + LLVMSetLinkage(imp, LLVMLinkage.LLVMInternalLinkage) + val methods = selectorsToDefine.map { (selector, bridge) -> ObjCDataGenerator.Method(selector, getEncoding(bridge), constPointer(imp)) } @@ -858,7 +860,7 @@ private val ObjCExportBlockCodeGenerator.mappedFunctionNClasses get() = .filter { it.descriptor.isMappedFunctionClass() } private fun ObjCExportBlockCodeGenerator.emitFunctionConverters() { - require(context.producedLlvmModuleContainsStdlib) + require(context.shouldDefineFunctionClasses) mappedFunctionNClasses.forEach { functionClass -> val convertToRetained = kotlinFunctionToRetainedBlockConverter(BlockPointerBridge(functionClass.arity, returnsVoid = false)) @@ -868,7 +870,7 @@ private fun ObjCExportBlockCodeGenerator.emitFunctionConverters() { } private fun ObjCExportBlockCodeGenerator.emitBlockToKotlinFunctionConverters() { - require(context.producedLlvmModuleContainsStdlib) + require(context.shouldDefineFunctionClasses) val functionClassesByArity = mappedFunctionNClasses.associateBy { it.arity } val arityLimit = (functionClassesByArity.keys.maxOrNull() ?: -1) + 1 diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt new file mode 100644 index 00000000000..92a1b7716f2 --- /dev/null +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.konan.lower + +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.backend.konan.DECLARATION_ORIGIN_FUNCTION_CLASS +import org.jetbrains.kotlin.backend.konan.serialization.KonanIrLinker +import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerIr +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.expressions.IrCall +import org.jetbrains.kotlin.ir.expressions.IrConstructorCall +import org.jetbrains.kotlin.ir.expressions.IrFunctionReference +import org.jetbrains.kotlin.ir.expressions.IrPropertyReference +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid + +internal class CacheInfoBuilder(private val context: Context, private val moduleDeserializer: KonanIrLinker.KonanPartialModuleDeserializer) { + fun build() = with(moduleDeserializer) { + moduleFragment.acceptChildrenVoid(object : IrElementVisitorVoid { + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) + } + + override fun visitClass(declaration: IrClass) { + declaration.acceptChildrenVoid(this) + + if (!declaration.isInterface && !declaration.isLocal + && declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS + ) { + context.classFields.add(buildClassFields(declaration, context.getLayoutBuilder(declaration).getDeclaredFields())) + } + } + + override fun visitFunction(declaration: IrFunction) { + // Don't need to visit the children here: classes would be all local and wouldn't be handled anyway, + // as for functions - both their callees will be handled and inline bodies will be built for the top function. + + if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) { + context.inlineFunctionBodies.add(buildInlineFunctionReference(declaration)) + trackCallees(declaration) + } + } + }) + } + + private val IrDeclaration.isExported + get() = with(KonanManglerIr) { isExported(compatibleMode = moduleDeserializer.compatibilityMode.oldSignatures) } + + private val visitedInlineFunctions = mutableSetOf() + + private fun trackCallees(irFunction: IrFunction) { + if (irFunction in visitedInlineFunctions) return + visitedInlineFunctions += irFunction + + irFunction.acceptChildrenVoid(object : IrElementVisitorVoid { + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) + } + + private fun processFunction(function: IrFunction) { + if (function.getPackageFragment() !is IrExternalPackageFragment) { + context.calledFromExportedInlineFunctions.add(function) + (function as? IrConstructor)?.constructedClass?.let { context.constructedFromExportedInlineFunctions.add(it) } + if (function.isInline && !function.isExported) { + // An exported inline function calls a non-exported inline function: + // should track its callees as well as it won't be handled by the main visitor. + trackCallees(function) + } + } + } + + override fun visitCall(expression: IrCall) { + expression.acceptChildrenVoid(this) + + processFunction(expression.symbol.owner) + } + + override fun visitConstructorCall(expression: IrConstructorCall) { + expression.acceptChildrenVoid(this) + + processFunction(expression.symbol.owner) + } + + override fun visitFunctionReference(expression: IrFunctionReference) { + expression.acceptChildrenVoid(this) + + processFunction(expression.symbol.owner) + } + + override fun visitPropertyReference(expression: IrPropertyReference) { + expression.acceptChildrenVoid(this) + + expression.getter?.owner?.let { processFunction(it) } + expression.setter?.owner?.let { processFunction(it) } + } + }) + } +} \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/InteropLowering.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/InteropLowering.kt index 66720bdb0ad..7946ceece67 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/InteropLowering.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/InteropLowering.kt @@ -69,9 +69,10 @@ private abstract class BaseInteropIrTransformer(private val context: Context) : val uniqueModuleName = irFile.packageFragmentDescriptor.module.name.asString() .let { it.substring(1, it.lastIndex) } + val uniqueFileName = irFile.fileEntry.name val uniquePrefix = buildString { append('_') - uniqueModuleName.toByteArray().joinTo(this, "") { + (uniqueModuleName + uniqueFileName).toByteArray().joinTo(this, "") { (0xFF and it.toInt()).toString(16).padStart(2, '0') } append('_') diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/NativeInlineFunctionResolver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/NativeInlineFunctionResolver.kt index 9c253d3243f..c3af05ff1f9 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/NativeInlineFunctionResolver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/NativeInlineFunctionResolver.kt @@ -7,20 +7,16 @@ package org.jetbrains.kotlin.backend.konan.lower import org.jetbrains.kotlin.backend.common.DeclarationTransformer import org.jetbrains.kotlin.backend.common.lower.* -import org.jetbrains.kotlin.backend.common.lower.inline.DefaultInlineFunctionResolver -import org.jetbrains.kotlin.backend.common.lower.inline.LocalClassesExtractionFromInlineFunctionsLowering -import org.jetbrains.kotlin.backend.common.lower.inline.LocalClassesInInlineFunctionsLowering -import org.jetbrains.kotlin.backend.common.lower.inline.LocalClassesInInlineLambdasLowering +import org.jetbrains.kotlin.backend.common.lower.inline.* import org.jetbrains.kotlin.backend.konan.Context import org.jetbrains.kotlin.backend.konan.InlineFunctionOriginInfo import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.ir.symbols.IrClassSymbol import org.jetbrains.kotlin.ir.symbols.IrFunctionSymbol -import org.jetbrains.kotlin.ir.util.dump -import org.jetbrains.kotlin.ir.util.file -import org.jetbrains.kotlin.ir.util.render -import org.jetbrains.kotlin.ir.util.getPackageFragment +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.name.Name // TODO: This is a bit hacky. Think about adopting persistent IR ideas. internal class NativeInlineFunctionResolver(override val context: Context) : DefaultInlineFunctionResolver(context) { @@ -40,6 +36,9 @@ internal class NativeInlineFunctionResolver(override val context: Context) : Def val moduleDescriptor = packageFragment.packageFragmentDescriptor.containingDeclaration val moduleDeserializer = context.irLinker.moduleDeserializers[moduleDescriptor] ?: error("No module deserializer for ${function.render()}") + require(context.config.cachedLibraries.isLibraryCached(moduleDeserializer.klib)) { + "No IR and no cache for ${function.render()}" + } context.specialDeclarationsFactory.loweredInlineFunctions[function] = moduleDeserializer.deserializeInlineFunction(function) function } @@ -60,7 +59,7 @@ internal class NativeInlineFunctionResolver(override val context: Context) : Def LocalClassesInInlineLambdasLowering(context).lower(body, notLoweredFunction) - if (context.llvmModuleSpecification.containsPackageFragment(packageFragment)) { + if (packageFragment !is IrExternalPackageFragment) { // Do not extract local classes off of inline functions from cached libraries. LocalClassesInInlineFunctionsLowering(context).lower(body, notLoweredFunction) LocalClassesExtractionFromInlineFunctionsLowering(context).lower(body, notLoweredFunction) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt index 52e44691192..1d2059017a2 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt @@ -79,7 +79,7 @@ internal class ObjCExport(val context: Context, symbolTable: SymbolTable) { internal fun generate(codegen: CodeGenerator) { if (!target.family.isAppleFamily) return - if (context.producedLlvmModuleContainsStdlib) { + if (context.shouldDefineFunctionClasses) { ObjCExportBlockCodeGenerator(codegen).generate() } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt index b6492bf1d5e..63f14543fe5 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt @@ -122,8 +122,7 @@ internal object InlineFunctionBodyReferenceSerializer { return stream.buf } - fun deserialize(data: ByteArray): List { - val result = mutableListOf() + fun deserializeTo(data: ByteArray, result: MutableList) { val stream = ByteArrayStream(data) while (stream.hasData()) { val file = stream.readInt() @@ -144,7 +143,6 @@ internal object InlineFunctionBodyReferenceSerializer { result.add(SerializedInlineFunctionReference(file, functionSignature, body, startOffset, endOffset, extensionReceiverSig, dispatchReceiverSig, outerReceiverSigs, valueParameterSigs, typeParameterSigs, defaultValues)) } - return result } } @@ -180,8 +178,7 @@ internal object ClassFieldsSerializer { return stream.buf } - fun deserialize(data: ByteArray): List { - val result = mutableListOf() + fun deserializeTo(data: ByteArray, result: MutableList) { val stream = ByteArrayStream(data) while (stream.hasData()) { val file = stream.readInt() @@ -199,7 +196,6 @@ internal object ClassFieldsSerializer { } result.add(SerializedClassFields(file, classSignature, typeParameterSigs, outerThisIndex, fields)) } - return result } } @@ -345,6 +341,7 @@ internal class KonanIrLinker( FakeOverrideBuilder(this, symbolTable, KonanManglerIr, IrTypeSystemContextImpl(builtIns), friendModules, KonanFakeOverrideClassFilter) val moduleDeserializers = mutableMapOf() + val klibToModuleDeserializerMap = mutableMapOf() override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: KotlinLibrary?, strategyResolver: (String) -> DeserializationStrategy) = when { @@ -365,6 +362,7 @@ internal class KonanIrLinker( } KonanPartialModuleDeserializer(moduleDescriptor, klib, strategyResolver, deserializationStrategy).also { moduleDeserializers[moduleDescriptor] = it + klibToModuleDeserializerMap[klib] = it } } } @@ -383,6 +381,22 @@ internal class KonanIrLinker( ?: error("Unknown external package fragment: ${packageFragment.packageFragmentDescriptor}") } + private tailrec fun IdSignature.fileSignature(): IdSignature.FileSignature? = when (this) { + is IdSignature.FileSignature -> this + is IdSignature.CompositeSignature -> this.container.fileSignature() + else -> null + } + + fun getExternalDeclarationFileName(declaration: IrDeclaration) = with(declaration) { + val externalPackageFragment = getPackageFragment() as? IrExternalPackageFragment + ?: error("Expected an external package fragment for ${render()}") + val moduleDescriptor = externalPackageFragment.packageFragmentDescriptor.containingDeclaration + val moduleDeserializer = moduleDeserializers[moduleDescriptor] + ?: error("No module deserializer for $moduleDescriptor") + val idSig = moduleDeserializer.descriptorSignatures[descriptor] ?: error("No signature for $descriptor") + idSig.topLevelSignature().fileSignature()?.fileName ?: error("No file for $idSig") + } + private val IrClass.firstNonClassParent: IrDeclarationParent get() { var parent = parent @@ -414,7 +428,8 @@ internal class KonanIrLinker( if (cacheDeserializationStrategy.contains(fileName)) strategyResolver(fileName) else DeserializationStrategy.ON_DEMAND - }, klib.versions.abiVersion ?: KotlinAbiVersion.CURRENT, containsErrorCode) { + }, klib.versions.abiVersion ?: KotlinAbiVersion.CURRENT, containsErrorCode + ) { override val moduleFragment: IrModuleFragment = KonanIrModuleFragmentImpl(moduleDescriptor, builtIns) fun buildInlineFunctionReference(irFunction: IrFunction): SerializedInlineFunctionReference { @@ -616,7 +631,8 @@ internal class KonanIrLinker( override fun deserializedSymbolNotFound(idSig: IdSignature): Nothing = error("No descriptor found for $idSig") private val inlineFunctionReferences by lazy { - cachedLibraries.getLibraryCache(klib)!!.serializedInlineFunctionBodies.associateBy { + (cachedLibraries.getLibraryCache(klib) + ?: error("No cache for ${klib.libraryName}")).serializedInlineFunctionBodies.associateBy { fileDeserializationStates[it.file].declarationDeserializer.symbolDeserializer.deserializeIdSignature(it.functionSignature) } } @@ -704,12 +720,15 @@ internal class KonanIrLinker( } private val classesFields by lazy { - cachedLibraries.getLibraryCache(klib)!!.serializedClassFields.associateBy { + (cachedLibraries.getLibraryCache(klib) + ?: error("No cache for ${klib.libraryName}")).serializedClassFields.associateBy { fileDeserializationStates[it.file].declarationDeserializer.symbolDeserializer.deserializeIdSignature(it.classSignature) } } fun deserializeClassFields(irClass: IrClass, outerThisField: IrField?): List { + irClass.getPackageFragment() as? IrExternalPackageFragment + ?: error("Expected an external package fragment for ${irClass.render()}") val signature = irClass.symbol.signature ?: error("No signature for ${irClass.render()}") val serializedClassFields = classesFields[signature] @@ -730,6 +749,9 @@ internal class KonanIrLinker( referenceIrSymbol(symbolDeserializer, sigIndex, parameter.symbol) } } + require(endToEndTypeParameterIndex == serializedClassFields.typeParameterSigs.size) { + "Not all type parameters have been referenced" + } fun getByClassId(classId: ClassId): IrClassSymbol { val classIdSig = getPublicSignature(classId.packageFqName, classId.relativeClassName.asString()) @@ -764,6 +786,12 @@ internal class KonanIrLinker( } } } + + val sortedFileIds by lazy { + fileDeserializationStates + .sortedBy { it.file.fileEntry.name } + .map { CacheSupport.cacheFileId(it.file.fqName.asString(), it.file.fileEntry.name) } + } } private inner class KonanInteropModuleDeserializer( diff --git a/kotlin-native/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt b/kotlin-native/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt index 8782b3f4cc9..65f30e80259 100644 --- a/kotlin-native/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt +++ b/kotlin-native/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt @@ -299,7 +299,7 @@ private fun getLibraryCacheDir( ): File { val cacheBaseName = CachedLibraries.getCachedLibraryName(libraryName) val cacheOutputKind = CompilerOutputKind.valueOf(cacheKind.uppercase()) - return OutputFiles(cacheDirectory.child(cacheBaseName).absolutePath, target, cacheOutputKind).mainFile.File() + return OutputFiles(cacheDirectory.child(cacheBaseName).absolutePath, target, cacheOutputKind, producePerFileCache = false).mainFile } private fun buildCache(