diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 431ab5fa132..f262ced9406 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -90,7 +90,8 @@ class K2Native : CLICompiler() { } val K2NativeCompilerArguments.isUsefulWithoutFreeArgs: Boolean - get() = listTargets || listPhases || checkDependencies || !includes.isNullOrEmpty() + get() = listTargets || listPhases || checkDependencies || !includes.isNullOrEmpty() || + !librariesToCache.isNullOrEmpty() fun Array?.toNonNullList(): List { return this?.asList() ?: listOf() @@ -211,6 +212,10 @@ class K2Native : CLICompiler() { put(LIBRARIES_TO_COVER, arguments.coveredLibraries.toNonNullList()) arguments.coverageFile?.let { put(PROFRAW_PATH, it) } put(OBJC_GENERICS, arguments.objcGenerics) + + put(LIBRARIES_TO_CACHE, parseLibrariesToCache(arguments, configuration, outputKind)) + put(CACHE_DIRECTORIES, arguments.cacheDirectories.toNonNullList()) + put(CACHED_LIBRARIES, parseCachedLibraries(arguments, configuration)) } } } @@ -301,5 +306,36 @@ private fun selectIncludes( } } +private fun parseCachedLibraries( + arguments: K2NativeCompilerArguments, + configuration: CompilerConfiguration +): Map = arguments.cachedLibraries?.asList().orEmpty().mapNotNull { + val libraryAndCache = it.split(",") + if (libraryAndCache.size != 2) { + configuration.report( + ERROR, + "incorrect $CACHED_LIBRARY format: expected ',', got '$it'" + ) + null + } else { + libraryAndCache[0] to libraryAndCache[1] + } +}.toMap() + +private fun parseLibrariesToCache( + arguments: K2NativeCompilerArguments, + configuration: CompilerConfiguration, + outputKind: CompilerOutputKind +): List { + val input = arguments.librariesToCache?.asList().orEmpty() + + return if (input.isNotEmpty() && !outputKind.isCache) { + configuration.report(ERROR, "$MAKE_CACHE can't be used when not producing cache") + emptyList() + } else { + input + } +} + fun main(args: Array) = K2Native.main(args) diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt index bc607c91bb9..effb906645c 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt @@ -102,6 +102,22 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { // Make sure to prepend them with -X. // Keep the list lexically sorted. + @Argument( + value = "-Xcache-directory", + valueDescription = "", + description = "Path to the directory containing caches", + delimiter = "" + ) + var cacheDirectories: Array? = null + + @Argument( + value = CACHED_LIBRARY, + valueDescription = ",", + description = "Comma-separated paths of a library and its cache", + delimiter = "" + ) + var cachedLibraries: Array? = null + @Argument(value="-Xcheck-dependencies", deprecatedName = "--check_dependencies", description = "Check dependencies and download the missing ones") var checkDependencies: Boolean = false @@ -139,6 +155,14 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { @Argument(value = "-Xg0", description = "Add light debug information") var lightDebug: Boolean = false + @Argument( + value = MAKE_CACHE, + valueDescription = "", + description = "Path of the library to be compiled to cache", + delimiter = "" + ) + var librariesToCache: Array? = null + @Argument(value = "-Xprint-bitcode", deprecatedName = "--print_bitcode", description = "Print llvm bitcode") var printBitCode: Boolean = false @@ -220,3 +244,5 @@ const val EMBED_BITCODE_FLAG = "-Xembed-bitcode" const val EMBED_BITCODE_MARKER_FLAG = "-Xembed-bitcode-marker" const val STATIC_FRAMEWORK_FLAG = "-Xstatic-framework" const val INCLUDE_ARG = "-Xinclude" +const val CACHED_LIBRARY = "-Xcached-library" +const val MAKE_CACHE = "-Xmake-cache" \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BuiltInFictitiousFunctionIrClassFactory.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BuiltInFictitiousFunctionIrClassFactory.kt index f751d494662..2841a4c1b1d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BuiltInFictitiousFunctionIrClassFactory.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BuiltInFictitiousFunctionIrClassFactory.kt @@ -48,6 +48,16 @@ internal class BuiltInFictitiousFunctionIrClassFactory( class FunctionalInterface(val irClass: IrClass, val arity: Int) + fun buildAllClasses() { + val maxArity = 255 // See [BuiltInFictitiousFunctionClassFactory]. + (0 .. maxArity).forEach { arity -> + function(arity) + kFunction(arity) + suspendFunction(arity) + kSuspendFunction(arity) + } + } + fun function(n: Int) = buildClass(irBuiltIns.builtIns.getFunction(n) as FunctionClassDescriptor) fun kFunction(n: Int) = buildClass(reflectionTypes.getKFunction(n) as FunctionClassDescriptor) fun suspendFunction(n: Int) = buildClass(irBuiltIns.builtIns.getSuspendFunction(n) as FunctionClassDescriptor) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt new file mode 100644 index 00000000000..bb6fe1ff0bb --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt @@ -0,0 +1,93 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.kotlin.backend.konan + +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.target.CompilerOutputKind +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult + +class CacheSupport( + configuration: CompilerConfiguration, + resolvedLibraries: KotlinLibraryResolveResult, + target: KonanTarget, + produce: CompilerOutputKind +) { + private val allLibraries = resolvedLibraries.getFullList() + + // TODO: consider using [FeaturedLibraries.kt]. + private val fileToLibrary = allLibraries.associateBy { it.libraryFile } + + internal val cachedLibraries: CachedLibraries = run { + val explicitCacheFiles = configuration.get(KonanConfigKeys.CACHED_LIBRARIES)!! + + val explicitCaches = explicitCacheFiles.entries.associate { (libraryPath, cachePath) -> + val library = fileToLibrary[File(libraryPath)] + ?: configuration.reportCompilationError("cache not applied: library $libraryPath in $cachePath") + + library to cachePath + } + + val implicitCacheDirectories = configuration.get(KonanConfigKeys.CACHE_DIRECTORIES)!! + .map { + File(it).takeIf { it.isDirectory } + ?: configuration.reportCompilationError("cache directory $it is not found or not a directory") + } + + CachedLibraries( + target = target, + allLibraries = allLibraries, + explicitCaches = explicitCaches, + implicitCacheDirectories = implicitCacheDirectories + ) + } + + internal val librariesToCache: Set = configuration.get(KonanConfigKeys.LIBRARIES_TO_CACHE)!! + .map { File(it) }.map { + fileToLibrary[it] ?: error("library to cache\n" + + " ${it.absolutePath}\n" + + "not found among resolved libraries:\n " + + allLibraries.joinToString("\n ") { it.libraryFile.absolutePath }) + }.toSet() + .also { if (!produce.isCache) check(it.isEmpty()) } + + init { + // Ensure dependencies of every cached library are cached too: + resolvedLibraries.getFullList { libraries -> + libraries.map { library -> + val cache = cachedLibraries.getLibraryCache(library.library) + if (cache != null || library.library in librariesToCache) { + library.resolvedDependencies.forEach { + if (!cachedLibraries.isLibraryCached(it.library) && it.library !in librariesToCache) { + val description = if (cache != null) { + "cached (in ${cache.path})" + } else { + "going to be cached" + } + configuration.reportCompilationError( + "${library.library.libraryName} is $description, " + + "but its dependency isn't: ${it.library.libraryName}" + ) + } + } + } + + library + } + } + + // Ensure not making cache for libraries that are already cached: + librariesToCache.forEach { + val cache = cachedLibraries.getLibraryCache(it) + if (cache != null) { + configuration.reportCompilationError("Can't cache library '${it.libraryName}' " + + "that is already cached in '${cache.path}'") + } + } + } +} \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt new file mode 100644 index 00000000000..10b8551677b --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.kotlin.backend.konan + +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.target.CompilerOutputKind +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.utils.addToStdlib.firstNotNullResult + +internal class CachedLibraries( + private val target: KonanTarget, + allLibraries: List, + explicitCaches: Map, + implicitCacheDirectories: List +) { + + class Cache(val kind: Kind, val path: String) { + enum class Kind { DYNAMIC, STATIC } + } + + private val allCaches: Map = allLibraries.mapNotNull { library -> + val explicitPath = explicitCaches[library] + + val cache = if (explicitPath != null) { + val kind = when { + explicitPath.endsWith(target.family.dynamicSuffix) -> Cache.Kind.DYNAMIC + explicitPath.endsWith(target.family.staticSuffix) -> Cache.Kind.STATIC + else -> error("unexpected cache: $explicitPath") + } + Cache(kind, explicitPath) + } else { + implicitCacheDirectories.firstNotNullResult { dir -> + val baseName = "${library.uniqueName}-cache" + val dynamicFile = dir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE)) + val staticFile = dir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE)) + + when { + dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath) + staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath) + else -> null + } + } + } + + cache?.let { library to it } + }.toMap() + + private fun getArtifactName(baseName: String, kind: CompilerOutputKind) = + "${kind.prefix(target)}$baseName${kind.suffix(target)}" + + fun isLibraryCached(library: KotlinLibrary): Boolean = + getLibraryCache(library) != null + + fun getLibraryCache(library: KotlinLibrary): Cache? = + allCaches[library] + + val hasStaticCaches = allCaches.values.any { + when (it.kind) { + Cache.Kind.STATIC -> true + Cache.Kind.DYNAMIC -> false + } + } +} \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt index c5501fa09fc..900c8db801a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt @@ -40,6 +40,9 @@ val CompilerOutputKind.involvesLinkStage: Boolean CompilerOutputKind.LIBRARY, CompilerOutputKind.BITCODE -> false } +val CompilerOutputKind.isCache: Boolean + get() = (this == CompilerOutputKind.STATIC_CACHE || this == CompilerOutputKind.DYNAMIC_CACHE) + internal fun produceCStubs(context: Context) { val llvmModule = context.llvmModule!! context.cStubsManager.compile(context.config.clang, context.messageCollector, context.inVerbosePhase)?.let { @@ -96,6 +99,8 @@ internal fun produceOutput(context: Context) { CompilerOutputKind.STATIC, CompilerOutputKind.DYNAMIC, CompilerOutputKind.FRAMEWORK, + CompilerOutputKind.DYNAMIC_CACHE, + CompilerOutputKind.STATIC_CACHE, CompilerOutputKind.PROGRAM -> { val output = tempFiles.nativeBinaryFileName context.bitcodeFileName = output diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index 1b5297d7f33..232eb9709ad 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -51,7 +51,6 @@ import org.jetbrains.kotlin.backend.common.serialization.KotlinMangler import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport import org.jetbrains.kotlin.backend.konan.llvm.coverage.CoverageManager import org.jetbrains.kotlin.ir.symbols.impl.IrTypeParameterSymbolImpl -import org.jetbrains.kotlin.konan.library.KonanLibrary import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.konan.library.KonanLibraryLayout import org.jetbrains.kotlin.library.SerializedIrModule @@ -465,14 +464,11 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { lateinit var compilerOutput: List - val llvmModuleSpecification: LlvmModuleSpecification = object : LlvmModuleSpecification { - // Currently all code is compiled to single LLVM module. - override fun importsKotlinDeclarationsFromOtherObjectFiles(): Boolean = false - override fun containsLibrary(library: KonanLibrary): Boolean = true - override fun containsModule(module: ModuleDescriptor): Boolean = true - override fun containsModule(module: IrModuleFragment): Boolean = true - override fun containsDeclaration(declaration: IrDeclaration): Boolean = true - } + val llvmModuleSpecification: LlvmModuleSpecification = LlvmModuleSpecificationImpl( + config.cachedLibraries, + producingCache = config.produce.isCache, + librariesToCache = config.librariesToCache + ) } private fun MemberScope.getContributedClassifier(name: String) = diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 2c07c9d2a25..5109d634e00 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -25,6 +25,7 @@ import org.jetbrains.kotlin.util.Logger import kotlin.system.exitProcess import org.jetbrains.kotlin.library.toUnresolvedLibraries import org.jetbrains.kotlin.konan.KonanVersion +import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.resolver.impl.libraryResolver import org.jetbrains.kotlin.library.UnresolvedLibrary @@ -42,6 +43,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration val infoArgsOnly = configuration.kotlinSourceRoots.isEmpty() && configuration[KonanConfigKeys.INCLUDED_LIBRARIES].isNullOrEmpty() + && configuration[KonanConfigKeys.LIBRARIES_TO_CACHE].isNullOrEmpty() // TODO: debug info generation mode and debug/release variant selection probably requires some refactoring. val debug: Boolean get() = configuration.getBoolean(KonanConfigKeys.DEBUG) @@ -91,6 +93,9 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration private val includedLibraryFiles get() = configuration.getList(KonanConfigKeys.INCLUDED_LIBRARIES).map { File(it) } + private val librariesToCacheFiles + get() = configuration.getList(KonanConfigKeys.LIBRARIES_TO_CACHE).map { File(it) } + private val unresolvedLibraries = libraryNames.toUnresolvedLibraries private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES) @@ -124,8 +129,9 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration // But currently the resolver is in the middle of a complex refactoring so it was decided to avoid changes in its logic. // TODO: Handle included libraries in KonanLibraryResolver when it's refactored and moved into the big Kotlin repo. internal val resolvedLibraries by lazy { + val additionalLibraryFiles = includedLibraryFiles + librariesToCacheFiles resolver.resolveWithDependencies( - unresolvedLibraries + includedLibraryFiles.map { UnresolvedLibrary(it.absolutePath, null) }, + unresolvedLibraries + additionalLibraryFiles.map { UnresolvedLibrary(it.absolutePath, null) }, noStdLib = configuration.getBoolean(KonanConfigKeys.NOSTDLIB), noDefaultLibs = configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS), noEndorsedLibs = configuration.getBoolean(KonanConfigKeys.NOENDORSEDLIBS) @@ -144,6 +150,16 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration getIncludedLibraries(includedLibraryFiles, configuration, resolvedLibraries) } + internal val cacheSupport: CacheSupport by lazy { + CacheSupport(configuration, resolvedLibraries, target, produce) + } + + internal val cachedLibraries: CachedLibraries + get() = cacheSupport.cachedLibraries + + internal val librariesToCache: Set + get() = cacheSupport.librariesToCache + fun librariesWithDependencies(moduleDescriptor: ModuleDescriptor?): List { if (moduleDescriptor == null) error("purgeUnneeded() only works correctly after resolve is over, and we have successfully marked package files as needed or not needed.") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index e07d243d236..822e99d4814 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -32,6 +32,12 @@ class KonanConfigKeys { = CompilerConfigurationKey.create("fully qualified main() name") val EXPORTED_LIBRARIES: CompilerConfigurationKey> = CompilerConfigurationKey.create>("libraries included into produced framework API") + val LIBRARIES_TO_CACHE: CompilerConfigurationKey> + = CompilerConfigurationKey.create>("paths to libraries that to be compiled to cache") + val CACHE_DIRECTORIES: CompilerConfigurationKey> + = CompilerConfigurationKey.create>("paths to directories containing caches") + val CACHED_LIBRARIES: CompilerConfigurationKey> + = CompilerConfigurationKey.create>("mapping from library paths to cache paths") val FRAMEWORK_IMPORT_HEADERS: CompilerConfigurationKey> = CompilerConfigurationKey.create>("headers imported to framework header") val FRIEND_MODULES: CompilerConfigurationKey> diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt index 8b888565449..400b95d5748 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt @@ -4,7 +4,6 @@ import org.jetbrains.kotlin.konan.KonanExternalToolFailure import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.Family -import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.LinkerOutputKind internal fun determineLinkerOutput(context: Context): LinkerOutputKind = @@ -13,7 +12,9 @@ internal fun determineLinkerOutput(context: Context): LinkerOutputKind = val staticFramework = context.config.produceStaticFramework if (staticFramework) LinkerOutputKind.STATIC_LIBRARY else LinkerOutputKind.DYNAMIC_LIBRARY } + CompilerOutputKind.DYNAMIC_CACHE, CompilerOutputKind.DYNAMIC -> LinkerOutputKind.DYNAMIC_LIBRARY + CompilerOutputKind.STATIC_CACHE, CompilerOutputKind.STATIC -> LinkerOutputKind.STATIC_LIBRARY CompilerOutputKind.PROGRAM -> LinkerOutputKind.EXECUTABLE else -> TODO("${context.config.produce} should not reach native linker stage") @@ -81,12 +82,16 @@ internal class Linker(val context: Context) { val needsProfileLibrary = context.coverage.enabled + val caches = determineCachesToLink(context) + try { File(executable).delete() linker.linkCommands(objectFiles = objectFiles, executable = executable, - libraries = linker.linkStaticLibraries(includedBinaries) + context.config.defaultSystemLibraries, + libraries = linker.linkStaticLibraries(includedBinaries) + context.config.defaultSystemLibraries + + caches.static.takeIf { context.config.produce != CompilerOutputKind.STATIC_CACHE }.orEmpty(), linkerArgs = asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) + BitcodeEmbedding.getLinkerOptions(context.config) + + caches.dynamic + libraryProvidedLinkerFlags + frameworkLinkerArgs, optimize = optimize, debug = debug, kind = linkerOutput, outputDsymBundle = context.config.outputFile + ".dSYM", @@ -100,4 +105,36 @@ internal class Linker(val context: Context) { return executable } -} \ No newline at end of file +} + +private class CachesToLink(val static: List, val dynamic: List) + +private fun determineCachesToLink(context: Context): CachesToLink { + val staticCaches = mutableListOf() + val dynamicCaches = mutableListOf() + + // TODO: suboptimal, see e.g. [LlvmImports]. + context.librariesWithDependencies.forEach { library -> + val currentBinaryContainsLibrary = context.llvmModuleSpecification.containsLibrary(library) + val cache = context.config.cachedLibraries.getLibraryCache(library) + val libraryIsCached = cache != null + + // Consistency check. Generally guaranteed by implementation. + if (currentBinaryContainsLibrary && libraryIsCached) { + error("Library ${library.libraryName} is found in both cache and current binary") + } else if (!currentBinaryContainsLibrary && !libraryIsCached) { + error("Library ${library.libraryName} is not found neither in cache nor in current binary") + } + + if (cache != null) { + val list = when (cache.kind) { + CachedLibraries.Cache.Kind.DYNAMIC -> dynamicCaches + CachedLibraries.Cache.Kind.STATIC -> staticCaches + } + + list += cache.path + } + } + + return CachesToLink(static = staticCaches.distinct(), dynamic = dynamicCaches.distinct()) +} diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecification.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecification.kt index db88aa4a558..22a05f89a94 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecification.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecification.kt @@ -8,14 +8,15 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrModuleFragment -import org.jetbrains.kotlin.konan.library.KonanLibrary +import org.jetbrains.kotlin.library.KotlinLibrary /** * Defines what LLVM module should consist of. */ interface LlvmModuleSpecification { + val isFinal: Boolean fun importsKotlinDeclarationsFromOtherObjectFiles(): Boolean - fun containsLibrary(library: KonanLibrary): Boolean + fun containsLibrary(library: KotlinLibrary): Boolean fun containsModule(module: ModuleDescriptor): Boolean fun containsModule(module: IrModuleFragment): Boolean fun containsDeclaration(declaration: IrDeclaration): Boolean diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt new file mode 100644 index 00000000000..5b7fcd53384 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LlvmModuleSpecificationImpl.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.kotlin.backend.konan + +import org.jetbrains.kotlin.backend.konan.descriptors.konanLibrary +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.util.module +import org.jetbrains.kotlin.library.KotlinLibrary + +// TODO: consider making two implementations: for producing cache and anything else. +internal class LlvmModuleSpecificationImpl( + private val cachedLibraries: CachedLibraries, + private val producingCache: Boolean, + private val librariesToCache: Set +) : LlvmModuleSpecification { + override val isFinal: Boolean + get() = !producingCache + + override fun importsKotlinDeclarationsFromOtherObjectFiles(): Boolean = + cachedLibraries.hasStaticCaches // A bit conservative but still valid. + + override fun containsLibrary(library: KotlinLibrary): Boolean = if (producingCache) { + library in librariesToCache + } else { + !cachedLibraries.isLibraryCached(library) + } + + override fun containsModule(module: IrModuleFragment): Boolean = + containsModule(module.descriptor) + + override fun containsModule(module: ModuleDescriptor): Boolean = + module.konanLibrary.let { it == null || containsLibrary(it) } + + override fun containsDeclaration(declaration: IrDeclaration): Boolean = + declaration.module.konanLibrary.let { it == null || containsLibrary(it) } +} diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OptimizationPipeline.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OptimizationPipeline.kt index 1eb8f595f91..4ff62129b22 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OptimizationPipeline.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OptimizationPipeline.kt @@ -114,7 +114,9 @@ internal fun runClosedWorldCleanup(context: Context) { initializeLlvmGlobalPassRegistry() val llvmModule = context.llvmModule!! val modulePasses = LLVMCreatePassManager() - LLVMAddInternalizePass(modulePasses, 0) + if (context.llvmModuleSpecification.isFinal) { + LLVMAddInternalizePass(modulePasses, 0) + } LLVMAddGlobalDCEPass(modulePasses) LLVMRunPassManager(modulePasses, llvmModule) LLVMDisposePassManager(modulePasses) @@ -149,9 +151,11 @@ internal fun runLlvmOptimizationPipeline(context: Context) { LLVMKotlinAddTargetLibraryInfoWrapperPass(modulePasses, config.targetTriple) // TargetTransformInfo pass. LLVMAddAnalysisPasses(targetMachine, modulePasses) - // Since we are in a "closed world" internalization and global dce - // can be safely used to reduce size of a bitcode. - LLVMAddInternalizePass(modulePasses, 0) + if (context.llvmModuleSpecification.isFinal) { + // Since we are in a "closed world" internalization can be safely used + // to reduce size of a bitcode with global dce. + LLVMAddInternalizePass(modulePasses, 0) + } LLVMAddGlobalDCEPass(modulePasses) config.customInlineThreshold?.let { threshold -> diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Reporting.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Reporting.kt index 0f9ef27b1e4..b38bc7d206f 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Reporting.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Reporting.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.backend.common.CommonBackendContext +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.util.render @@ -20,6 +22,11 @@ internal fun CommonBackendContext.reportCompilationError(message: String): Nothi throw KonanCompilationException() } +internal fun CompilerConfiguration.reportCompilationError(message: String): Nothing { + report(CompilerMessageSeverity.ERROR, message) + throw KonanCompilationException() +} + internal fun CommonBackendContext.reportCompilationWarning(message: String) { report(null, null, message, false) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt index fd969e7e1f6..f47dd2cf255 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt @@ -141,13 +141,20 @@ internal val psiToIrPhase = konanUnitPhase( val forwardDeclarationsModuleDescriptor = moduleDescriptor.allDependencyModules.firstOrNull { it.isForwardDeclarationModule } + val modulesWithoutDCE = moduleDescriptor.allDependencyModules + .filter { !llvmModuleSpecification.isFinal && llvmModuleSpecification.containsModule(it) } + + // Note: using [llvmModuleSpecification] since this phase produces IR for generating single LLVM module. + + val exportedDependencies = (getExportedDependencies() + modulesWithoutDCE).distinct() + val deserializer = KonanIrLinker( moduleDescriptor, this as LoggingContext, generatorContext.irBuiltIns, symbolTable, forwardDeclarationsModuleDescriptor, - getExportedDependencies() + exportedDependencies ) var dependenciesCount = 0 @@ -176,12 +183,16 @@ internal val psiToIrPhase = konanUnitPhase( val module = translator.generateModuleFragment(generatorContext, environment.getSourceFiles(), deserializer, listOf(functionIrClassFactory)) + if (this.stdlibModule in modulesWithoutDCE) { + functionIrClassFactory.buildAllClasses() + } + irModule = module - irModules = deserializer.modules + irModules = deserializer.modules.filterValues { llvmModuleSpecification.containsModule(it) } ir.symbols = symbols functionIrClassFactory.module = - (listOf(irModule!!) + irModules.values) + (listOf(irModule!!) + deserializer.modules.values) .single { it.descriptor.isKonanStdlib() } }, name = "Psi2Ir", diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index b0a3b3d1c0a..b5bfecc2b2f 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -400,9 +400,20 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val bitcodeToLink: List by lazy { (context.config.resolvedLibraries.getFullList(TopologicalLibraryOrder) as List) - .filter { (!it.isDefault && !context.config.purgeUserLibs) || imports.bitcodeIsUsed(it) } - // TODO: the filter above is incorrect when compiling to multiple LLVM modules. - .filter { context.llvmModuleSpecification.containsLibrary(it) } + .filter { shouldContainBitcode(it) } + } + + private fun shouldContainBitcode(library: KonanLibrary): Boolean { + if (!context.llvmModuleSpecification.containsLibrary(library)) { + return false + } + + if (!context.llvmModuleSpecification.isFinal) { + return true + } + + // Apply some DCE: + return (!library.isDefault && !context.config.purgeUserLibs) || imports.bitcodeIsUsed(library) } val additionalProducedBitcodeFiles = mutableListOf()