From bc812499874f5642bc276d351192416151a43c4c Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Fri, 23 Aug 2019 19:59:20 +0700 Subject: [PATCH] Included libraries: Don't resolve included libraries --- .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 16 ++-- .../cli/bc/K2NativeCompilerArguments.kt | 6 +- .../kotlin/backend/konan/FeaturedLibraries.kt | 83 +++++++++++-------- .../kotlin/backend/konan/KonanConfig.kt | 21 +++-- .../backend/konan/KonanConfigurationKeys.kt | 4 +- .../backend/konan/lower/TestProcessor.kt | 8 +- .../backend/konan/objcexport/ObjCExport.kt | 3 +- .../org/jetbrains/kotlin/KonanTest.groovy | 2 +- .../plugin/konan/tasks/KonanCompileTask.kt | 1 - 9 files changed, 82 insertions(+), 62 deletions(-) diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 70358905112..3b003411987 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -90,7 +90,7 @@ class K2Native : CLICompiler() { } val K2NativeCompilerArguments.isUsefulWithoutFreeArgs: Boolean - get() = listTargets || listPhases || checkDependencies || !sourceLibraries.isNullOrEmpty() + get() = listTargets || listPhases || checkDependencies || !includes.isNullOrEmpty() fun Array?.toNonNullList(): List { return this?.asList() ?: listOf() @@ -193,14 +193,14 @@ class K2Native : CLICompiler() { put( CHECK_DEPENDENCIES, configuration.kotlinSourceRoots.isNotEmpty() - || !arguments.sourceLibraries.isNullOrEmpty() + || !arguments.includes.isNullOrEmpty() || arguments.checkDependencies ) if (arguments.friendModules != null) put(FRIEND_MODULES, arguments.friendModules!!.split(File.pathSeparator).filterNot(String::isEmpty)) put(EXPORTED_LIBRARIES, selectExportedLibraries(configuration, arguments, outputKind)) - put(SOURCE_LIBRARIES, selectSourceLibraries(configuration, arguments, outputKind)) + put(INCLUDED_LIBRARIES, selectIncludes(configuration, arguments, outputKind)) put(FRAMEWORK_IMPORT_HEADERS, arguments.frameworkImportHeaders.toNonNullList()) arguments.emitLazyObjCHeader?.let { put(EMIT_LAZY_OBJC_HEADER_FILE, it) } @@ -302,23 +302,23 @@ private fun selectExportedLibraries( } } -private fun selectSourceLibraries( +private fun selectIncludes( configuration: CompilerConfiguration, arguments: K2NativeCompilerArguments, outputKind: CompilerOutputKind ): List { - val sourceLibraries = arguments.sourceLibraries?.toList().orEmpty() + val includes = arguments.includes?.toList().orEmpty() val produceBinaryOrBitcode = outputKind.let { it.isNativeBinary || it == CompilerOutputKind.BITCODE } - return if (sourceLibraries.isNotEmpty() && !produceBinaryOrBitcode) { + return if (includes.isNotEmpty() && !produceBinaryOrBitcode) { configuration.report( ERROR, - "The $SOURCE_LIBRARY_ARG flag is only supported when producing native binaries or bitcode files, " + + "The $INCLUDE_ARG flag is only supported when producing native binaries or bitcode files, " + "but the compiler is producing ${outputKind.name.toLowerCase()}" ) emptyList() } else { - sourceLibraries + includes } } diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt index 818fe628276..be4bf9332f1 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt @@ -158,11 +158,11 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { var runtimeFile: String? = null @Argument( - value = SOURCE_LIBRARY_ARG, + value = INCLUDE_ARG, valueDescription = "", description = "A path to an intermediate library that should be processed in the same manner as source files.\n" ) - var sourceLibraries: Array? = null + var includes: Array? = null @Argument(value = STATIC_FRAMEWORK_FLAG, description = "Create a framework with a static library instead of a dynamic one") var staticFramework: Boolean = false @@ -216,4 +216,4 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { const val EMBED_BITCODE_FLAG = "-Xembed-bitcode" const val EMBED_BITCODE_MARKER_FLAG = "-Xembed-bitcode-marker" const val STATIC_FRAMEWORK_FLAG = "-Xstatic-framework" -const val SOURCE_LIBRARY_ARG = "-Xinclude" +const val INCLUDE_ARG = "-Xinclude" diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FeaturedLibraries.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FeaturedLibraries.kt index f8435696181..218a529782a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FeaturedLibraries.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FeaturedLibraries.kt @@ -19,14 +19,14 @@ import org.jetbrains.kotlin.konan.library.isInterop import org.jetbrains.kotlin.konan.library.resolver.KonanLibraryResolveResult import org.jetbrains.kotlin.library.toUnresolvedLibraries -internal fun Context.getExportedDependencies(): List = getDescriptorsFromFeaturedLibraries((config.exportedLibraries + config.sourceLibraries).toSet()) -internal fun Context.getSourceLibraryDescriptors(): List = getDescriptorsFromFeaturedLibraries(config.sourceLibraries.toSet()) +internal fun Context.getExportedDependencies(): List = getDescriptorsFromLibraries((config.exportedLibraries + config.includedLibraries).toSet()) +internal fun Context.getIncludedLibraryDescriptors(): List = getDescriptorsFromLibraries(config.includedLibraries.toSet()) -private fun Context.getDescriptorsFromFeaturedLibraries(featuredLibraries: Set) = +private fun Context.getDescriptorsFromLibraries(libraries: Set) = moduleDescriptor.allDependencyModules.filter { when (val origin = it.konanModuleOrigin) { CurrentKonanModuleOrigin, SyntheticModulesOrigin -> false - is DeserializedKonanModuleOrigin -> origin.library in featuredLibraries + is DeserializedKonanModuleOrigin -> origin.library in libraries } } @@ -42,15 +42,14 @@ internal fun getExportedLibraries( if (report) FeaturedLibrariesReporter.forExportedLibraries(configuration) else FeaturedLibrariesReporter.Silent ) -internal fun getSourceLibraries( +internal fun getIncludedLibraries( + includedLibraryFiles: List, configuration: CompilerConfiguration, - resolvedLibraries: KonanLibraryResolveResult, - resolver: SearchPathResolver + resolvedLibraries: KonanLibraryResolveResult ): List = getFeaturedLibraries( - configuration.getList(KonanConfigKeys.SOURCE_LIBRARIES), + includedLibraryFiles.toSet(), resolvedLibraries, - resolver, - FeaturedLibrariesReporter.forSourceLibraries(configuration) + FeaturedLibrariesReporter.forIncludedLibraries(configuration) ) internal fun getCoveredLibraries( @@ -69,10 +68,16 @@ private sealed class FeaturedLibrariesReporter { abstract fun reportIllegalKind(library: KonanLibrary) abstract fun reportNotIncludedLibraries(includedLibraries: List, remainingFeaturedLibraries: Set) + protected val KonanLibrary.reportedKind: String + get() = when { + isInterop -> "Interop" + isDefault -> "Default" + else -> "Unknown kind" + } + object Silent: FeaturedLibrariesReporter() { override fun reportIllegalKind(library: KonanLibrary) {} override fun reportNotIncludedLibraries(includedLibraries: List, remainingFeaturedLibraries: Set) {} - } abstract class BaseReporter(val configuration: CompilerConfiguration) : FeaturedLibrariesReporter() { @@ -80,12 +85,10 @@ private sealed class FeaturedLibrariesReporter { protected abstract fun notIncludedLibraryMessageTitle(): String override fun reportIllegalKind(library: KonanLibrary) { - val kind = when { - library.isInterop -> "Interop" - library.isDefault -> "Default" - else -> "Unknown kind" - } - configuration.report(CompilerMessageSeverity.STRONG_WARNING, illegalKindMessage(kind, library.libraryName)) + configuration.report( + CompilerMessageSeverity.STRONG_WARNING, + illegalKindMessage(library.reportedKind, library.libraryName) + ) } override fun reportNotIncludedLibraries(includedLibraries: List, remainingFeaturedLibraries: Set) { @@ -101,6 +104,18 @@ private sealed class FeaturedLibrariesReporter { } } + private class IncludedLibrariesReporter(val configuration: CompilerConfiguration) : FeaturedLibrariesReporter() { + override fun reportIllegalKind(library: KonanLibrary) = with(library) { + val message = "$reportedKind library $libraryName cannot be passed with -Xinclude " + + "(library path: ${libraryFile.absolutePath})" + configuration.report(CompilerMessageSeverity.STRONG_WARNING, message) + } + + override fun reportNotIncludedLibraries(includedLibraries: List, remainingFeaturedLibraries: Set) { + error("An included library is not found among resolved libraries") + } + } + private class ExportedLibrariesReporter(configuration: CompilerConfiguration) : BaseReporter(configuration) { override fun illegalKindMessage(kind: String, libraryName: String): String = "$kind library $libraryName can't be exported with -Xexport-library" @@ -109,14 +124,6 @@ private sealed class FeaturedLibrariesReporter { "Following libraries are specified to be exported with -Xexport-library, but not included to the build:" } - private class SourceLibrariesReporter(configuration: CompilerConfiguration) : BaseReporter(configuration) { - override fun illegalKindMessage(kind: String, libraryName: String): String = - "$kind library $libraryName can't be used as a source library" - - override fun notIncludedLibraryMessageTitle(): String = - "Following libraries are declared as source libraries with -Xinclude, but not included to the build:" - } - private class CoveredLibraryReporter(configuration: CompilerConfiguration): BaseReporter(configuration) { override fun illegalKindMessage(kind: String, libraryName: String): String = "Cannot provide the code coverage for the $kind library $libraryName." @@ -126,23 +133,33 @@ private sealed class FeaturedLibrariesReporter { } companion object { - fun forExportedLibraries(configuration: CompilerConfiguration): FeaturedLibrariesReporter = ExportedLibrariesReporter(configuration) - fun forSourceLibraries(configuration: CompilerConfiguration): FeaturedLibrariesReporter = SourceLibrariesReporter(configuration) - fun forCoveredLibraries(configuration: CompilerConfiguration): FeaturedLibrariesReporter = CoveredLibraryReporter(configuration) + fun forExportedLibraries(configuration: CompilerConfiguration): FeaturedLibrariesReporter = + ExportedLibrariesReporter(configuration) + fun forCoveredLibraries(configuration: CompilerConfiguration): FeaturedLibrariesReporter = + CoveredLibraryReporter(configuration) + fun forIncludedLibraries(configuration: CompilerConfiguration): FeaturedLibrariesReporter = + IncludedLibrariesReporter(configuration) } } private fun getFeaturedLibraries( - featuredLibraries: List, + featuredLibraries: List, + resolvedLibraries: KonanLibraryResolveResult, + resolver: SearchPathResolver, + reporter: FeaturedLibrariesReporter +) = getFeaturedLibraries( + featuredLibraries.toUnresolvedLibraries.map { resolver.resolve(it).libraryFile }.toSet(), + resolvedLibraries, + reporter +) + +private fun getFeaturedLibraries( + featuredLibraryFiles: Set, resolvedLibraries: KonanLibraryResolveResult, - resolver: SearchPathResolver, reporter: FeaturedLibrariesReporter ) : List { - val featuredLibraryFiles = featuredLibraries.toUnresolvedLibraries.map { resolver.resolve(it).libraryFile }.toSet() val remainingFeaturedLibraries = featuredLibraryFiles.toMutableSet() - val result = mutableListOf() - val libraries = resolvedLibraries.getFullList(null) for (library in libraries) { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 9838ad2bb8e..a6d820d03bd 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -26,6 +26,8 @@ import org.jetbrains.kotlin.konan.util.Logger import kotlin.system.exitProcess import org.jetbrains.kotlin.library.toUnresolvedLibraries import org.jetbrains.kotlin.konan.KonanVersion +import org.jetbrains.kotlin.konan.library.isInterop +import org.jetbrains.kotlin.library.UnresolvedLibrary class KonanConfig(val project: Project, val configuration: CompilerConfiguration) { @@ -40,7 +42,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val phaseConfig = configuration.get(CLIConfigurationKeys.PHASE_CONFIG)!! val infoArgsOnly = configuration.kotlinSourceRoots.isEmpty() - && configuration[KonanConfigKeys.SOURCE_LIBRARIES].isNullOrEmpty() + && configuration[KonanConfigKeys.INCLUDED_LIBRARIES].isNullOrEmpty() // TODO: debug info generation mode and debug/release variant selection probably requires some refactoring. val debug: Boolean get() = configuration.getBoolean(KonanConfigKeys.DEBUG) @@ -87,6 +89,9 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration private val libraryNames: List get() = configuration.getList(KonanConfigKeys.LIBRARY_FILES) + private val includedLibraryFiles + get() = configuration.getList(KonanConfigKeys.INCLUDED_LIBRARIES).map { File(it) } + private val unresolvedLibraries = libraryNames.toUnresolvedLibraries private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES) @@ -115,9 +120,13 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration resolverLogger ).libraryResolver() + // We pass included libraries by absolute paths to avoid repository-based resolution for them. + // Strictly speaking such "direct" libraries should be specially handled by the resolver, not by KonanConfig. + // But currently the resolver is in the middle of a complex refactoring so it was decided to avoid changes in its logic. + // TODO: Handle included libraries in KonanLibraryResolver when it's refactored and moved into the big Kotlin repo. internal val resolvedLibraries by lazy { resolver.resolveWithDependencies( - unresolvedLibraries, + unresolvedLibraries + includedLibraryFiles.map { UnresolvedLibrary(it.absolutePath, null) }, noStdLib = configuration.getBoolean(KonanConfigKeys.NOSTDLIB), noDefaultLibs = configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS) ) @@ -127,14 +136,14 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration getExportedLibraries(configuration, resolvedLibraries, resolver.searchPathResolver, report = true) } - internal val sourceLibraries by lazy { - getSourceLibraries(configuration, resolvedLibraries, resolver.searchPathResolver) - } - internal val coveredLibraries by lazy { getCoveredLibraries(configuration, resolvedLibraries, resolver.searchPathResolver) } + internal val includedLibraries by lazy { + getIncludedLibraries(includedLibraryFiles, configuration, resolvedLibraries) + } + fun librariesWithDependencies(moduleDescriptor: ModuleDescriptor?): List { if (moduleDescriptor == null) error("purgeUnneeded() only works correctly after resolve is over, and we have successfully marked package files as needed or not needed.") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index d1c4505d6b0..901607c4a51 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -94,8 +94,8 @@ class KonanConfigKeys { = CompilerConfigurationKey.create("library search path repositories") val RUNTIME_FILE: CompilerConfigurationKey = CompilerConfigurationKey.create("override default runtime file path") - val SOURCE_LIBRARIES: CompilerConfigurationKey> - = CompilerConfigurationKey("libraries used to be processed in the same manner as source files") + val INCLUDED_LIBRARIES: CompilerConfigurationKey> + = CompilerConfigurationKey("klibs processed in the same manner as source files") val SOURCE_MAP: CompilerConfigurationKey> = CompilerConfigurationKey.create("generate source map") val STATIC_FRAMEWORK: CompilerConfigurationKey diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/TestProcessor.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/TestProcessor.kt index 0e5c719371f..ea5e999fdc7 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/TestProcessor.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/TestProcessor.kt @@ -17,15 +17,11 @@ import org.jetbrains.kotlin.backend.common.reportWarning import org.jetbrains.kotlin.backend.konan.Context import org.jetbrains.kotlin.backend.konan.descriptors.isAbstract import org.jetbrains.kotlin.backend.konan.descriptors.synthesizedName -import org.jetbrains.kotlin.backend.konan.getSourceLibraryDescriptors +import org.jetbrains.kotlin.backend.konan.getIncludedLibraryDescriptors import org.jetbrains.kotlin.backend.konan.ir.typeWithStarProjections import org.jetbrains.kotlin.backend.konan.ir.typeWithoutArguments import org.jetbrains.kotlin.backend.konan.reportCompilationError import org.jetbrains.kotlin.descriptors.* -import org.jetbrains.kotlin.descriptors.konan.CurrentKonanModuleOrigin -import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin -import org.jetbrains.kotlin.descriptors.konan.SyntheticModulesOrigin -import org.jetbrains.kotlin.descriptors.konan.konanModuleOrigin import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.builders.* import org.jetbrains.kotlin.ir.declarations.* @@ -607,7 +603,7 @@ internal class TestProcessor (val context: Context) { private fun shouldProcessFile(irFile: IrFile): Boolean = irFile.packageFragmentDescriptor.module.let { // Process test annotations in source libraries too. - it == context.moduleDescriptor || it in context.getSourceLibraryDescriptors() + it == context.moduleDescriptor || it in context.getIncludedLibraryDescriptors() } fun process(irFile: IrFile) { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt index c5ff015adb6..fca07ed1e01 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/objcexport/ObjCExport.kt @@ -14,7 +14,6 @@ import org.jetbrains.kotlin.backend.konan.llvm.CodeGenerator import org.jetbrains.kotlin.backend.konan.llvm.objcexport.ObjCExportCodeGenerator import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor import org.jetbrains.kotlin.descriptors.ClassDescriptor -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.SourceFile import org.jetbrains.kotlin.ir.util.SymbolTable import org.jetbrains.kotlin.konan.exec.Command @@ -275,7 +274,7 @@ internal class ObjCExport(val context: Context, symbolTable: SymbolTable) { } private fun guessMainPackage(): FqName { - val allPackages = (context.getSourceLibraryDescriptors() + context.moduleDescriptor).flatMap { + val allPackages = (context.getIncludedLibraryDescriptors() + context.moduleDescriptor).flatMap { it.getPackageFragments() // Includes also all parent packages, e.g. the root one. } diff --git a/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index 79e8c8e75d8..c9e23136446 100644 --- a/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -622,7 +622,7 @@ fun runTest() { // Two-stage compilation. def klibPath = "${exePath}.klib" runCompiler(compileList, klibPath, flags + ["-p", "library"]) - runCompiler([], exePath, flags + ["-l", klibPath, "-Xinclude=$klibPath"]) + runCompiler([], exePath, flags + ["-Xinclude=$klibPath"]) } else { // Regular compilation. runCompiler(compileList, exePath, flags) diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt index d371ce77572..836b6e5e4cb 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt @@ -183,7 +183,6 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec { addAll(secondStageExtraOpts()) - addArg("-l", klibPath) add("-Xinclude=${klibPath}") }