From e9720b7e1355195011d94e9f9ece9e3733c62974 Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Fri, 8 Sep 2017 14:34:56 +0300 Subject: [PATCH] Allow including external binaries within klibs with -includeBinary flag. Link such binaries in case they are javascripts on wasm32. Otherwise don't do anything useful yet. --- .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 2 ++ .../cli/bc/K2NativeCompilerArguments.kt | 5 +++- .../kotlin/backend/konan/KonanConfig.kt | 3 +++ .../backend/konan/KonanConfigurationKeys.kt | 2 ++ .../kotlin/backend/konan/LinkStage.kt | 23 +++++++++++++++++-- .../backend/konan/library/KonanLibrary.kt | 2 ++ .../konan/library/KonanLibraryReader.kt | 1 + .../konan/library/KonanLibraryWriter.kt | 1 + .../konan/library/impl/KonanLibrary.kt | 4 ++++ .../library/impl/KonanLibraryReaderImpl.kt | 3 +++ .../library/impl/KonanLibraryWriterImpl.kt | 10 ++++++++ .../kotlin/backend/konan/llvm/IrToBitcode.kt | 1 + 12 files changed, 54 insertions(+), 3 deletions(-) diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 99d8c80c5e3..74cb4569740 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -87,6 +87,8 @@ class K2Native : CLICompiler() { arguments.target ?.let{ put(TARGET, it) } + put(INCLUDED_BINARY_FILES, + arguments.includeBinaries.toNonNullList()) put(NATIVE_LIBRARY_FILES, arguments.nativeLibraries.toNonNullList()) put(REPOSITORIES, diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt index b790f0e4d12..d5cd95e640b 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt @@ -30,6 +30,9 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { @Argument(value = "-enable_assertions", shortName = "-ea", description = "Enable runtime assertions in generated code") var enableAssertions: Boolean = false + @Argument(value = "-includeBinary", shortName = "-ib", valueDescription = "", description = "Pack external binary within the klib") + var includeBinaries: Array? = null + @Argument(value = "-library", shortName = "-l", valueDescription = "", description = "Link with the library") var libraries: Array? = null @@ -39,7 +42,7 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { @Argument(value = "-manifest", valueDescription = "", description = "Provide a maniferst addend file") var manifestFile: String? = null - @Argument(value = "-nativelibrary", shortName = "-nl", valueDescription = "", description = "Include the native library") + @Argument(value = "-nativelibrary", shortName = "-nl", valueDescription = "", description = "Include the native bitcode library") var nativeLibraries: Array? = null @Argument(value = "-nomain", description = "Assume 'main' entry point to be provided by external libraries") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 648084e2002..eb34fc317d0 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -98,6 +98,9 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val nativeLibraries: List = configuration.getList(KonanConfigKeys.NATIVE_LIBRARY_FILES) + internal val includeBinaries: List = + configuration.getList(KonanConfigKeys.INCLUDED_BINARY_FILES) + fun loadLibMetadata(): List { val allMetadata = mutableListOf() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index 493a462da38..51291ef3fc2 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -37,6 +37,8 @@ class KonanConfigKeys { = CompilerConfigurationKey.create("enable backend phases") val ENTRY: CompilerConfigurationKey = CompilerConfigurationKey.create("fully qualified main() name") + val INCLUDED_BINARY_FILES: CompilerConfigurationKey> + = CompilerConfigurationKey.create("included binary file paths") val LIBRARY_FILES: CompilerConfigurationKey> = CompilerConfigurationKey.create("library file paths") val LINKER_ARGS: CompilerConfigurationKey> diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt index d3c1ca31dfb..380d8a22b0a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt @@ -292,7 +292,7 @@ internal class LinkStage(val context: Context) { private val entryPointSelector: List get() = if (nomain) emptyList() else platform.entrySelector - private fun link(objectFiles: List, libraryProvidedLinkerFlags: List): ExecutableFile? { + private fun link(objectFiles: List, includedBinaries: List, libraryProvidedLinkerFlags: List): ExecutableFile? { val executable = context.config.outputFile val linkCommand = platform.linkCommand(objectFiles, executable, optimize, debug) + @@ -313,9 +313,25 @@ internal class LinkStage(val context: Context) { runTool(*platform.dsymutilDryRunVerboseCommand(executable).toTypedArray()) runTool(*platform.dsymutilCommand(executable).toTypedArray()) } + if (platform is WasmPlatform) { + JavaScriptLinker(includedBinaries.filter{it.isJavaScript}, executable) + } return executable } + private val String.isJavaScript + get() = this.endsWith(".js") + + private fun JavaScriptLinker(jsFiles: List, executable: String): String { + val linkedJavaScript = File("$executable.js") + linkedJavaScript.writeBytes(ByteArray(0)); + + jsFiles.forEach { + linkedJavaScript.appendBytes(File(it).readBytes()) + } + return linkedJavaScript.name + } + private fun executeCommand(vararg command: String): Int { context.log{""} @@ -362,6 +378,9 @@ internal class LinkStage(val context: Context) { val bitcodeFiles = listOf(emitted) + libraries.map{it -> it.bitcodePaths}.flatten() + val includedBinaries = + libraries.map{it -> it.includedPaths}.flatten() + val libraryProvidedLinkerFlags = libraries.map{it -> it.linkerOpts}.flatten() @@ -377,7 +396,7 @@ internal class LinkStage(val context: Context) { ) } phaser.phase(KonanPhase.LINKER) { - link(objectFiles, libraryProvidedLinkerFlags) + link(objectFiles, includedBinaries, libraryProvidedLinkerFlags) } } } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibrary.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibrary.kt index f2db1436989..a3fa2136d45 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibrary.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibrary.kt @@ -37,6 +37,8 @@ interface KonanLibraryLayout { get() = File(targetDir, "kotlin") val nativeDir get() = File(targetDir, "native") + val includedDir + get() = File(targetDir, "included") val linkdataDir get() = File(libDir, "linkdata") val moduleHeaderFile diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt index e7337514119..5a7ccd6cca4 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt @@ -22,6 +22,7 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor interface KonanLibraryReader { val libraryName: String val bitcodePaths: List + val includedPaths: List val linkerOpts: List val escapeAnalysis: ByteArray? fun moduleDescriptor(specifics: LanguageVersionSettings): ModuleDescriptor diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt index 7a63f2adaf0..7a1f5867f06 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt @@ -21,6 +21,7 @@ import llvm.LLVMModuleRef interface KonanLibraryWriter { fun addLinkData(linkData: LinkData) fun addNativeBitcode(library: String) + fun addIncludedBinary(library: String) fun addKotlinBitcode(llvmModule: LLVMModuleRef) fun addManifestAddend(path: String) fun addEscapeAnalysis(escapeAnalysis: ByteArray) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibrary.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibrary.kt index f270db3da74..784a03295fd 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibrary.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibrary.kt @@ -63,6 +63,10 @@ class FileExtractor(zippedLibrary: KonanLibrary): KonanLibrary by zippedLibrary extractDir(super.resourcesDir) } + override val includedDir: File by lazy { + extractDir(super.includedDir) + } + override val kotlinDir: File by lazy { extractDir(super.kotlinDir) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt index 6ebe3eb8d11..8775765c6f9 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt @@ -55,6 +55,9 @@ class LibraryReaderImpl(var libraryFile: File, val currentAbiVersion: Int, val t override val bitcodePaths: List get() = (realFiles.kotlinDir.listFiles + realFiles.nativeDir.listFiles).map{it.absolutePath} + override val includedPaths: List + get() = (realFiles.includedDir.listFiles).map{it.absolutePath} + override val linkerOpts: List get() = manifestProperties.propertyList("linkerOpts", target!!.targetSuffix) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryWriterImpl.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryWriterImpl.kt index be157fa6e06..7b7ef78ce0a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryWriterImpl.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryWriterImpl.kt @@ -56,6 +56,7 @@ class LibraryWriterImpl(override val libDir: File, currentAbiVersion: Int, targetDir.mkdirs() kotlinDir.mkdirs() nativeDir.mkdirs() + includedDir.mkdirs() resourcesDir.mkdirs() manifestProperties.setProperty("abi_version", "$currentAbiVersion") } @@ -76,6 +77,11 @@ class LibraryWriterImpl(override val libDir: File, currentAbiVersion: Int, File(library).copyTo(File(nativeDir, basename)) } + override fun addIncludedBinary(library: String) { + val basename = File(library).name + File(library).copyTo(File(includedDir, basename)) + } + override fun addManifestAddend(path: String) { val properties = File(path).loadProperties() manifestProperties.putAll(properties) @@ -97,6 +103,7 @@ class LibraryWriterImpl(override val libDir: File, currentAbiVersion: Int, internal fun buildLibrary( natives: List, + included: List, linkData: LinkData, abiVersion: Int, target: KonanTarget, @@ -113,6 +120,9 @@ internal fun buildLibrary( natives.forEach { library.addNativeBitcode(it) } + included.forEach { + library.addIncludedBinary(it) + } manifest ?.let { library.addManifestAddend(it) } escapeAnalysis?.let { library.addEscapeAnalysis(it) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 561ddc9470d..90fc7176feb 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -121,6 +121,7 @@ internal fun produceOutput(context: Context) { val library = buildLibrary( context.config.nativeLibraries, + context.config.includeBinaries, context.serializedLinkData!!, abiVersion, target,