From e8ad4712d380b1c7950e150624998ddf79c2e940 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Fri, 1 Nov 2019 17:46:12 +0700 Subject: [PATCH] [interop] Prepare interop driver for a metadata-only mode. --- .../kotlin/native/interop/gen/StubIrDriver.kt | 76 ++++++++++++++-- .../native/interop/gen/StubIrTextEmitter.kt | 23 +---- .../native/interop/gen/jvm/CommandLine.kt | 7 ++ .../native/interop/gen/jvm/GenerationMode.kt | 13 +++ .../interop/gen/jvm/InteropLibraryCreation.kt | 37 ++++++++ .../kotlin/native/interop/gen/jvm/main.kt | 91 ++++++++++++------- .../utilities/GeneratePlatformLibraries.kt | 2 +- .../kotlin/cli/utilities/InteropCompiler.kt | 8 +- .../jetbrains/kotlin/cli/utilities/main.kt | 4 +- 9 files changed, 194 insertions(+), 67 deletions(-) create mode 100644 Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/GenerationMode.kt create mode 100644 Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/InteropLibraryCreation.kt diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrDriver.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrDriver.kt index 38da841d11a..7c56697ce4c 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrDriver.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrDriver.kt @@ -4,6 +4,7 @@ */ package org.jetbrains.kotlin.native.interop.gen +import org.jetbrains.kotlin.native.interop.gen.jvm.GenerationMode import org.jetbrains.kotlin.native.interop.gen.jvm.InteropConfiguration import org.jetbrains.kotlin.native.interop.gen.jvm.KotlinPlatform import org.jetbrains.kotlin.native.interop.indexer.* @@ -86,18 +87,75 @@ class StubIrContext( } } -class StubIrDriver(private val context: StubIrContext) { - fun run(outKtFile: File, outCFile: File, entryPoint: String?) { +class StubIrDriver( + private val context: StubIrContext, + private val options: DriverOptions +) { + data class DriverOptions( + val mode: GenerationMode, + val entryPoint: String?, + val outCFile: File, + val outKtFileCreator: () -> File + ) + + sealed class Result { + object SourceCode : Result() + + // Will contain Km* objects. + class Metadata(): Result() + } + + fun run(): Result { + val (mode, entryPoint, outCFile, outKtFile) = options + val builderResult = StubIrBuilder(context).build() val bridgeBuilderResult = StubIrBridgeBuilder(context, builderResult).build() - outKtFile.bufferedWriter().use { ktFile -> - File(outCFile.absolutePath).bufferedWriter().use { cFile -> - StubIrTextEmitter( - context, - builderResult, - bridgeBuilderResult - ).emit(ktFile, cFile, entryPoint) + + outCFile.bufferedWriter().use { + emitCFile(context, it, entryPoint, bridgeBuilderResult.nativeBridges) + } + + return when (mode) { + GenerationMode.SOURCE_CODE -> { + emitSourceCode(outKtFile(), builderResult, bridgeBuilderResult) } + GenerationMode.METADATA -> emitMetadata(builderResult) + } + } + + private fun emitSourceCode( + outKtFile: File, builderResult: StubIrBuilderResult, bridgeBuilderResult: BridgeBuilderResult + ): Result.SourceCode { + outKtFile.bufferedWriter().use { ktFile -> + StubIrTextEmitter(context, builderResult, bridgeBuilderResult).emit(ktFile) + } + return Result.SourceCode + } + + private fun emitMetadata(builderResult: StubIrBuilderResult): Result.Metadata { + return Result.Metadata() + } + + private fun emitCFile(context: StubIrContext, cFile: Appendable, entryPoint: String?, nativeBridges: NativeBridges) { + val out = { it: String -> cFile.appendln(it) } + + context.libraryForCStubs.preambleLines.forEach { + out(it) + } + out("") + + out("// NOTE THIS FILE IS AUTO-GENERATED") + out("") + + nativeBridges.nativeLines.forEach { out(it) } + + if (entryPoint != null) { + out("extern int Konan_main(int argc, char** argv);") + out("") + out("__attribute__((__used__))") + out("int $entryPoint(int argc, char** argv) {") + out(" return Konan_main(argc, argv);") + out("}") } } } \ No newline at end of file diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrTextEmitter.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrTextEmitter.kt index a544321e005..2d9c408f4fe 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrTextEmitter.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrTextEmitter.kt @@ -148,28 +148,7 @@ class StubIrTextEmitter( out("// NOTE THIS FILE IS AUTO-GENERATED") } - fun emit(ktFile: Appendable, cFile: Appendable, entryPoint: String?) { - - withOutput(cFile) { - context.libraryForCStubs.preambleLines.forEach { - out(it) - } - out("") - - out("// NOTE THIS FILE IS AUTO-GENERATED") - out("") - - nativeBridges.nativeLines.forEach(out) - - if (entryPoint != null) { - out("extern int Konan_main(int argc, char** argv);") - out("") - out("__attribute__((__used__))") - out("int $entryPoint(int argc, char** argv) {") - out(" return Konan_main(argc, argv);") - out("}") - } - } + fun emit(ktFile: Appendable) { // Stubs generation may affect imports list so do it before header generation. val stubLines = generateKotlinFragmentBy { diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/CommandLine.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/CommandLine.kt index a6f7b80daa4..53a425fbda6 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/CommandLine.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/CommandLine.kt @@ -48,6 +48,8 @@ open class CommonInteropArguments(val argParser: ArgParser) { .multiple() val repo by argParser.option(ArgType.String, shortName = "r", description = "repository to resolve dependencies") .multiple() + val mode by argParser.option(ArgType.Choice(listOf(MODE_METADATA, MODE_SOURCECODE)), description = "the way interop library is generated") + .default(MODE_SOURCECODE) val nodefaultlibs by argParser.option(ArgType.Boolean, NODEFAULTLIBS, description = "don't link the libraries from dist/klib automatically").default(false) val nodefaultlibsDeprecated by argParser.option(ArgType.Boolean, NODEFAULTLIBS_DEPRECATED, @@ -59,6 +61,11 @@ open class CommonInteropArguments(val argParser: ArgParser) { description = "don't link unused libraries even explicitly specified").default(false) val tempDir by argParser.option(ArgType.String, TEMP_DIR, description = "save temporary files to the given directory") + + companion object { + const val MODE_SOURCECODE = "sourcecode" + const val MODE_METADATA = "metadata" + } } class CInteropArguments(argParser: ArgParser = diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/GenerationMode.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/GenerationMode.kt new file mode 100644 index 00000000000..6174374932d --- /dev/null +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/GenerationMode.kt @@ -0,0 +1,13 @@ +package org.jetbrains.kotlin.native.interop.gen.jvm + +import org.jetbrains.kotlin.native.interop.tool.CommonInteropArguments + +enum class GenerationMode { + SOURCE_CODE, METADATA +} + +fun parseGenerationMode(mode: String): GenerationMode? = when(mode) { + CommonInteropArguments.MODE_METADATA -> GenerationMode.METADATA + CommonInteropArguments.MODE_SOURCECODE -> GenerationMode.SOURCE_CODE + else -> null +} \ No newline at end of file diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/InteropLibraryCreation.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/InteropLibraryCreation.kt new file mode 100644 index 00000000000..f6a5076d77f --- /dev/null +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/InteropLibraryCreation.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin.native.interop.gen.jvm + +import org.jetbrains.kotlin.konan.CURRENT +import org.jetbrains.kotlin.konan.KonanVersion +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.library.impl.KonanLibraryWriterImpl +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.library.KonanLibraryVersioning +import org.jetbrains.kotlin.library.KotlinAbiVersion +import java.util.* + +data class LibraryCreationArguments( + val outputPath: String, + val moduleName: String, + val nativeBitcodePath: String, + val target: KonanTarget, + val manifest: Properties +) + +fun createInteropLibrary(arguments: LibraryCreationArguments) { + val version = KonanLibraryVersioning( + libraryVersion = null, + abiVersion = KotlinAbiVersion.CURRENT, + compilerVersion = KonanVersion.CURRENT + ) + KonanLibraryWriterImpl( + File(arguments.outputPath),arguments.moduleName, version, arguments.target + ).apply { + addNativeBitcode(arguments.nativeBitcodePath) + addManifestAddend(arguments.manifest) + commit() + } +} \ No newline at end of file diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index f0c4b4a84a4..066eaee18bd 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -34,8 +34,10 @@ fun main(args: Array) { processCLib(args) } -fun interop(flavor: String, args: Array, additionalArgs: Map = mapOf()) = - when(flavor) { +fun interop( + flavor: String, args: Array, + additionalArgs: Map = mapOf() +): Array? = when(flavor) { "jvm", "native" -> processCLib(args, additionalArgs) "wasm" -> processIdlLib(args, additionalArgs) else -> error("Unexpected flavor") @@ -159,7 +161,7 @@ private fun findFilesByGlobs(roots: List, globs: List): Map, additionalArgs: Map = mapOf()): Array { +private fun processCLib(args: Array, additionalArgs: Map = mapOf()): Array? { val cinteropArguments = CInteropArguments() cinteropArguments.argParser.parse(args) val ktGenRoot = cinteropArguments.generated @@ -204,11 +206,7 @@ private fun processCLib(args: Array, additionalArgs: Map = val fqParts = (cinteropArguments.pkg ?: def.config.packageName)?.split('.') ?: defFile!!.name.split('.').reversed().drop(1) - val outKtFileName = fqParts.last() + ".kt" - val outKtPkg = fqParts.joinToString(".") - val outKtFileRelative = (fqParts + outKtFileName).joinToString("/") - val outKtFile = File(ktGenRoot, outKtFileRelative) val libName = (additionalArgs["cstubsname"] as? String)?: fqParts.joinToString("") + "stubs" @@ -240,7 +238,6 @@ private fun processCLib(args: Array, additionalArgs: Map = target = target ) - outKtFile.parentFile.mkdirs() File(nativeLibsDir).mkdirs() val outCFile = tempFiles.create(libName, ".${language.sourceFileExtension}") @@ -251,9 +248,22 @@ private fun processCLib(args: Array, additionalArgs: Map = {} } + val mode = parseGenerationMode(cinteropArguments.mode) + ?: error ("Unexpected interop generation mode: ${cinteropArguments.mode}") + val stubIrContext = StubIrContext(logger, configuration, nativeIndex, imports, flavor, libName) - val stubIrDriver = StubIrDriver(stubIrContext) - stubIrDriver.run(outKtFile, File(outCFile.absolutePath), entryPoint) + val stubIrOutput = run { + val outKtFileCreator = { + val outKtFileName = fqParts.last() + ".kt" + val outKtFileRelative = (fqParts + outKtFileName).joinToString("/") + val file = File(ktGenRoot, outKtFileRelative) + file.parentFile.mkdirs() + file + } + val driverOptions = StubIrDriver.DriverOptions(mode, entryPoint, File(outCFile.absolutePath), outKtFileCreator) + val stubIrDriver = StubIrDriver(stubIrContext, driverOptions) + stubIrDriver.run() + } // TODO: if a library has partially included headers, then it shouldn't be used as a dependency. def.manifestAddendProperties["includedHeaders"] = nativeIndex.includedHeaders.joinToString(" ") { it.value } @@ -270,31 +280,48 @@ private fun processCLib(args: Array, additionalArgs: Map = manifestAddend?.parentFile?.mkdirs() manifestAddend?.let { def.manifestAddendProperties.storeProperties(it) } - if (flavor == KotlinPlatform.JVM) { + val compilerArgs = stubIrContext.libraryForCStubs.compilerArgs.toTypedArray() + val nativeOutputPath: String = when (flavor) { + KotlinPlatform.JVM -> { + val outOFile = tempFiles.create(libName,".o") + val compilerCmd = arrayOf(compiler, *compilerArgs, + "-c", outCFile.absolutePath, "-o", outOFile.absolutePath) + runCmd(compilerCmd, verbose) - val outOFile = tempFiles.create(libName,".o") + val outLib = File(nativeLibsDir, System.mapLibraryName(libName)) + val linkerCmd = arrayOf(linker, + outOFile.absolutePath, "-shared", "-o", outLib.absolutePath, + *linkerOpts) + runCmd(linkerCmd, verbose) + outOFile.absolutePath + } + KotlinPlatform.NATIVE -> { + val outLib = File(nativeLibsDir, "$libName.bc") + val compilerCmd = arrayOf(compiler, *compilerArgs, + "-emit-llvm", "-c", outCFile.absolutePath, "-o", outLib.absolutePath) - val compilerCmd = arrayOf(compiler, *stubIrContext.libraryForCStubs.compilerArgs.toTypedArray(), - "-c", outCFile.absolutePath, "-o", outOFile.absolutePath) - - runCmd(compilerCmd, verbose) - - val outLib = File(nativeLibsDir, System.mapLibraryName(libName)) - - val linkerCmd = arrayOf(linker, - outOFile.absolutePath, "-shared", "-o", outLib.absolutePath, - *linkerOpts) - - runCmd(linkerCmd, verbose) - } else if (flavor == KotlinPlatform.NATIVE) { - val outBcName = libName + ".bc" - val outLib = File(nativeLibsDir, outBcName) - val compilerCmd = arrayOf(compiler, *stubIrContext.libraryForCStubs.compilerArgs.toTypedArray(), - "-emit-llvm", "-c", outCFile.absolutePath, "-o", outLib.absolutePath) - - runCmd(compilerCmd, verbose) + runCmd(compilerCmd, verbose) + outLib.absolutePath + } + } + + return when (stubIrOutput) { + is StubIrDriver.Result.SourceCode -> { + argsToCompiler(staticLibraries, libraryPaths) + } + is StubIrDriver.Result.Metadata -> { + val moduleName = File(cinteropArguments.output).nameWithoutExtension + val args = LibraryCreationArguments( + nativeBitcodePath = nativeOutputPath, + target = tool.target, + moduleName = moduleName, + outputPath = cinteropArguments.output, + manifest = def.manifestAddendProperties + ) + createInteropLibrary(args) + return null + } } - return argsToCompiler(staticLibraries, libraryPaths) } internal fun prepareTool(target: String?, flavor: KotlinPlatform): ToolConfig { diff --git a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt index 59ab9de1429..fd055608ec1 100644 --- a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt +++ b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt @@ -83,7 +83,7 @@ private fun generatePlatformLibraries(target: String, inputDirectory: File, outp "-no-default-libs", "-no-endorsed-libs", "-Xpurge-user-libs", *def.depends.flatMap { listOf("-l", "$outputDirectory/${it.name}") }.toTypedArray()) println("Processing ${def.name}...") - K2Native.mainNoExit(invokeInterop("native", args)) + invokeInterop("native", args)?.let { K2Native.mainNoExit(it) } org.jetbrains.kotlin.cli.klib.main(arrayOf("install", outKlib, "-target", target, "-repository", "${outputDirectory.absolutePath}" diff --git a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/InteropCompiler.kt b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/InteropCompiler.kt index f249081d8d0..81ac6c446b8 100644 --- a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/InteropCompiler.kt +++ b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/InteropCompiler.kt @@ -19,7 +19,11 @@ import org.jetbrains.kotlin.native.interop.tool.* // TODO: this function should eventually be eliminated from 'utilities'. // The interaction of interop and the compiler should be streamlined. -fun invokeInterop(flavor: String, args: Array): Array { +/** + * @return null if there is no need in compiler invocation. + * Otherwise returns array of compiler args. + */ +fun invokeInterop(flavor: String, args: Array): Array? { val arguments = if (flavor == "native") CInteropArguments() else JSInteropArguments() arguments.argParser.parse(args) val outputFileName = arguments.output @@ -68,7 +72,9 @@ fun invokeInterop(flavor: String, args: Array): Array { additionalProperties.putAll(mapOf("cstubsname" to cstubsName, "import" to imports)) } + val cinteropArgsToCompiler = interop(flavor, args + additionalArgs, additionalProperties) + ?: return null // There is no need in compiler invocation if we're generating only metadata. val nativeStubs = if (flavor == "wasm") diff --git a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt index 3f489f21bdf..6c24dff69b0 100644 --- a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt +++ b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt @@ -17,11 +17,11 @@ private fun mainImpl(args: Array, konancMain: (Array) -> Unit) { konancMain(utilityArgs) "cinterop" -> { val konancArgs = invokeInterop("native", utilityArgs) - konancMain(konancArgs) + konancArgs?.let { konancMain(it) } } "jsinterop" -> { val konancArgs = invokeInterop("wasm", utilityArgs) - konancMain(konancArgs) + konancArgs?.let { konancMain(it) } } "klib" -> klibMain(utilityArgs)