From 174e440d9d5eb10ebf2e9b622a2ccd6e9a2f43bc Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Wed, 4 Oct 2017 19:52:30 +0300 Subject: [PATCH] Extracted Konan dependent pieces of Stub Generator to separate components and ported them from string manipolations to KonanTarget + KonanProperties. --- .../native/interop/gen/jvm/ClangDefaults.kt | 70 +++++++ .../native/interop/gen/jvm/ToolConfig.kt | 69 +++++++ .../kotlin/native/interop/gen/jvm/main.kt | 179 +++--------------- .../kotlin/konan/target/KonanTarget.kt | 46 +++-- .../jetbrains/kotlin/cli/utilities/main.kt | 16 ++ 5 files changed, 213 insertions(+), 167 deletions(-) create mode 100644 Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt create mode 100644 Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt new file mode 100644 index 00000000000..2ef41519857 --- /dev/null +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.native.interop.tool + +import org.jetbrains.kotlin.konan.target.* +import org.jetbrains.kotlin.konan.target.KonanTarget.* +import org.jetbrains.kotlin.konan.properties.* + +// TODO: Half of these calculations are already provided +// by ClangHost class of "shared" project. +// But there are some discrepancies, in particular the -B claculation. +fun KonanProperties.defaultCompilerOpts(): List { + + // TODO: eliminate this. below + val targetToolchain = absoluteTargetToolchain + val targetSysRoot = absoluteTargetSysRoot + val llvmHome = absolute(hostString("llvmHome")) + val llvmVersion = hostString("llvmVersion")!! + + // StubGenerator passes the arguments to libclang which + // works not exactly the same way as the clang binary and + // (in particular) uses different default header search path. + // See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html + // We workaround the problem with -isystem flag below. + val isystem = "$llvmHome/lib/clang/$llvmVersion/include" + val quadruple = targetString("quadruple") + val arch = targetString("arch") + val archSelector = if (quadruple != null) + listOf("-target", quadruple) else listOf("-arch", arch!!) + val commonArgs = listOf("-isystem", isystem, "--sysroot=$targetSysRoot") + + val host = TargetManager.host + val hostSpecificArgs = when (host) { + MACBOOK -> { + val osVersionMinFlag = targetString("osVersionMinFlagClang") + val osVersionMinValue = targetString("osVersionMin") + listOf("-B$targetToolchain/bin") + + (if (osVersionMinFlag != null && osVersionMinValue != null) + listOf("$osVersionMinFlag=$osVersionMinValue") else emptyList()) + } + LINUX -> { + val libGcc = targetString("libGcc") + val binDir = "$targetSysRoot/${libGcc ?: "bin"}" + listOf( + "-B$binDir", "--gcc-toolchain=$targetToolchain", + "-fuse-ld=$targetToolchain/bin/ld") + } + MINGW -> { + listOf("-B$targetSysRoot/bin") + } + else -> error("Unexpected host: ${host}") + } + + return archSelector + commonArgs + hostSpecificArgs +} + diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt new file mode 100644 index 00000000000..13df2c6a476 --- /dev/null +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.native.interop.tool + +import org.jetbrains.kotlin.konan.file.* +import org.jetbrains.kotlin.konan.properties.* +import org.jetbrains.kotlin.konan.target.* +import kotlin.reflect.KFunction + +class ToolConfig(userProvidedTargetName: String?, userProvidedKonanProperties: String?, runnerProvidedKonanHome: String) { + + private val targetManager = TargetManager(userProvidedTargetName) + private val host = TargetManager.host + private val target = targetManager.target + + private val konanHome = File(runnerProvidedKonanHome).absolutePath + private val konanPropertiesFile = userProvidedKonanProperties ?. File() ?: File(konanHome, "konan/konan.properties") + private val properties = konanPropertiesFile.loadProperties() + + private val dependencies = File(konanHome, "dependencies") + + private val targetProperties = KonanProperties(target, properties, dependencies.path) + + val substitutions = mapOf ( + "target" to target.detailedName, + "arch" to target.architecture.userName) + + fun downloadDependencies() = maybeExecuteHelper(dependencies.absolutePath, + properties, targetProperties.dependencies) + + val llvmHome = targetProperties.absolute(targetProperties.hostString("llvmHome")) + + val defaultCompilerOpts = + targetProperties.defaultCompilerOpts() + + val libclang = when (host) { + KonanTarget.MINGW -> "$llvmHome/bin/libclang.dll" + else -> "$llvmHome/lib/${System.mapLibraryName("clang")}" + } +} + +private fun maybeExecuteHelper(dependenciesRoot: String, properties: Properties, dependencies: List) { + try { + val kClass = Class.forName("org.jetbrains.kotlin.konan.util.Helper0").kotlin + @Suppress("UNCHECKED_CAST") + val ctor = kClass.constructors.single() as KFunction + val result = ctor.call(dependenciesRoot, properties, dependencies) + result.run() + } catch (notFound: ClassNotFoundException) { + // Just ignore, no helper. + } catch (e: Throwable) { + throw IllegalStateException("Cannot download dependencies.", e) + } +} + diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index e414f145aff..7744656d67b 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.native.interop.gen.jvm +import org.jetbrains.kotlin.native.interop.tool.* import org.jetbrains.kotlin.konan.util.DefFile import org.jetbrains.kotlin.native.interop.gen.HeadersInclusionPolicyImpl import org.jetbrains.kotlin.native.interop.gen.ImportsImpl @@ -28,13 +29,8 @@ import kotlin.reflect.KFunction fun main(args: Array) = interop(args, null) fun interop(args: Array, argsToCompiler: MutableList? = null) { - val konanHome = File(System.getProperty("konan.home")).absolutePath - val substitutions = mapOf( - "arch" to (System.getenv("TARGET_ARCH") ?: "x86-64"), - "os" to (System.getenv("TARGET_OS") ?: host) - ) - processLib(konanHome, substitutions, parseArgs(args), argsToCompiler) + processLib(parseArgs(args), argsToCompiler) } // Options, whose values are space-separated and can be escaped. @@ -62,46 +58,6 @@ private fun parseArgs(args: Array): Map> { return commandLine } -private val host: String by lazy { - val os = System.getProperty("os.name") - when { - os == "Linux" -> "linux" - os.startsWith("Windows") -> "mingw" - os == "Mac OS X" -> "osx" - os == "FreeBSD" -> "freebsd" - else -> { - throw IllegalArgumentException("we don't know ${os} value") - } - } -} - -private val defaultTarget: String by lazy { - host -} - -// TODO: share KonanTarget class here. -private val knownTargets = mapOf( - "host" to host, - "linux" to "linux", - "macbook" to "osx", - "osx" to "osx", - "iphone" to "ios", - "ios" to "ios", - "iphone_sim" to "ios_sim", - "ios_sim" to "ios_sim", - "raspberrypi" to "raspberrypi", - "linux_mips32" to "linux_mips32", - "linux_mipsel32" to "linux_mipsel32", - "android_arm32" to "android_arm32", - "android_arm64" to "android_arm64", - "mingw" to "mingw", - "wasm32" to "wasm32" -) - - -private fun String.targetSuffix(): String = - knownTargets[this] ?: error("Unsupported target $this.") - // Performs substitution similar to: // foo = ${foo} ${foo.${arch}} ${foo.${os}} private fun substitute(properties: Properties, substitutions: Map) { @@ -134,18 +90,6 @@ private fun Collection.atMostOne(): T? { } } -private fun Properties.getHostSpecific( - name: String) = getProperty("$name.$host") - -private fun Properties.getTargetSpecific( - name: String, target: String) = getProperty("$name.$target") - -private fun Properties.getHostTargetSpecific( - name: String, target: String) = if (host != target) - getProperty("$name.$host-$target") -else - getProperty("$name.$host") - private fun List?.isTrue(): Boolean { // The rightmost wins, null != "true". return this?.last() == "true" @@ -183,75 +127,6 @@ private fun runCmd(command: Array, workDir: File, verbose: Boolean = fal } } -private fun maybeExecuteHelper(dependenciesRoot: String, properties: Properties, dependencies: List) { - try { - val kClass = Class.forName("org.jetbrains.kotlin.konan.util.Helper0").kotlin - @Suppress("UNCHECKED_CAST") - val ctor = kClass.constructors.single() as KFunction - val result = ctor.call(dependenciesRoot, properties, dependencies) - result.run() - } catch (notFound: ClassNotFoundException) { - // Just ignore, no helper. - } catch (e: Throwable) { - throw IllegalStateException("Cannot download dependencies.", e) - } -} - -private fun Properties.getClangFlags(target: String, targetSysRoot: String): List { - val flags = getTargetSpecific("clangFlags", target) - if (flags == null) return emptyList() - return flags.replace("", targetSysRoot).split(' ') -} - -private fun Properties.defaultCompilerOpts(target: String, dependencies: String): List { - val targetToolchainDir = getHostTargetSpecific("targetToolchain", target)!! - val targetToolchain = "$dependencies/$targetToolchainDir" - val targetSysRootDir = getTargetSpecific("targetSysRoot", target)!! - val targetSysRoot = "$dependencies/$targetSysRootDir" - val llvmHomeDir = getHostSpecific("llvmHome")!! - val llvmHome = "$dependencies/$llvmHomeDir" - - val libclang = when (host) { - "mingw" -> "$llvmHome/bin/libclang.dll" - else -> "$llvmHome/lib/${System.mapLibraryName("clang")}" - } - System.load(libclang) - - val llvmVersion = getHostSpecific("llvmVersion")!! - - // StubGenerator passes the arguments to libclang which - // works not exactly the same way as the clang binary and - // (in particular) uses different default header search path. - // See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html - // We workaround the problem with -isystem flag below. - val isystem = "$llvmHome/lib/clang/$llvmVersion/include" - val quadruple = getTargetSpecific("quadruple", target) - val arch = getTargetSpecific("arch", target) - val archSelector = if (quadruple != null) - listOf("-target", quadruple) else listOf("-arch", arch!!) - val commonArgs = listOf("-isystem", isystem, "--sysroot=$targetSysRoot") + getClangFlags(target, targetSysRoot) - when (host) { - "osx" -> { - val osVersionMinFlag = getTargetSpecific("osVersionMinFlagClang", target) - val osVersionMinValue = getTargetSpecific("osVersionMin", target) - return archSelector + commonArgs + listOf("-B$targetToolchain/bin") + - (if (osVersionMinFlag != null && osVersionMinValue != null) - listOf("$osVersionMinFlag=$osVersionMinValue") else emptyList()) - } - "linux" -> { - val libGcc = getTargetSpecific("libGcc", target) - val binDir = "$targetSysRoot/${libGcc ?: "bin"}" - return archSelector + commonArgs + listOf( - "-B$binDir", "--gcc-toolchain=$targetToolchain", - "-fuse-ld=$targetToolchain/bin/ld") - } - "mingw" -> { - return archSelector + commonArgs + listOf("-B$targetSysRoot/bin") - } - else -> error("Unexpected target: ${target}") - } -} - private fun loadProperties(file: File?, substitutions: Map): Properties { val result = Properties() file?.bufferedReader()?.use { reader -> @@ -281,11 +156,6 @@ Following flags are supported: """) } -private fun downloadDependencies(dependenciesRoot: String, target: String, konanProperties: Properties) { - val dependencyList = konanProperties.getHostTargetSpecific("dependencies", target)?.split(' ') ?: listOf() - maybeExecuteHelper(dependenciesRoot, konanProperties, dependencyList) -} - private fun selectNativeLanguage(config: DefFile.DefFileConfig): Language { val languages = mapOf( "C" to Language.C, @@ -329,9 +199,7 @@ private fun parseImports(args: Map>): ImportsImpl { return ImportsImpl(headerIdToPackage) } -private fun processLib(konanHome: String, - substitutions: Map, - args: Map>, +private fun processLib(args: Map>, argsToCompiler: MutableList?) { val userDir = System.getProperty("user.dir") @@ -339,39 +207,36 @@ private fun processLib(konanHome: String, val nativeLibsDir = args["-natives"]?.single() ?: userDir val flavorName = args["-flavor"]?.single() ?: "jvm" val flavor = KotlinPlatform.values().single { it.name.equals(flavorName, ignoreCase = true) } - val target = args["-target"]?.single()?.targetSuffix() ?: defaultTarget val defFile = args["-def"]?.single()?.let { File(it) } val manifestAddend = args["-manifest"]?.single()?.let { File(it) } - + if (defFile == null && args["-pkg"] == null) { usage() return } - val def = DefFile(defFile, substitutions) + val tool = ToolConfig( + args["-target"]?.single(), + args["-properties"]?.single(), + System.getProperty("konan.home") + ) + tool.downloadDependencies() - val konanFileName = args["-properties"]?.single() ?: - "${konanHome}/konan/konan.properties" - val konanFile = File(konanFileName) - val konanProperties = loadProperties(konanFile, mapOf()) - val dependencies = "$konanHome/dependencies" - downloadDependencies(dependencies, target, konanProperties) + val def = DefFile(defFile, tool.substitutions) - // TODO: We can provide a set of flags to find the components in the absence of 'dist' or 'dist/dependencies'. - val llvmHome = konanProperties.getHostSpecific("llvmHome")!! - val llvmInstallPath = "$dependencies/$llvmHome" val additionalHeaders = args["-h"].orEmpty() val additionalCompilerOpts = args["-copt"].orEmpty() + args["-compilerOpts"].orEmpty() val additionalLinkerOpts = args["-lopt"].orEmpty() + args["-linkerOpts"].orEmpty() val generateShims = args["-shims"].isTrue() val verbose = args["-verbose"].isTrue() - val defaultOpts = konanProperties.defaultCompilerOpts(target, dependencies) + System.load(tool.libclang) + val headerFiles = def.config.headers + additionalHeaders val language = selectNativeLanguage(def.config) val compilerOpts: List = mutableListOf().apply { addAll(def.config.compilerOpts) - addAll(defaultOpts) + addAll(tool.defaultCompilerOpts) addAll(additionalCompilerOpts) addAll(when (language) { Language.C -> emptyList() @@ -385,14 +250,18 @@ private fun processLib(konanHome: String, } }) } - val compiler = "clang" + val excludeSystemLibs = def.config.excludeSystemLibs val excludeDependentModules = def.config.excludeDependentModules val entryPoint = def.config.entryPoints.atMostOne() val linkerOpts = - def.config.linkerOpts.toTypedArray() + defaultOpts + additionalLinkerOpts - val linker = args["-linker"]?.atMostOne() ?: def.config.linker + def.config.linkerOpts.toTypedArray() + + tool.defaultCompilerOpts + + additionalLinkerOpts + val linkerName = args["-linker"]?.atMostOne() ?: def.config.linker + val linker = "${tool.llvmHome}/bin/$linkerName" + val compiler = "${tool.llvmHome}/bin/clang" val excludedFunctions = def.config.excludedFunctions.toSet() val staticLibraries = def.config.staticLibraries + args["-staticLibrary"].orEmpty() val libraryPaths = def.config.libraryPaths + args["-libraryPath"].orEmpty() @@ -462,14 +331,14 @@ private fun processLib(konanHome: String, val outOFile = createTempFile(suffix = ".o") - val compilerCmd = arrayOf("$llvmInstallPath/bin/$compiler", *gen.libraryForCStubs.compilerArgs.toTypedArray(), + val compilerCmd = arrayOf(compiler, *gen.libraryForCStubs.compilerArgs.toTypedArray(), "-c", outCFile.absolutePath, "-o", outOFile.absolutePath) runCmd(compilerCmd, workDir, verbose) val outLib = File(nativeLibsDir, System.mapLibraryName(libName)) - val linkerCmd = arrayOf("$llvmInstallPath/bin/$linker", + val linkerCmd = arrayOf(linker, outOFile.absolutePath, "-shared", "-o", outLib.absolutePath, *linkerOpts) @@ -479,7 +348,7 @@ private fun processLib(konanHome: String, } else if (flavor == KotlinPlatform.NATIVE) { val outBcName = libName + ".bc" val outLib = File(nativeLibsDir, outBcName) - val compilerCmd = arrayOf("$llvmInstallPath/bin/$compiler", *gen.libraryForCStubs.compilerArgs.toTypedArray(), + val compilerCmd = arrayOf(compiler, *gen.libraryForCStubs.compilerArgs.toTypedArray(), "-emit-llvm", "-c", outCFile.absolutePath, "-o", outLib.absolutePath) runCmd(compilerCmd, workDir, verbose) diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTarget.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTarget.kt index 36eea353e19..2796aaf30e9 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTarget.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTarget.kt @@ -24,19 +24,41 @@ enum class Family(name:String, val exeSuffix:String) { WASM("wasm", "wasm") } +enum class Bitness { + BITSNESS_32, + BITNESS_64 +} -enum class KonanTarget(val family: Family, val detailedName: String, var enabled: Boolean = false) { - ANDROID_ARM32(Family.ANDROID, "android_arm32"), - ANDROID_ARM64(Family.ANDROID, "android_arm64"), - IPHONE(Family.OSX, "ios"), - IPHONE_SIM(Family.OSX, "ios_sim"), - LINUX(Family.LINUX, "linux"), - MINGW(Family.WINDOWS, "mingw"), - MACBOOK(Family.OSX, "osx"), - RASPBERRYPI(Family.LINUX, "raspberrypi"), - LINUX_MIPS32(Family.LINUX, "linux_mips32"), - LINUX_MIPSEL32(Family.LINUX, "linux_mipsel32"), - WASM32(Family.WASM, "wasm32"); +enum class Architecture(val bitness: Int) { + X86_64(64), + ARM64(64), + ARM32(32), + RASPBERRYPI(32), + MIPS32(32), + MIPSEL32(32), + WASM32(32); + + val userName: String + get() { + return if (this == X86_64) + "x86-64" // Dash instead of underscore. + else + this.name.toLowerCase() + } +} + +enum class KonanTarget(val family: Family, val architecture: Architecture, val detailedName: String, var enabled: Boolean = false) { + ANDROID_ARM32( Family.ANDROID, Architecture.ARM32, "android_arm32"), + ANDROID_ARM64( Family.ANDROID, Architecture.ARM64, "android_arm64"), + IPHONE( Family.OSX, Architecture.ARM32, "ios"), + IPHONE_SIM( Family.OSX, Architecture.X86_64, "ios_sim"), + LINUX( Family.LINUX, Architecture.X86_64, "linux"), + MINGW( Family.WINDOWS, Architecture.X86_64, "mingw"), + MACBOOK( Family.OSX, Architecture.X86_64, "osx"), + RASPBERRYPI( Family.LINUX, Architecture.RASPBERRYPI, "raspberrypi"), + LINUX_MIPS32( Family.LINUX, Architecture.MIPS32, "linux_mips32"), + LINUX_MIPSEL32( Family.LINUX, Architecture.MIPSEL32, "linux_mipsel32"), + WASM32( Family.WASM, Architecture.WASM32, "wasm32"); val userName get() = name.toLowerCase() } diff --git a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt index 064eaf1c5ba..1fdf26dab7b 100644 --- a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt +++ b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt @@ -1,3 +1,19 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.jetbrains.kotlin.cli.utilities import org.jetbrains.kotlin.backend.konan.library.defaultResolver