Refactor KotlinNativeToolRunner to make it extendable

Now, it's possible to add new tool runners independent of Kotlin/Native
distribution. Ex: Upcoming KotlinNativeKlibCommonizerToolRunner
This commit is contained in:
Dmitriy Dolovov
2020-02-03 18:00:57 +07:00
parent cb07c9172f
commit e302818e26
6 changed files with 311 additions and 253 deletions
@@ -0,0 +1,23 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
// TODO: implement this runner
internal class KotlinNativeKlibCommonizerToolRunner(project: Project) : KotlinToolRunner(project) {
override val displayName get() = "Kotlin/Native KLIB commonizer"
override val mainClass: String get() = TODO("not implemented")
override val classpath: FileCollection get() = TODO("not implemented")
override fun getIsolatedClassLoader(): ClassLoader {
TODO("not implemented")
}
override val mustRunViaExec get() = false
}
@@ -1,243 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Named
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
import org.jetbrains.kotlin.gradle.dsl.NativeCacheKind
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.KOTLIN_NATIVE_HOME
import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
import org.jetbrains.kotlin.konan.CompilerVersion
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.DependencyDirectories
import java.io.File
import java.lang.reflect.InvocationTargetException
import java.net.URLClassLoader
import java.util.*
import java.util.concurrent.ConcurrentHashMap
/** Copied from Kotlin/Native repository. */
private val Project.jvmArgs
get() = PropertiesProvider(this).nativeJvmArgs?.split("\\s+".toRegex()).orEmpty()
internal val Project.konanHome: String
get() = PropertiesProvider(this).nativeHome?.let {
file(it).absolutePath
} ?: NativeCompilerDownloader(project).compilerDirectory.absolutePath
internal val Project.konanVersion: CompilerVersion
get() = PropertiesProvider(this).nativeVersion?.let {
CompilerVersion.fromString(it.toString())
} ?: NativeCompilerDownloader.DEFAULT_KONAN_VERSION
internal val Project.disableKonanDaemon: Boolean
get() = PropertiesProvider(this).nativeDisableCompilerDaemon == true
internal val Project.konanCacheKind: NativeCacheKind
get() = PropertiesProvider(this).nativeCacheKind
internal interface KonanToolRunner : Named {
val mainClass: String
val classpath: FileCollection
val jvmArgs: List<String>
val environment: Map<String, Any>
val additionalSystemProperties: Map<String, String>
fun run(args: List<String>)
fun run(vararg args: String) = run(args.toList())
}
internal abstract class KonanCliRunner(
val toolName: String,
val fullName: String,
val project: Project,
private val additionalJvmArgs: List<String>
) : KonanToolRunner {
override val mainClass = "org.jetbrains.kotlin.cli.utilities.MainKt"
override fun getName() = toolName
// We need to unset some environment variables which are set by XCode and may potentially affect the tool executed.
protected val blacklistEnvironment: List<String> by lazy {
KonanToolRunner::class.java.getResourceAsStream("/env_blacklist")?.let { stream ->
stream.reader().use { it.readLines() }
} ?: emptyList<String>()
}
protected val blacklistProperties: Set<String> =
setOf("java.endorsed.dirs")
override val classpath: FileCollection =
project.fileTree("${project.konanHome}/konan/lib/")
.apply { include("*.jar") }
override val jvmArgs = mutableListOf("-ea").apply {
if (additionalJvmArgs.none { it.startsWith("-Xmx") } &&
project.jvmArgs.none { it.startsWith("-Xmx") }) {
add("-Xmx3G")
}
// Disable C2 compiler for HotSpot VM to improve compilation speed.
System.getProperty("java.vm.name")?.let {
if (it.contains("HotSpot", true)) {
add("-XX:TieredStopAtLevel=1")
}
}
addAll(additionalJvmArgs)
addAll(project.jvmArgs)
}
override val additionalSystemProperties = mutableMapOf(
"konan.home" to project.konanHome,
MessageRenderer.PROPERTY_KEY to MessageRenderer.GRADLE_STYLE.name,
"java.library.path" to "${project.konanHome}/konan/nativelib"
)
override val environment = mutableMapOf("LIBCLANG_DISABLE_CRASH_RECOVERY" to "1")
private fun String.escapeQuotes() = replace("\"", "\\\"")
private fun Sequence<Pair<String, String>>.escapeQuotesForWindows() =
if (HostManager.hostIsMingw) {
map { (key, value) -> key.escapeQuotes() to value.escapeQuotes() }
} else {
this
}
protected open fun transformArgs(args: List<String>): List<String> = args
override fun run(args: List<String>) {
project.logger.info("Run tool: $toolName with args: ${args.joinToString(separator = " ")}")
if (classpath.isEmpty) {
throw IllegalStateException(
"Classpath of the tool is empty: $toolName\n" +
"Probably the '$KOTLIN_NATIVE_HOME' project property contains an incorrect path.\n" +
"Please change it to the compiler root directory and rerun the build."
)
}
if (project.disableKonanDaemon || toolName == "cinterop") {
project.javaexec { spec ->
spec.main = mainClass
spec.classpath = classpath
spec.jvmArgs(jvmArgs)
spec.systemProperties(
System.getProperties().asSequence()
.map { (k, v) -> k.toString() to v.toString() }
.filter { (k, _) -> k !in blacklistProperties }
.escapeQuotesForWindows()
.toMap()
)
spec.systemProperties(additionalSystemProperties)
spec.args(listOf(toolName) + transformArgs(args))
blacklistEnvironment.forEach { spec.environment.remove(it) }
spec.environment(environment)
}
} else {
val oldProperties = mutableMapOf<String, String?>()
additionalSystemProperties.forEach {
oldProperties[it.key] = System.getProperty(it.key)
}
System.getProperties().toList()
.map { (k, v) -> k.toString() to v.toString() }
.filter { (k, _) -> k in blacklistProperties }
.forEach { (k, v) ->
oldProperties[k] = v
System.clearProperty(k)
}
additionalSystemProperties.forEach { System.setProperty(it.key, it.value) }
val konanCompilerClassLoader = konanCompilerClassLoadersMap.computeIfAbsent(project.konanHome) {
val arrayOfURLs = classpath.map { File(it.absolutePath).toURI().toURL() }.toTypedArray()
URLClassLoader(arrayOfURLs, null)
}
val mainClass = konanCompilerClassLoader.loadClass(mainClass)
val mainMethod = mainClass.methods.single { it.name == "daemonMain" }
try {
mainMethod.invoke(null, (listOf(toolName) + transformArgs(args)).toTypedArray())
} catch (t: InvocationTargetException) {
throw t.targetException
} finally {
oldProperties.forEach {
val value = it.value
if (value == null)
System.clearProperty(it.key)
else System.setProperty(it.key, value)
}
}
}
}
}
private val konanCompilerClassLoadersMap = ConcurrentHashMap<String, ClassLoader>()
internal class KonanInteropRunner(project: Project, additionalJvmArgs: List<String> = emptyList()) :
KonanCliRunner("cinterop", "Kotlin/Native cinterop tool", project, additionalJvmArgs) {
init {
if (HostManager.host == KonanTarget.MINGW_X64) {
// TODO: Read it from Platform properties when it is accessible.
val konanProperties = Properties().apply {
project.file("${project.konanHome}/konan/konan.properties").inputStream().use(::load)
}
val toolchainDir = konanProperties.getProperty("llvmHome.mingw_x64")
if (toolchainDir != null) {
environment.put(
"PATH",
DependencyDirectories.defaultDependenciesRoot
.resolve("$toolchainDir/bin")
.absolutePath + ";${System.getenv("PATH")}"
)
}
}
}
}
internal class KonanCompilerRunner(
project: Project,
additionalJvmArgs: List<String> = emptyList(),
val useArgFile: Boolean = project.disableKonanDaemon
) : KonanCliRunner("konanc", "Kotlin/Native compiler", project, additionalJvmArgs) {
override fun transformArgs(args: List<String>): List<String> {
if (!useArgFile) {
return args
}
val argFile = createTempFile(prefix = "konancArgs", suffix = ".lst").apply {
deleteOnExit()
}
argFile.printWriter().use { writer ->
args.forEach { arg ->
val escapedArg = arg
.replace("\\", "\\\\")
.replace("\"", "\\\"")
writer.println("\"$escapedArg\"")
}
}
return listOf("@${argFile.absolutePath}")
}
}
internal class KonanKlibRunner(project: Project, additionalJvmArgs: List<String> = emptyList()) :
KonanCliRunner("klib", "Klib management tool", project, additionalJvmArgs)
@@ -0,0 +1,133 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.jetbrains.kotlin.konan.target.HostManager
import java.lang.reflect.InvocationTargetException
internal abstract class KotlinToolRunner(
val project: Project
) {
// name that will be used in logs
abstract val displayName: String
abstract val mainClass: String
open val daemonEntryPoint: String get() = "main"
open val environment: Map<String, String> = emptyMap()
open val environmentBlacklist: Set<String> = emptySet()
open val systemProperties: Map<String, String> = emptyMap()
open val systemPropertiesBlacklist: Set<String> = emptySet()
abstract val classpath: FileCollection
open fun checkClasspath(): Unit = check(!classpath.isEmpty) { "Classpath of the tool is empty: $displayName" }
abstract fun getIsolatedClassLoader(): ClassLoader
open val defaultMaxHeapSize: String get() = "3G"
open val enableAssertions: Boolean get() = true
open val disableC2: Boolean get() = true
abstract val mustRunViaExec: Boolean
open fun transformArgs(args: List<String>): List<String> = args
// for the purpose if there is a way to specify JVM args, for instance, straight in project configs
open fun getCustomJvmArgs(): List<String> = emptyList()
private val jvmArgs: List<String> by lazy {
mutableListOf<String>().apply {
if (enableAssertions) add("-ea")
val customJvmArgs = getCustomJvmArgs()
if (customJvmArgs.none { it.startsWith("-Xmx") }) add("-Xmx$defaultMaxHeapSize")
// Disable C2 compiler for HotSpot VM to improve compilation speed.
if (disableC2) {
System.getProperty("java.vm.name")?.let { vmName ->
if (vmName.contains("HotSpot", true)) add("-XX:TieredStopAtLevel=1")
}
}
addAll(customJvmArgs)
}
}
fun run(args: List<String>) {
project.logger.info("Run tool: \"$displayName\" with args: ${args.joinToString(separator = " ")}")
checkClasspath()
if (mustRunViaExec) runViaExec(args) else runInProcess(args)
}
private fun runViaExec(args: List<String>) {
project.javaexec { spec ->
spec.main = mainClass
spec.classpath = classpath
spec.jvmArgs(jvmArgs)
spec.systemProperties(
System.getProperties().asSequence()
.map { (k, v) -> k.toString() to v.toString() }
.filter { (k, _) -> k !in systemPropertiesBlacklist }
.escapeQuotesForWindows()
.toMap()
)
spec.systemProperties(systemProperties)
environmentBlacklist.forEach { spec.environment.remove(it) }
spec.environment(environment)
spec.args(transformArgs(args))
}
}
private fun runInProcess(args: List<String>) {
val oldProperties = setUpSystemProperties()
try {
val classLoader = getIsolatedClassLoader()
val mainClass = classLoader.loadClass(mainClass)
val entryPoint = mainClass.methods.single { it.name == daemonEntryPoint }
entryPoint.invoke(null, transformArgs(args).toTypedArray())
} catch (t: InvocationTargetException) {
throw t.targetException
} finally {
restoreSystemProperties(oldProperties)
}
}
private fun setUpSystemProperties(): Map<String, String?> {
val oldProperties = mutableMapOf<String, String?>()
systemProperties.forEach { (k, v) -> oldProperties[k] = System.getProperty(v) }
System.getProperties().toList()
.map { (k, v) -> k.toString() to v.toString() }
.filter { (k, _) -> k in systemPropertiesBlacklist }
.forEach { (k, v) ->
oldProperties[k] = v
System.clearProperty(k)
}
systemProperties.forEach { (k, v) -> System.setProperty(k, v) }
return oldProperties
}
private fun restoreSystemProperties(oldProperties: Map<String, String?>) {
oldProperties.forEach { (k, v) ->
if (v == null) System.clearProperty(k) else System.setProperty(k, v)
}
}
companion object {
private fun String.escapeQuotes() = replace("\"", "\\\"")
private fun Sequence<Pair<String, String>>.escapeQuotesForWindows() =
if (HostManager.hostIsMingw) map { (key, value) -> key.escapeQuotes() to value.escapeQuotes() } else this
}
}
@@ -0,0 +1,149 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
import org.jetbrains.kotlin.gradle.dsl.NativeCacheKind
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
import org.jetbrains.kotlin.konan.CompilerVersion
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.DependencyDirectories
import java.io.File
import java.net.URLClassLoader
import java.util.*
import java.util.concurrent.ConcurrentHashMap
private val Project.jvmArgs
get() = PropertiesProvider(this).nativeJvmArgs?.split("\\s+".toRegex()).orEmpty()
internal val Project.konanHome: String
get() = PropertiesProvider(this).nativeHome?.let { file(it).absolutePath }
?: NativeCompilerDownloader(project).compilerDirectory.absolutePath
internal val Project.disableKonanDaemon: Boolean
get() = PropertiesProvider(this).nativeDisableCompilerDaemon == true
internal val Project.konanVersion: CompilerVersion
get() = PropertiesProvider(this).nativeVersion?.let { CompilerVersion.fromString(it) }
?: NativeCompilerDownloader.DEFAULT_KONAN_VERSION
internal val Project.konanCacheKind: NativeCacheKind
get() = PropertiesProvider(this).nativeCacheKind
internal abstract class KotlinNativeToolRunner(
protected val toolName: String,
project: Project
) : KotlinToolRunner(project) {
final override val displayName get() = toolName
final override val mainClass get() = "org.jetbrains.kotlin.cli.utilities.MainKt"
final override val daemonEntryPoint get() = "daemonMain"
// We need to unset some environment variables which are set by XCode and may potentially affect the tool executed.
override val environment = mapOf("LIBCLANG_DISABLE_CRASH_RECOVERY" to "1")
final override val environmentBlacklist: Set<String> by lazy {
HashSet<String>().also { collector ->
KotlinNativeToolRunner::class.java.getResourceAsStream("/env_blacklist")?.let { stream ->
stream.reader().use { r -> r.forEachLine { collector.add(it) } }
}
}
}
final override val systemProperties by lazy {
mapOf(
"konan.home" to project.konanHome,
MessageRenderer.PROPERTY_KEY to MessageRenderer.GRADLE_STYLE.name,
"java.library.path" to "${project.konanHome}/konan/nativelib"
)
}
final override val systemPropertiesBlacklist = setOf("java.endorsed.dirs")
final override val classpath: FileCollection = project.fileTree("${project.konanHome}/konan/lib/").apply { include("*.jar") }
final override fun checkClasspath() =
check(!classpath.isEmpty) {
"""
Classpath of the tool is empty: $toolName
Probably the '${PropertiesProvider.KOTLIN_NATIVE_HOME}' project property contains an incorrect path.
Please change it to the compiler root directory and rerun the build.
""".trimIndent()
}
final override fun getIsolatedClassLoader() =
isolatedClassLoadersMap.computeIfAbsent(project.konanHome) {
val arrayOfURLs = classpath.map { File(it.absolutePath).toURI().toURL() }.toTypedArray()
URLClassLoader(arrayOfURLs, null)
}
override fun transformArgs(args: List<String>) = listOf(toolName) + args
final override fun getCustomJvmArgs() = project.jvmArgs
companion object {
private val isolatedClassLoadersMap = ConcurrentHashMap<String, ClassLoader>()
}
}
/** Kotlin/Native C-interop tool runner */
internal class KotlinNativeCInteropRunner(project: Project) : KotlinNativeToolRunner("cinterop", project) {
override val mustRunViaExec get() = true
override val environment by lazy {
val llvmExecutablesPath = llvmExecutablesPath
if (llvmExecutablesPath != null)
super.environment + ("PATH" to "$llvmExecutablesPath;${System.getenv("PATH")}")
else
super.environment
}
private val llvmExecutablesPath: String? by lazy {
if (HostManager.host != KonanTarget.MINGW_X64) {
// TODO: Read it from Platform properties when it is accessible.
val konanProperties = Properties().apply {
project.file("${project.konanHome}/konan/konan.properties").inputStream().use(::load)
}
konanProperties.getProperty("llvmHome.mingw_x64")?.let { toolchainDir ->
DependencyDirectories.defaultDependenciesRoot
.resolve("$toolchainDir/bin")
.absolutePath
}
} else
null
}
}
/** Kotlin/Native compiler runner */
internal class KotlinNativeCompilerRunner(project: Project) : KotlinNativeToolRunner("kotlinc-native", project) {
private val useArgFile get() = project.disableKonanDaemon
override val mustRunViaExec get() = project.disableKonanDaemon
override fun transformArgs(args: List<String>): List<String> {
if (!useArgFile) return super.transformArgs(args)
val argFile = createTempFile(prefix = "kotlinc-native-args", suffix = ".lst").apply { deleteOnExit() }
argFile.printWriter().use { w ->
args.forEach { arg ->
val escapedArg = arg
.replace("\\", "\\\\")
.replace("\"", "\\\"")
w.println("\"$escapedArg\"")
}
}
return listOf(toolName, "@${argFile.absolutePath}")
}
}
/** Klib management tool runner */
internal class KotlinNativeKlibRunner(project: Project) : KotlinNativeToolRunner("klib", project) {
override val mustRunViaExec get() = project.disableKonanDaemon
}
@@ -10,7 +10,7 @@ import org.gradle.api.Project
import org.gradle.api.artifacts.repositories.ArtifactRepository
import org.gradle.api.file.FileTree
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.compilerRunner.KonanCompilerRunner
import org.jetbrains.kotlin.compilerRunner.KotlinNativeCompilerRunner
import org.jetbrains.kotlin.compilerRunner.konanVersion
import org.jetbrains.kotlin.gradle.logging.kotlinInfo
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
@@ -134,7 +134,7 @@ class NativeCompilerDownloader(
}
fun downloadIfNeeded() {
if (KonanCompilerRunner(project).classpath.isEmpty) {
if (KotlinNativeCompilerRunner(project).classpath.isEmpty) {
downloadAndExtract()
}
}
@@ -16,10 +16,6 @@ import org.gradle.api.provider.Provider
import org.gradle.api.tasks.*
import org.gradle.api.tasks.compile.AbstractCompile
import org.jetbrains.kotlin.compilerRunner.*
import org.jetbrains.kotlin.compilerRunner.KonanCompilerRunner
import org.jetbrains.kotlin.compilerRunner.KonanInteropRunner
import org.jetbrains.kotlin.compilerRunner.konanHome
import org.jetbrains.kotlin.compilerRunner.konanVersion
import org.jetbrains.kotlin.gradle.dsl.KotlinCommonOptions
import org.jetbrains.kotlin.gradle.dsl.KotlinCommonToolOptions
import org.jetbrains.kotlin.gradle.dsl.KotlinCompile
@@ -270,7 +266,7 @@ abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions> : Abstra
open fun compile() {
val output = outputFile.get()
output.parentFile.mkdirs()
KonanCompilerRunner(project).run(buildArgs())
KotlinNativeCompilerRunner(project).run(buildArgs())
}
}
@@ -781,7 +777,7 @@ class CacheBuilder(val project: Project, val binary: NativeBinary) {
args += "-l"
args += it.libraryFile.absolutePath
}
KonanCompilerRunner(project).run(args)
KotlinNativeCompilerRunner(project).run(args)
}
}
@@ -808,7 +804,7 @@ class CacheBuilder(val project: Project, val binary: NativeBinary) {
args += "-g"
args += "-Xadd-cache=${platformLib.absolutePath}"
args += "-Xcache-directory=${rootCacheDirectory.absolutePath}"
KonanCompilerRunner(project).run(args)
KotlinNativeCompilerRunner(project).run(args)
}
private fun ensureCompilerProvidedLibsPrecached() {
@@ -937,6 +933,6 @@ open class CInteropProcess : DefaultTask() {
}
outputFile.parentFile.mkdirs()
KonanInteropRunner(project).run(args)
KotlinNativeCInteropRunner(project).run(args)
}
}