azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tests: Support two-stage compilation (src -> klib -> kexe)

Browse Source
This commit is contained in:
Ilya Matveev
2019-07-30 17:39:05 +07:00
committed by Ilya Matveev
parent 00d780b650
commit e8d8df6928
7 changed files with 155 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HACKING.md
+5
View File
@@ -93,6 +93,11 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
* **-Ptest_verbose** enables printing compiler args and other helpful information during a test execution.
./gradlew -Ptest_verbose :backend.native:tests:mpp_optional_expectation
* **-Ptest_two_stage** enables two-stage compilation of tests. If two-stage compilation is enabled, test sources are compiled into a klibrary
and then a final native binary is produced from this klibrary using the -Xsource-library compiler flag.
./gradlew -Ptest_two_stage backend.native:tests:array0
## Performance measurement
Interop/StubGenerator/build.gradle
+1 -1
View File
@@ -31,7 +31,7 @@ repositories {
dependencies {
compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
compile "org.jetbrains.kotlin:kotlin-compiler:$buildKotlinVersion"
compile "org.jetbrains.kotlin:kotlin-compiler:$kotlinVersion"
compile project(':Interop:Indexer')
compile "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
compile project(path: ":endorsedLibraries:kliopt", configuration: "jvmRuntimeElements")
backend.native/tests/build.gradle
+9 -1
View File
@@ -3,6 +3,7 @@
* that can be found in the LICENSE file.
*/
import shadow.org.jetbrains.kotlin.gradle.plugin.tasks.KonanCompileNativeBinary
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.konan.target.KonanTarget
@@ -16,7 +17,7 @@ buildscript {
maven {
url buildKotlinCompilerRepo
}
maven {
maven {
url kotlinCompilerRepo
}
}
@@ -87,6 +88,13 @@ project.convention.plugins.executor = ExecutorServiceKt.create(project)
// Do not generate run tasks for konan built artifacts
ext.konanNoRun = true
// Enable two-stage compilation if the test_two_stage property is set.
if (project.hasProperty("test_two_stage")) {
tasks.withType(KonanCompileNativeBinary.class) {
enableTwoStageCompilation = true
}
}
allprojects {
// Root directories for test output (logs, compiled files, statistics etc). Only single path must be in each set.
// backend.native/tests
gradle.properties
+3
View File
@@ -26,3 +26,6 @@ testKotlinCompilerVersion=1.3.60-dev-1210
konanVersion=1.3.60
org.gradle.jvmargs='-Dfile.encoding=UTF-8'
org.gradle.workers.max=4
# Uncomment to enable composite build
#kotlinProjectPath=<insert the path to Kotlin project root here>
tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanBuildingTask.kt
+1 -9
View File
@@ -47,16 +47,8 @@ abstract class KonanBuildingTask: KonanArtifactWithLibrariesTask(), KonanBuildin
val konanVersion
@Input get() = project.konanVersion.toString(true, true)
protected abstract fun buildArgs(): List<String>
@TaskAction
open fun run() {
destinationDir.mkdirs()
if (dumpParameters) {
dumpProperties(this)
}
toolRunner.run(buildArgs())
}
abstract fun run()
// DSL.
tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt
+135 -15
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,8 +25,6 @@ import org.gradle.process.CommandLineArgumentProvider
import org.jetbrains.kotlin.gradle.plugin.konan.*
import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifact
import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifactImpl
import org.jetbrains.kotlin.konan.CURRENT
import org.jetbrains.kotlin.konan.KonanVersion
import org.jetbrains.kotlin.konan.library.defaultResolver
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.Distribution
@@ -97,6 +95,14 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
val apiVersion : String?
@Optional @Input get() = project.konanExtension.apiVersion
/**
* Is the two-stage compilation enabled.
*
* In regular (one-stage) compilation, sources are directly compiled into a final native binary.
* In two-stage compilation, sources are compiled into a klib first and then a final native binary is produced from this klib.
*/
abstract val enableTwoStageCompilation: Boolean
protected fun directoryToKt(dir: Any) = project.fileTree(dir).apply {
include("**/*.kt")
exclude { it.file.startsWith(project.buildDir) }
@@ -104,9 +110,85 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
// Command line ------------------------------------------------------------
override fun buildArgs() = mutableListOf<String>().apply {
addArg("-output", artifact.canonicalPath)
// Exclude elements matching the predicate.
private fun List<String>.excludeFlags(predicate: (String) -> Boolean) = filterNot(predicate)
// Exclude the listed elements.
private fun List<String>.excludeFlags(vararg keys: String) = keys.toSet().let { keysToExclude ->
excludeFlags { it in keysToExclude }
}
// Exclude the arguments passed by the given keys.
// E.g. if the list contains the following elements: ["-l", "foo", "-r", "bar"],
// call exclude("-r") returns the following list: ["-l", "foo"].
private fun List<String>.excludeArguments(vararg args: String): List<String> {
val argsToExclude = args.toSet()
val xPrefixesToExclude = argsToExclude.filter { it.startsWith("-X") }.map { "$it=" }
val result = mutableListOf<String>()
var i = 0
while (i < size) {
val key = this[i]
when {
key in argsToExclude -> {
// Skip the key and the following arg.
i++
}
// Support args passed as -X<arg>=<value>.
xPrefixesToExclude.any { key.startsWith(it) } -> { /* Skip the key. */ }
else -> result += key
}
i++
}
return result
}
// Don't include coverage flags into the first stage because they are not supported when compiling a klib.
private fun firstStageExtraOpts() = extraOpts
.excludeFlags("-Xcoverage")
.excludeArguments("-Xcoverage-file", "-Xlibrary-to-cover")
// Don't include the -Xemit-lazy-objc-header flag into
// the second stage because this stage have no sources.
private fun secondStageExtraOpts() = extraOpts
.excludeArguments("-Xemit-lazy-objc-header")
/** Args passed to the compiler at the first stage of two-stage compilation (klib building). */
protected fun buildFirstStageArgs(klibPath: String) = mutableListOf<String>().apply {
addArg("-output", klibPath)
addArg("-produce", CompilerOutputKind.LIBRARY.name.toLowerCase())
addAll(buildCommonArgs())
addAll(firstStageExtraOpts())
allSourceFiles.mapTo(this) { it.absolutePath }
commonSrcFiles
.flatMap { it.files }
.mapTo(this) { "-Xcommon-sources=${it.absolutePath}" }
}
/** Args passed to the compiler at the second stage of two-stage compilation (producing a final binary from the klib). */
protected fun buildSecondStageArgs(klibPath: String) = mutableListOf<String>().apply {
addArg("-output", artifact.canonicalPath)
addArg("-produce", produce.name.toLowerCase())
addArgIfNotNull("-entry", entryPoint)
addAll(buildCommonArgs())
addFileArgs("-native-library", nativeLibraries)
linkerOpts.forEach {
addArg("-linker-option", it)
}
addAll(secondStageExtraOpts())
addArg("-l", klibPath)
add("-Xsource-library=${klibPath}")
}
/** Args passed to the compiler at both stages of the two-stage compilation and during the singe-stage compilation. */
protected fun buildCommonArgs() = mutableListOf<String>().apply {
addArgs("-repo", libraries.repos.map { it.canonicalPath })
if (platformConfiguration.files.isNotEmpty()) {
@@ -120,12 +202,6 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
// The library's directory is added in libraries.repos.
addArgs("-library", libraries.artifacts.map { it.artifact.nameWithoutExtension })
addFileArgs("-native-library", nativeLibraries)
addArg("-produce", produce.name.toLowerCase())
linkerOpts.forEach {
addArg("-linker-option", it)
}
addArgIfNotNull("-target", konanTarget.visibleName)
addArgIfNotNull("-language-version", languageVersion)
addArgIfNotNull("-api-version", apiVersion)
@@ -143,6 +219,21 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
if (libraries.friends.isNotEmpty())
addArg("-friend-modules", libraries.friends.joinToString(File.pathSeparator))
}
/** Args passed to the compiler if the two-stage compilation is disabled. */
fun buildSingleStageArgs() = mutableListOf<String>().apply {
addArg("-output", artifact.canonicalPath)
addArg("-produce", produce.name.toLowerCase())
addArgIfNotNull("-entry", entryPoint)
addAll(buildCommonArgs())
addFileArgs("-native-library", nativeLibraries)
linkerOpts.forEach {
addArg("-linker-option", it)
}
addAll(extraOpts)
@@ -253,9 +344,37 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
)
}
// endregion
override fun run() {
destinationDir.mkdirs()
if (dumpParameters) {
dumpProperties(this)
}
if (enableTwoStageCompilation) {
logger.info("Start two-stage compilation")
val intermediateDir = project.konanBuildRoot
.resolve("intermediate")
.targetSubdir(konanTarget)
.apply { mkdirs() }
val klibPrefix = CompilerOutputKind.LIBRARY.prefix(konanTarget)
val klibSuffix = CompilerOutputKind.LIBRARY.suffix(konanTarget)
val intermediateKlib = intermediateDir.resolve("$klibPrefix$artifactName$klibSuffix").absolutePath
logger.info("Start first stage")
toolRunner.run(buildFirstStageArgs(intermediateKlib))
logger.info("Start second stage")
toolRunner.run(buildSecondStageArgs(intermediateKlib))
} else {
toolRunner.run(buildSingleStageArgs())
}
}
}
open class KonanCompileProgramTask: KonanCompileTask() {
abstract class KonanCompileNativeBinary: KonanCompileTask() {
@Input
override var enableTwoStageCompilation: Boolean = false
}
open class KonanCompileProgramTask: KonanCompileNativeBinary() {
override val produce: CompilerOutputKind get() = CompilerOutputKind.PROGRAM
var runTask: Exec? = null
@@ -287,14 +406,14 @@ open class KonanCompileProgramTask: KonanCompileTask() {
}
open class KonanCompileDynamicTask: KonanCompileTask() {
open class KonanCompileDynamicTask: KonanCompileNativeBinary() {
override val produce: CompilerOutputKind get() = CompilerOutputKind.DYNAMIC
val headerFile: File
@OutputFile get() = destinationDir.resolve("$artifactPrefix${artifactName}_api.h")
}
open class KonanCompileFrameworkTask: KonanCompileTask() {
open class KonanCompileFrameworkTask: KonanCompileNativeBinary() {
override val produce: CompilerOutputKind get() = CompilerOutputKind.FRAMEWORK
override val artifact
@@ -303,8 +422,9 @@ open class KonanCompileFrameworkTask: KonanCompileTask() {
open class KonanCompileLibraryTask: KonanCompileTask() {
override val produce: CompilerOutputKind get() = CompilerOutputKind.LIBRARY
override val enableTwoStageCompilation: Boolean = false
}
open class KonanCompileBitcodeTask: KonanCompileTask() {
open class KonanCompileBitcodeTask: KonanCompileNativeBinary() {
override val produce: CompilerOutputKind get() = CompilerOutputKind.BITCODE
}
tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanInteropTask.kt
+1 -1
View File
@@ -75,7 +75,7 @@ open class KonanInteropTask @Inject constructor(val workerExecutor: WorkerExecut
@InputFiles val headers = mutableSetOf<FileCollection>()
@InputFiles val linkFiles = mutableSetOf<FileCollection>()
override fun buildArgs() = mutableListOf<String>().apply {
fun buildArgs() = mutableListOf<String>().apply {
addArg("-o", artifact.canonicalPath)
addArgIfNotNull("-target", konanTarget.visibleName)