From e8d8df6928de1ccb7ef31e8f79b37beaec2362e2 Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Tue, 30 Jul 2019 17:39:05 +0700 Subject: [PATCH] Tests: Support two-stage compilation (src -> klib -> kexe) --- HACKING.md | 5 + Interop/StubGenerator/build.gradle | 2 +- backend.native/tests/build.gradle | 10 +- gradle.properties | 3 + .../plugin/konan/tasks/KonanBuildingTask.kt | 10 +- .../plugin/konan/tasks/KonanCompileTask.kt | 150 ++++++++++++++++-- .../plugin/konan/tasks/KonanInteropTask.kt | 2 +- 7 files changed, 155 insertions(+), 27 deletions(-) diff --git a/HACKING.md b/HACKING.md index 12b8457b63f..f10f4ae7bbe 100644 --- a/HACKING.md +++ b/HACKING.md @@ -93,6 +93,11 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope * **-Ptest_verbose** enables printing compiler args and other helpful information during a test execution. ./gradlew -Ptest_verbose :backend.native:tests:mpp_optional_expectation + +* **-Ptest_two_stage** enables two-stage compilation of tests. If two-stage compilation is enabled, test sources are compiled into a klibrary +and then a final native binary is produced from this klibrary using the -Xsource-library compiler flag. + + ./gradlew -Ptest_two_stage backend.native:tests:array0 ## Performance measurement diff --git a/Interop/StubGenerator/build.gradle b/Interop/StubGenerator/build.gradle index e5581599a5f..cfb1158033a 100644 --- a/Interop/StubGenerator/build.gradle +++ b/Interop/StubGenerator/build.gradle @@ -31,7 +31,7 @@ repositories { dependencies { compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion" - compile "org.jetbrains.kotlin:kotlin-compiler:$buildKotlinVersion" + compile "org.jetbrains.kotlin:kotlin-compiler:$kotlinVersion" compile project(':Interop:Indexer') compile "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion" compile project(path: ":endorsedLibraries:kliopt", configuration: "jvmRuntimeElements") diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index 6331b012c06..3a2fbf8c305 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -3,6 +3,7 @@ * that can be found in the LICENSE file. */ +import shadow.org.jetbrains.kotlin.gradle.plugin.tasks.KonanCompileNativeBinary import org.jetbrains.kotlin.* import org.jetbrains.kotlin.konan.target.KonanTarget @@ -16,7 +17,7 @@ buildscript { maven { url buildKotlinCompilerRepo } - maven { + maven { url kotlinCompilerRepo } } @@ -87,6 +88,13 @@ project.convention.plugins.executor = ExecutorServiceKt.create(project) // Do not generate run tasks for konan built artifacts ext.konanNoRun = true +// Enable two-stage compilation if the test_two_stage property is set. +if (project.hasProperty("test_two_stage")) { + tasks.withType(KonanCompileNativeBinary.class) { + enableTwoStageCompilation = true + } +} + allprojects { // Root directories for test output (logs, compiled files, statistics etc). Only single path must be in each set. // backend.native/tests diff --git a/gradle.properties b/gradle.properties index 008ade1fed4..e6c6bcf0828 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,3 +26,6 @@ testKotlinCompilerVersion=1.3.60-dev-1210 konanVersion=1.3.60 org.gradle.jvmargs='-Dfile.encoding=UTF-8' org.gradle.workers.max=4 + +# Uncomment to enable composite build +#kotlinProjectPath= \ No newline at end of file diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanBuildingTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanBuildingTask.kt index e999eeb715b..6d5642d8ace 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanBuildingTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanBuildingTask.kt @@ -47,16 +47,8 @@ abstract class KonanBuildingTask: KonanArtifactWithLibrariesTask(), KonanBuildin val konanVersion @Input get() = project.konanVersion.toString(true, true) - protected abstract fun buildArgs(): List - @TaskAction - open fun run() { - destinationDir.mkdirs() - if (dumpParameters) { - dumpProperties(this) - } - toolRunner.run(buildArgs()) - } + abstract fun run() // DSL. diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt index e894d6aff63..3ebfde67199 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2017 JetBrains s.r.o. + * Copyright 2010-2019 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ import org.gradle.process.CommandLineArgumentProvider import org.jetbrains.kotlin.gradle.plugin.konan.* import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifact import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifactImpl -import org.jetbrains.kotlin.konan.CURRENT -import org.jetbrains.kotlin.konan.KonanVersion import org.jetbrains.kotlin.konan.library.defaultResolver import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.Distribution @@ -97,6 +95,14 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec { val apiVersion : String? @Optional @Input get() = project.konanExtension.apiVersion + /** + * Is the two-stage compilation enabled. + * + * In regular (one-stage) compilation, sources are directly compiled into a final native binary. + * In two-stage compilation, sources are compiled into a klib first and then a final native binary is produced from this klib. + */ + abstract val enableTwoStageCompilation: Boolean + protected fun directoryToKt(dir: Any) = project.fileTree(dir).apply { include("**/*.kt") exclude { it.file.startsWith(project.buildDir) } @@ -104,9 +110,85 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec { // Command line ------------------------------------------------------------ - override fun buildArgs() = mutableListOf().apply { - addArg("-output", artifact.canonicalPath) + // Exclude elements matching the predicate. + private fun List.excludeFlags(predicate: (String) -> Boolean) = filterNot(predicate) + // Exclude the listed elements. + private fun List.excludeFlags(vararg keys: String) = keys.toSet().let { keysToExclude -> + excludeFlags { it in keysToExclude } + } + + // Exclude the arguments passed by the given keys. + // E.g. if the list contains the following elements: ["-l", "foo", "-r", "bar"], + // call exclude("-r") returns the following list: ["-l", "foo"]. + private fun List.excludeArguments(vararg args: String): List { + val argsToExclude = args.toSet() + val xPrefixesToExclude = argsToExclude.filter { it.startsWith("-X") }.map { "$it=" } + val result = mutableListOf() + + var i = 0 + while (i < size) { + val key = this[i] + when { + key in argsToExclude -> { + // Skip the key and the following arg. + i++ + } + // Support args passed as -X=. + xPrefixesToExclude.any { key.startsWith(it) } -> { /* Skip the key. */ } + else -> result += key + } + i++ + } + return result + } + + // Don't include coverage flags into the first stage because they are not supported when compiling a klib. + private fun firstStageExtraOpts() = extraOpts + .excludeFlags("-Xcoverage") + .excludeArguments("-Xcoverage-file", "-Xlibrary-to-cover") + + // Don't include the -Xemit-lazy-objc-header flag into + // the second stage because this stage have no sources. + private fun secondStageExtraOpts() = extraOpts + .excludeArguments("-Xemit-lazy-objc-header") + + /** Args passed to the compiler at the first stage of two-stage compilation (klib building). */ + protected fun buildFirstStageArgs(klibPath: String) = mutableListOf().apply { + addArg("-output", klibPath) + addArg("-produce", CompilerOutputKind.LIBRARY.name.toLowerCase()) + + addAll(buildCommonArgs()) + + addAll(firstStageExtraOpts()) + + allSourceFiles.mapTo(this) { it.absolutePath } + commonSrcFiles + .flatMap { it.files } + .mapTo(this) { "-Xcommon-sources=${it.absolutePath}" } + } + + /** Args passed to the compiler at the second stage of two-stage compilation (producing a final binary from the klib). */ + protected fun buildSecondStageArgs(klibPath: String) = mutableListOf().apply { + addArg("-output", artifact.canonicalPath) + addArg("-produce", produce.name.toLowerCase()) + addArgIfNotNull("-entry", entryPoint) + + addAll(buildCommonArgs()) + + addFileArgs("-native-library", nativeLibraries) + linkerOpts.forEach { + addArg("-linker-option", it) + } + + addAll(secondStageExtraOpts()) + + addArg("-l", klibPath) + add("-Xsource-library=${klibPath}") + } + + /** Args passed to the compiler at both stages of the two-stage compilation and during the singe-stage compilation. */ + protected fun buildCommonArgs() = mutableListOf().apply { addArgs("-repo", libraries.repos.map { it.canonicalPath }) if (platformConfiguration.files.isNotEmpty()) { @@ -120,12 +202,6 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec { // The library's directory is added in libraries.repos. addArgs("-library", libraries.artifacts.map { it.artifact.nameWithoutExtension }) - addFileArgs("-native-library", nativeLibraries) - addArg("-produce", produce.name.toLowerCase()) - linkerOpts.forEach { - addArg("-linker-option", it) - } - addArgIfNotNull("-target", konanTarget.visibleName) addArgIfNotNull("-language-version", languageVersion) addArgIfNotNull("-api-version", apiVersion) @@ -143,6 +219,21 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec { if (libraries.friends.isNotEmpty()) addArg("-friend-modules", libraries.friends.joinToString(File.pathSeparator)) + } + + /** Args passed to the compiler if the two-stage compilation is disabled. */ + fun buildSingleStageArgs() = mutableListOf().apply { + addArg("-output", artifact.canonicalPath) + addArg("-produce", produce.name.toLowerCase()) + addArgIfNotNull("-entry", entryPoint) + + addAll(buildCommonArgs()) + + addFileArgs("-native-library", nativeLibraries) + + linkerOpts.forEach { + addArg("-linker-option", it) + } addAll(extraOpts) @@ -253,9 +344,37 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec { ) } // endregion + + override fun run() { + destinationDir.mkdirs() + if (dumpParameters) { + dumpProperties(this) + } + if (enableTwoStageCompilation) { + logger.info("Start two-stage compilation") + val intermediateDir = project.konanBuildRoot + .resolve("intermediate") + .targetSubdir(konanTarget) + .apply { mkdirs() } + val klibPrefix = CompilerOutputKind.LIBRARY.prefix(konanTarget) + val klibSuffix = CompilerOutputKind.LIBRARY.suffix(konanTarget) + val intermediateKlib = intermediateDir.resolve("$klibPrefix$artifactName$klibSuffix").absolutePath + logger.info("Start first stage") + toolRunner.run(buildFirstStageArgs(intermediateKlib)) + logger.info("Start second stage") + toolRunner.run(buildSecondStageArgs(intermediateKlib)) + } else { + toolRunner.run(buildSingleStageArgs()) + } + } } -open class KonanCompileProgramTask: KonanCompileTask() { +abstract class KonanCompileNativeBinary: KonanCompileTask() { + @Input + override var enableTwoStageCompilation: Boolean = false +} + +open class KonanCompileProgramTask: KonanCompileNativeBinary() { override val produce: CompilerOutputKind get() = CompilerOutputKind.PROGRAM var runTask: Exec? = null @@ -287,14 +406,14 @@ open class KonanCompileProgramTask: KonanCompileTask() { } -open class KonanCompileDynamicTask: KonanCompileTask() { +open class KonanCompileDynamicTask: KonanCompileNativeBinary() { override val produce: CompilerOutputKind get() = CompilerOutputKind.DYNAMIC val headerFile: File @OutputFile get() = destinationDir.resolve("$artifactPrefix${artifactName}_api.h") } -open class KonanCompileFrameworkTask: KonanCompileTask() { +open class KonanCompileFrameworkTask: KonanCompileNativeBinary() { override val produce: CompilerOutputKind get() = CompilerOutputKind.FRAMEWORK override val artifact @@ -303,8 +422,9 @@ open class KonanCompileFrameworkTask: KonanCompileTask() { open class KonanCompileLibraryTask: KonanCompileTask() { override val produce: CompilerOutputKind get() = CompilerOutputKind.LIBRARY + override val enableTwoStageCompilation: Boolean = false } -open class KonanCompileBitcodeTask: KonanCompileTask() { +open class KonanCompileBitcodeTask: KonanCompileNativeBinary() { override val produce: CompilerOutputKind get() = CompilerOutputKind.BITCODE } diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanInteropTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanInteropTask.kt index 3fa163f4c7b..18f5608c464 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanInteropTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanInteropTask.kt @@ -75,7 +75,7 @@ open class KonanInteropTask @Inject constructor(val workerExecutor: WorkerExecut @InputFiles val headers = mutableSetOf() @InputFiles val linkFiles = mutableSetOf() - override fun buildArgs() = mutableListOf().apply { + fun buildArgs() = mutableListOf().apply { addArg("-o", artifact.canonicalPath) addArgIfNotNull("-target", konanTarget.visibleName)