From c49128aa99cbe292870204d9b897f9c241c6cb99 Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Mon, 6 Feb 2023 19:39:58 +0100 Subject: [PATCH] [K/N][perf] Performance project build * Add build-tools plugin * Cleanup code * Add missing methods after the move --- kotlin-native/build-tools/gradle.properties | 2 +- .../main/kotlin/org/jetbrains/kotlin/Utils.kt | 20 ------- kotlin-native/performance/build.gradle | 6 +-- .../performance/buildSrc/build.gradle.kts | 27 +++------- .../performance/buildSrc/gradle.properties | 2 +- .../{PlatfromInfo.kt => PlatformInfo.kt} | 4 ++ .../src/main/kotlin/{Misc.kt => Utils.kt} | 32 ++++++++++- .../benchmark/SwiftBenchmarkingPlugin.kt | 53 +++++++++++++++++++ .../gradle/wrapper/gradle-wrapper.properties | 6 +++ kotlin-native/performance/settings.gradle | 2 + .../tools/benchmarksAnalyzer/build.gradle | 3 +- .../buildSrc/build.gradle.kts | 19 +++---- .../gradle/wrapper/gradle-wrapper.properties | 6 +++ .../plugin/konan/tasks/KonanCacheTask.kt | 2 +- .../tools/performance-server/build.gradle.kts | 3 +- .../buildSrc/build.gradle.kts | 25 +++++---- .../utilities/cli-runner/build.gradle | 1 + 17 files changed, 137 insertions(+), 76 deletions(-) rename kotlin-native/performance/buildSrc/src/main/kotlin/{PlatfromInfo.kt => PlatformInfo.kt} (86%) rename kotlin-native/performance/buildSrc/src/main/kotlin/{Misc.kt => Utils.kt} (67%) create mode 100644 kotlin-native/performance/gradle/wrapper/gradle-wrapper.properties create mode 100644 kotlin-native/tools/benchmarksAnalyzer/gradle/wrapper/gradle-wrapper.properties diff --git a/kotlin-native/build-tools/gradle.properties b/kotlin-native/build-tools/gradle.properties index f66525572dd..8c2dce3fd96 100644 --- a/kotlin-native/build-tools/gradle.properties +++ b/kotlin-native/build-tools/gradle.properties @@ -1,5 +1,5 @@ cacheRedirectorEnabled=true -bootstrap.kotlin.default.version=1.8.20-dev-5197 +bootstrap.kotlin.default.version=1.9.0-dev-877 kotlin.build.gradlePlugin.version=0.0.39 org.gradle.java.installations.fromEnv=\ diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt index cc789464e26..a2b3958a30e 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt @@ -293,26 +293,6 @@ fun Task.dependsOnKonanBuildingTask(artifact: String, target: KonanTarget) { } //endregion -// Run command line from string. -fun Array.runCommand( - workingDir: File = File("."), - timeoutAmount: Long = 60, - timeoutUnit: TimeUnit = TimeUnit.SECONDS -): String { - return try { - ProcessBuilder(*this) - .directory(workingDir) - .redirectOutput(ProcessBuilder.Redirect.PIPE) - .redirectError(ProcessBuilder.Redirect.PIPE) - .start().apply { - waitFor(timeoutAmount, timeoutUnit) - }.inputStream.bufferedReader().readText() - } catch (e: Exception) { - println("Couldn't run command ${this.joinToString(" ")}") - println(e.stackTrace.joinToString("\n")) - error(e.message!!) - } -} @JvmOverloads fun compileSwift( diff --git a/kotlin-native/performance/build.gradle b/kotlin-native/performance/build.gradle index 1a533db5d38..ce79eb34098 100644 --- a/kotlin-native/performance/build.gradle +++ b/kotlin-native/performance/build.gradle @@ -1,7 +1,6 @@ import org.jetbrains.kotlin.BuildRegister import org.jetbrains.kotlin.MPPTools import org.jetbrains.kotlin.UtilsKt -import org.jetbrains.kotlin.konan.CompilerVersionKt import org.jetbrains.kotlin.konan.target.DistributionKt import org.jetbrains.kotlin.konan.target.PlatformManager @@ -17,7 +16,7 @@ buildscript { } } ext["withoutEmbedabble"] = true - MiscKt.kotlinInit(project, findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean()) + UtilsKt.kotlinInit(project, findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean()) ext["bootstrapKotlinRepo"] = BootstrapKt.getBootstrapKotlinRepo(project) ext["bootstrapKotlinVersion"] = BootstrapKt.getBootstrapKotlinVersion(project) @@ -30,6 +29,7 @@ buildscript { } plugins { + id("kotlin.native.build-tools-conventions") id("org.jetbrains.kotlin.multiplatform") apply false } @@ -62,7 +62,7 @@ subprojects { proj -> proj.logger.info("${proj.name}<<<[$key] = $value>>>") proj.ext.set(key, value) } - MiscKt.kotlinInit(proj, findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() ?: false) + UtilsKt.kotlinInit(proj, findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() ?: false) proj.ext.platformManager = platformManager proj.ext["bootstrapKotlinRepo"] = BootstrapKt.getBootstrapKotlinRepo(proj) proj.ext["bootstrapKotlinVersion"] = BootstrapKt.getBootstrapKotlinVersion(proj) diff --git a/kotlin-native/performance/buildSrc/build.gradle.kts b/kotlin-native/performance/buildSrc/build.gradle.kts index 221e677b6df..b6a40e8b80d 100644 --- a/kotlin-native/performance/buildSrc/build.gradle.kts +++ b/kotlin-native/performance/buildSrc/build.gradle.kts @@ -38,19 +38,12 @@ tasks.validatePlugins.configure { } -sourceSets["main"].withConvention(org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet::class) { - kotlin.filter.exclude("**/FileCheckTest.kt") - // TODO: Consider moving required stuff from kotlin-native/build-tools/ to buildSrc/ here. - kotlin.filter.exclude("**/bitcode/**") - kotlin.filter.exclude("**/cpp/**") - kotlin.filter.exclude("**/testing/**") - - kotlin.srcDir("src/main/kotlin") - kotlin.srcDir("../../build-tools/src/main/kotlin") - kotlin.srcDir("../../shared/src/library/kotlin") - kotlin.srcDir("../../shared/src/main/kotlin") - kotlin.srcDir("../../tools/benchmarks/shared/src/main/kotlin/report") - kotlin.srcDir("../../../native/utils/src") +sourceSets["main"].kotlin { + srcDir("src/main/kotlin") + srcDir("../../shared/src/library/kotlin") + srcDir("../../shared/src/main/kotlin") + srcDir("../../tools/benchmarks/shared/src/main/kotlin/report") + srcDir("../../../native/utils/src") } tasks.withType().configureEach { @@ -80,8 +73,6 @@ dependencies { implementation("io.ktor:ktor-client-core:$ktorVersion") implementation("io.ktor:ktor-client-cio:$ktorVersion") - api("org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion") - // Located in /shared and always provided by the composite build. //api("org.jetbrains.kotlin:kotlin-native-shared:$konanVersion") implementation("gradle.plugin.com.github.johnrengelman:shadow:$shadowVersion") @@ -115,12 +106,6 @@ gradlePlugin { } afterEvaluate { - tasks.withType().configureEach { - kotlinOptions { - languageVersion = "1.4" - apiVersion = "1.4" - } - } tasks.withType { targetCompatibility = "1.8" } diff --git a/kotlin-native/performance/buildSrc/gradle.properties b/kotlin-native/performance/buildSrc/gradle.properties index fb90f559bb7..ddf93622930 100644 --- a/kotlin-native/performance/buildSrc/gradle.properties +++ b/kotlin-native/performance/buildSrc/gradle.properties @@ -1 +1 @@ -kotlin.build.gradlePlugin.version=0.0.39 +kotlin.build.gradlePlugin.version=0.0.39 \ No newline at end of file diff --git a/kotlin-native/performance/buildSrc/src/main/kotlin/PlatfromInfo.kt b/kotlin-native/performance/buildSrc/src/main/kotlin/PlatformInfo.kt similarity index 86% rename from kotlin-native/performance/buildSrc/src/main/kotlin/PlatfromInfo.kt rename to kotlin-native/performance/buildSrc/src/main/kotlin/PlatformInfo.kt index d87cfea237c..8c735de54fe 100644 --- a/kotlin-native/performance/buildSrc/src/main/kotlin/PlatfromInfo.kt +++ b/kotlin-native/performance/buildSrc/src/main/kotlin/PlatformInfo.kt @@ -15,4 +15,8 @@ object PlatformInfo { @JvmStatic fun isLinux() = HostManager.hostIsLinux + + @JvmStatic + val hostName: String + get() = HostManager.hostName } \ No newline at end of file diff --git a/kotlin-native/performance/buildSrc/src/main/kotlin/Misc.kt b/kotlin-native/performance/buildSrc/src/main/kotlin/Utils.kt similarity index 67% rename from kotlin-native/performance/buildSrc/src/main/kotlin/Misc.kt rename to kotlin-native/performance/buildSrc/src/main/kotlin/Utils.kt index c8fe6c3a109..68ce49c5f29 100644 --- a/kotlin-native/performance/buildSrc/src/main/kotlin/Misc.kt +++ b/kotlin-native/performance/buildSrc/src/main/kotlin/Utils.kt @@ -2,7 +2,16 @@ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ +package org.jetbrains.kotlin + +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.* +import java.util.concurrent.TimeUnit +import kotlinBuildProperties import org.gradle.api.Project +import org.jetbrains.kotlin.konan.target.* fun Project.kotlinInit(cacheRedirectorEnabled: Boolean) { extensions.extraProperties["defaultSnapshotVersion"] = kotlinBuildProperties.defaultSnapshotVersion @@ -37,4 +46,25 @@ fun sendGetRequest(url: String, username: String? = null, password: String? = nu connection.setRequestProperty("Accept", "application/json"); connection.connect() return connection.inputStream.use { it.reader().use { reader -> reader.readText() } } -} \ No newline at end of file +} + +val Project.platformManager + get() = findProperty("platformManager") as PlatformManager + +val validPropertiesNames = listOf( + "konan.home", + "org.jetbrains.kotlin.native.home", + "kotlin.native.home" +) + +val Project.kotlinNativeDist + get() = rootProject.currentKotlinNativeDist + +val Project.currentKotlinNativeDist + get() = file(validPropertiesNames.firstOrNull { hasProperty(it) }?.let { findProperty(it) } ?: "dist") + +val kotlinNativeHome + get() = validPropertiesNames.mapNotNull(System::getProperty).first() + +val Project.useCustomDist + get() = validPropertiesNames.any { hasProperty(it) } \ No newline at end of file diff --git a/kotlin-native/performance/buildSrc/src/main/kotlin/benchmark/SwiftBenchmarkingPlugin.kt b/kotlin-native/performance/buildSrc/src/main/kotlin/benchmark/SwiftBenchmarkingPlugin.kt index 54f8c280b51..60573fe7417 100644 --- a/kotlin-native/performance/buildSrc/src/main/kotlin/benchmark/SwiftBenchmarkingPlugin.kt +++ b/kotlin-native/performance/buildSrc/src/main/kotlin/benchmark/SwiftBenchmarkingPlugin.kt @@ -9,9 +9,13 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.Framework import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinNativeTargetPreset import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType +import org.jetbrains.kotlin.konan.target.* import java.io.File import javax.inject.Inject import java.nio.file.Paths +import java.net.URL +import java.util.concurrent.TimeUnit +import java.nio.file.Path import kotlin.reflect.KClass enum class CodeSizeEntity { FRAMEWORK, EXECUTABLE } @@ -93,6 +97,55 @@ open class SwiftBenchmarkingPlugin : BenchmarkingPlugin() { } } + fun Array.runCommand( + workingDir: File = File("."), + timeoutAmount: Long = 60, + timeoutUnit: TimeUnit = TimeUnit.SECONDS + ): String { + return try { + ProcessBuilder(*this) + .directory(workingDir) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .redirectError(ProcessBuilder.Redirect.PIPE) + .start().apply { + waitFor(timeoutAmount, timeoutUnit) + }.inputStream.bufferedReader().readText() + } catch (e: Exception) { + println("Couldn't run command ${this.joinToString(" ")}") + println(e.stackTrace.joinToString("\n")) + error(e.message!!) + } + } + fun compileSwift( + project: Project, target: KonanTarget, sources: List, options: List, + output: Path, fullBitcode: Boolean = false + ) { + val platform = project.platformManager.platform(target) + assert(platform.configurables is AppleConfigurables) + val configs = platform.configurables as AppleConfigurables + val compiler = configs.absoluteTargetToolchain + "/usr/bin/swiftc" + + val swiftTarget = configs.targetTriple.withOSVersion(configs.osVersionMin).toString() + + val args = listOf("-sdk", configs.absoluteTargetSysRoot, "-target", swiftTarget) + + options + "-o" + output.toString() + sources + + if (fullBitcode) listOf("-embed-bitcode", "-Xlinker", "-bitcode_verify") else listOf("-embed-bitcode-marker") + + val out = mutableListOf().apply { + add(compiler) + addAll(args) + }.toTypedArray().runCommand(timeoutAmount = 240) + + println( + """ + |$compiler finished with: + |options: ${args.joinToString(separator = " ")} + |output: $out + """.trimMargin() + ) + check(output.toFile().exists()) { "Compiler swiftc hasn't produced an output file: $output" } + } + override fun Project.collectCodeSize(applicationName: String) = getCodeSizeBenchmark(applicationName, if (benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK) diff --git a/kotlin-native/performance/gradle/wrapper/gradle-wrapper.properties b/kotlin-native/performance/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..9699c836aa8 --- /dev/null +++ b/kotlin-native/performance/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionSha256Sum=b586e04868a22fd817c8971330fec37e298f3242eb85c374181b12d637f80302 +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/kotlin-native/performance/settings.gradle b/kotlin-native/performance/settings.gradle index c6c40d44ba5..9574fa26a61 100644 --- a/kotlin-native/performance/settings.gradle +++ b/kotlin-native/performance/settings.gradle @@ -13,6 +13,8 @@ pluginManagement { } } +includeBuild '../build-tools' + include ':ring' include ':cinterop' include ':helloworld' diff --git a/kotlin-native/tools/benchmarksAnalyzer/build.gradle b/kotlin-native/tools/benchmarksAnalyzer/build.gradle index 0cba2a4da52..82210a8e75f 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/build.gradle +++ b/kotlin-native/tools/benchmarksAnalyzer/build.gradle @@ -1,4 +1,5 @@ import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.UtilsKt buildscript { ext.rootBuildDirectory = file('../..') @@ -14,7 +15,7 @@ buildscript { } } ext["withoutEmbedabble"] = true - MiscKt.kotlinInit(project, findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() ?: false) + UtilsKt.kotlinInit(project, findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() ?: false) ext["bootstrapKotlinRepo"] = BootstrapKt.getBootstrapKotlinRepo(project) ext["bootstrapKotlinVersion"] = BootstrapKt.getBootstrapKotlinVersion(project) ext["kotlinVersion"] = project.bootstrapKotlinVersion diff --git a/kotlin-native/tools/benchmarksAnalyzer/buildSrc/build.gradle.kts b/kotlin-native/tools/benchmarksAnalyzer/buildSrc/build.gradle.kts index 7d66c47d0c3..68230418705 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/buildSrc/build.gradle.kts +++ b/kotlin-native/tools/benchmarksAnalyzer/buildSrc/build.gradle.kts @@ -42,19 +42,12 @@ tasks.validatePlugins.configure { } -sourceSets["main"].withConvention(org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet::class) { - kotlin.filter.exclude("**/FileCheckTest.kt") - // TODO: Consider moving required stuff from kotlin-native/build-tools/ to buildSrc/ here. - kotlin.filter.exclude("**/bitcode/**") - kotlin.filter.exclude("**/cpp/**") - kotlin.filter.exclude("**/testing/**") - - kotlin.srcDir("../../../build-tools/src/main/kotlin") - kotlin.srcDir("../../../performance/buildSrc/src/main/kotlin") - kotlin.srcDir("../../../shared/src/library/kotlin") - kotlin.srcDir("../../../shared/src/main/kotlin") - kotlin.srcDir("../../benchmarks/shared/src/main/kotlin/report") - kotlin.srcDir("../../../../native/utils/src") +sourceSets["main"].kotlin { + srcDir("../../../performance/buildSrc/src/main/kotlin") + srcDir("../../../shared/src/library/kotlin") + srcDir("../../../shared/src/main/kotlin") + srcDir("../../benchmarks/shared/src/main/kotlin/report") + srcDir("../../../../native/utils/src") } tasks.withType().configureEach { diff --git a/kotlin-native/tools/benchmarksAnalyzer/gradle/wrapper/gradle-wrapper.properties b/kotlin-native/tools/benchmarksAnalyzer/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..9699c836aa8 --- /dev/null +++ b/kotlin-native/tools/benchmarksAnalyzer/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionSha256Sum=b586e04868a22fd817c8971330fec37e298f3242eb85c374181b12d637f80302 +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt b/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt index 9d3dc3faf7f..e33f120c1c0 100644 --- a/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt +++ b/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt @@ -74,7 +74,7 @@ open class KonanCacheTask: DefaultTask() { @TaskAction fun compile() { // This code uses bootstrap version of util-klib and fails due to the older default ABI than library being used - // A possible solution is to read it manually from manifest file or this check should be done bu the compiler itself + // A possible solution is to read it manually from manifest file or this check should be done by the compiler itself // check(klibUniqName == readKlibUniqNameFromManifest()) { // "klibUniqName mismatch: configured '$klibUniqName', resolved '${readKlibUniqNameFromManifest()}'" // } diff --git a/kotlin-native/tools/performance-server/build.gradle.kts b/kotlin-native/tools/performance-server/build.gradle.kts index f3915ec543f..08db0d2b094 100644 --- a/kotlin-native/tools/performance-server/build.gradle.kts +++ b/kotlin-native/tools/performance-server/build.gradle.kts @@ -13,7 +13,8 @@ buildscript { } extra["withoutEmbedabble"] = true - project.kotlinInit(findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() ?: false) + extra["defaultSnapshotVersion"] = kotlinBuildProperties.defaultSnapshotVersion + extra["kotlinVersion"] = findProperty("kotlinVersion") val bootstrapKotlinVersion: String by extra(project.bootstrapKotlinVersion) val kotlinVersion: String by extra(bootstrapKotlinVersion) diff --git a/kotlin-native/tools/performance-server/buildSrc/build.gradle.kts b/kotlin-native/tools/performance-server/buildSrc/build.gradle.kts index b0fb888cacb..5d1eda47f7a 100644 --- a/kotlin-native/tools/performance-server/buildSrc/build.gradle.kts +++ b/kotlin-native/tools/performance-server/buildSrc/build.gradle.kts @@ -34,23 +34,22 @@ repositories { mavenCentral() } +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(8)) + } +} + tasks.validatePlugins.configure { enabled = false } -sourceSets["main"].withConvention(org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet::class) { - kotlin.filter.exclude("**/FileCheckTest.kt") - // TODO: Consider moving required stuff from kotlin-native/build-tools/ to buildSrc/ here. - kotlin.filter.exclude("**/bitcode/**") - kotlin.filter.exclude("**/cpp/**") - kotlin.filter.exclude("**/testing/**") - - kotlin.srcDir("../../../build-tools/src/main/kotlin") - kotlin.srcDir("../../../performance/buildSrc/src/main/kotlin") - kotlin.srcDir("../../../shared/src/library/kotlin") - kotlin.srcDir("../../../shared/src/main/kotlin") - kotlin.srcDir("../../benchmarks/shared/src/main/kotlin/report") - kotlin.srcDir("../../../../native/utils/src") +sourceSets["main"].kotlin { + srcDir("../../../performance/buildSrc/src/main/kotlin") + srcDir("../../../shared/src/library/kotlin") + srcDir("../../../shared/src/main/kotlin") + srcDir("../../benchmarks/shared/src/main/kotlin/report") + srcDir("../../../../native/utils/src") } tasks.withType().configureEach { diff --git a/kotlin-native/utilities/cli-runner/build.gradle b/kotlin-native/utilities/cli-runner/build.gradle index ca82eceb076..f48c8a8d57f 100644 --- a/kotlin-native/utilities/cli-runner/build.gradle +++ b/kotlin-native/utilities/cli-runner/build.gradle @@ -24,4 +24,5 @@ dependencies { implementation project(':kotlin-native:klib') implementation project(":kotlin-native:utilities:basic-utils") implementation project(path: ":kotlin-native:endorsedLibraries:kotlinx.cli", configuration: "jvmRuntimeElements") + implementation project(":kotlin-native-shared") }