[Gradle IT] Added configuration for running IT with k/n from master

[Gradle] Updated kotlin-build-gradle-plugin to 0.0.40 #KT-45978 Ready for Review Merge-request: KT-MR-12509 Merged-by: Dmitrii Krasnov <Dmitrii.Krasnov@jetbrains.com>
2023-10-24 08:38:51 +00:00
parent 80485809d9
commit 10a6d8fd2c
22 changed files with 146 additions and 70 deletions
@@ -34,6 +34,19 @@ If you want to run only one test class, you need to append `--tests` flag with v
 ./gradlew :kotlin-gradle-plugin-integration-tests:kgpAllParallelTests --tests <class-name-with-package>
 ```

+#### How to Run Using Kotlin Native from Master
+
+Currently, Kotlin Native from master involves three configurations: `kgpMppTests`, `kgpNativeTests`, and `kgpOtherTests`. 
+Depending on your development environment, there are a few different ways you can run this.
+* **On Local Environment** In the case of Local Environment builds, you have two options, which you can add in your `local.properties` file:  
+  * `kotlin.native.enabled=true` - this property adds building Kotlin Native full bundle step before Integration Tests, then this bundle will be used in the Integration Tests.
+  * `kotlin.native.local.distribution.for.tests.enabled=false` - include this line if you need to disable running Integration Tests with Kotlin/Native from master, even when `kotlin.native.enabled` is set to true.
+* **On TeamCity** In the case of TeamCity builds, no extra setting is necessary. The mentioned configurations depend on the `full bundle`, which stores its artifacts in the `-DkonanDataDirForIntegrationTests` directory.
+Also, you can specify the konan directory for test by providing `-DkonanDataDirForIntegrationTests`, for example:
+```bash
+ ./gradlew :kotlin-gradle-plugin-integration-tests:kgpNativeTests -DkonanDataDirForIntegrationTests=/tmp/.konan
+```
+
 #### How to work with the tests

 Few rules you should follow while writing tests:
@@ -1,5 +1,6 @@
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
-import org.gradle.jvm.toolchain.internal.NoToolchainAvailableException
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.pill.PillExtension
 import java.nio.file.Paths

@@ -111,6 +112,11 @@ dependencies {
    testCompileOnly(commonDependency("org.jetbrains.intellij.deps:asm-all"))
 }

+val konanDataDir: String = System.getProperty("konanDataDirForIntegrationTests")
+    ?: project.rootDir
+        .resolve(".kotlin")
+        .resolve("konan-for-gradle-tests").absolutePath
+
 // Aapt2 from Android Gradle Plugin 3.2 and below does not handle long paths on Windows.
 val shortenTempRootName = project.providers.systemProperty("os.name").get().contains("Windows")

@@ -130,13 +136,52 @@ val cleanTestKitCacheTask = tasks.register<Delete>("cleanTestKitCache") {
 }

 tasks.register<Delete>("cleanUserHomeKonanDir") {
-    group = KGP_TEST_TASKS_GROUP
-    description = "Deletes ~/.konan dir before tests. This step is necessary to ensure that no test inadvertently creates this directory during execution."
+    description =
+        "Deletes ~/.konan dir before tests. This step is necessary to ensure that no test inadvertently creates this directory during execution."

    val userHomeKonanDir = Paths.get("${System.getProperty("user.home")}/.konan")
+
    delete(userHomeKonanDir)

-    println("Default .konan directory user's home has been deleted: $userHomeKonanDir")
+    doLast {
+        logger.info("Default .konan directory user's home has been deleted: $userHomeKonanDir")
+    }
+}
+
+tasks.register<Copy>("prepareNativeBundleForGradleIT") {
+
+    description = "This task adds depenecy on :kotlin-native:bundle and then copying built bundle into the tests' konan dir"
+
+    // 1. Build full Kotlin Native bundle
+    dependsOn(":kotlin-native:bundle")
+
+    // 2. Coping and extracting k/n artifacts from the 1st step to tests' konan data directory
+    val (extension, unzipFunction) = when (HostManager.host) {
+        KonanTarget.MINGW_X64 -> Pair("zip", ::zipTree)
+        else -> Pair("tar.gz", ::tarTree)
+    }
+
+    val kotlinNativeRootDir = rootProject.findProject(":kotlin-native")?.projectDir
+        ?: throw IllegalStateException("The path to kotlin-native module is undefined.")
+
+    from(
+        unzipFunction(
+            kotlinNativeRootDir.resolve("kotlin-native-${HostManager.platformName()}-${project.kotlinBuildProperties.defaultSnapshotVersion}.$extension")
+        )
+    )
+    from(
+        unzipFunction(
+            kotlinNativeRootDir.resolve("kotlin-native-prebuilt-${HostManager.platformName()}-${project.kotlinBuildProperties.defaultSnapshotVersion}.$extension")
+        )
+    )
+
+    into(
+        konanDataDir
+    )
+
+    doFirst {
+        delete(konanDataDir)
+    }
 }

 fun Test.includeMppAndAndroid(include: Boolean) = includeTestsWithPattern(include) {
@@ -147,6 +192,28 @@ fun Test.includeNative(include: Boolean) = includeTestsWithPattern(include) {
    addAll(listOf("org.jetbrains.kotlin.gradle.native.*", "*Commonizer*"))
 }

+fun Test.applyKotlinNativeFromCurrentBranchIfNeeded() {
+    val kotlinNativeFromMasterEnabled = project.kotlinBuildProperties.isKotlinNativeEnabled && project.kotlinBuildProperties.useKotlinNativeLocalDistributionForTests
+    if (kotlinNativeFromMasterEnabled && !project.kotlinBuildProperties.isTeamcityBuild) {
+        dependsOn(":kotlin-gradle-plugin-integration-tests:prepareNativeBundleForGradleIT")
+    }
+
+    // Providing necessary properties for running tests with k/n built from master on the local environment
+    val defaultSnapshotVersion = project.kotlinBuildProperties.defaultSnapshotVersion
+    if (kotlinNativeFromMasterEnabled && defaultSnapshotVersion != null) {
+        systemProperty("kotlinNativeVersion", defaultSnapshotVersion)
+        systemProperty("konanDataDirForIntegrationTests", konanDataDir)
+    }
+
+    // Providing necessary properties for running tests with k/n built from master on the TeamCity
+    if (project.kotlinBuildProperties.isTeamcityBuild) {
+        System.getProperty("kotlinNativeVersionForGradleIT")?.let {
+            systemProperty("kotlinNativeVersion", it)
+        }
+        systemProperty("konanDataDirForIntegrationTests", konanDataDir)
+    }
+}
+
 fun Test.includeTestsWithPattern(include: Boolean, patterns: (MutableSet<String>).() -> Unit) {
    if (splitGradleIntegrationTestTasks) {
        val filter = if (include)
@@ -250,6 +317,7 @@ val nativeTestsTask = tasks.register<Test>("kgpNativeTests") {
        excludeTags("JvmKGP", "JsKGP", "DaemonsKGP", "OtherKGP", "MppKGP", "AndroidKGP")
        includeEngines("junit-jupiter")
    }
+    applyKotlinNativeFromCurrentBranchIfNeeded()
 }

 // Daemon tests could run only sequentially as they could not be shared between parallel test builds
@@ -274,6 +342,7 @@ val otherPluginsTestTask = tasks.register<Test>("kgpOtherTests") {
        excludeTags("JvmKGP", "JsKGP", "NativeKGP", "DaemonsKGP", "MppKGP", "AndroidKGP")
        includeEngines("junit-jupiter")
    }
+    applyKotlinNativeFromCurrentBranchIfNeeded()
 }

 val mppTestsTask = tasks.register<Test>("kgpMppTests") {
@@ -285,6 +354,7 @@ val mppTestsTask = tasks.register<Test>("kgpMppTests") {
        excludeTags("JvmKGP", "JsKGP", "NativeKGP", "DaemonsKGP", "OtherKGP", "AndroidKGP")
        includeEngines("junit-jupiter")
    }
+    applyKotlinNativeFromCurrentBranchIfNeeded()
 }

 val androidTestsTask = tasks.register<Test>("kgpAndroidTests") {
@@ -66,7 +66,7 @@ open class CommonizerIT : KGPBaseTest() {
    @GradleTest
    fun testCleanNativeDistributionCommonization(gradleVersion: GradleVersion, @TempDir konanData: Path) {
        nativeProject("commonizeNativeDistributionWithIosLinuxWindows", gradleVersion) {
-            val buildOptions = defaultBuildOptions.copy(
+            val buildOptions = defaultBuildOptions.withBundledKotlinNative().copy(
                konanDataDir = konanData
            )

@@ -834,7 +834,11 @@ class GeneralNativeIT : KGPBaseTest() {
                }

                // Check that changing K/N version lead to tasks rerun
-                build(compileTasks, "-Porg.jetbrains.kotlin.native.version=${TestVersions.Kotlin.STABLE_RELEASE}") {
+                build(
+                    compileTasks, buildOptions = defaultBuildOptions.copy(
+                        nativeOptions = defaultBuildOptions.nativeOptions.copy(version = TestVersions.Kotlin.STABLE_RELEASE)
+                    )
+                ) {
                    assertTasksExecuted(compileTasks)
                }
            }
@@ -15,15 +15,12 @@ import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.presetName
 import org.junit.jupiter.api.Assumptions
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.DisplayName
 import org.junit.jupiter.api.condition.OS
 import org.junit.jupiter.api.io.TempDir
 import java.nio.file.Path
-import java.nio.file.Paths
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.appendText
-import kotlin.io.path.deleteRecursively

 // We temporarily disable it for windows until a proper fix is found for this issue:
 // https://youtrack.jetbrains.com/issue/KT-60138/NativeDownloadAndPlatformLibsIT-fails-on-Windows-OS
@@ -43,12 +40,12 @@ class NativeDownloadAndPlatformLibsIT : KGPBaseTest() {
    private val currentCompilerVersion = NativeCompilerDownloader.DEFAULT_KONAN_VERSION

    override val defaultBuildOptions: BuildOptions
-        get() = super.defaultBuildOptions.copy(
+        get() = super.defaultBuildOptions.withBundledKotlinNative().copy(
            // For each test in this class, we need to provide an isolated .konan directory,
            // so we create it within each test project folder
            konanDataDir = workingDir.resolve(".konan")
                .toFile()
-                .apply { mkdirs() }.toPath()
+                .apply { mkdirs() }.toPath(),
        )

    @OptIn(EnvironmentalVariablesOverride::class)
@@ -26,6 +26,9 @@ class NativeDownloadAndPlatformLibsNonParallelIT : KGPDaemonsBaseTest() {
    private val platformName: String = HostManager.platformName()
    private val currentCompilerVersion = NativeCompilerDownloader.DEFAULT_KONAN_VERSION

+    override val defaultBuildOptions: BuildOptions
+        get() = super.defaultBuildOptions.withBundledKotlinNative().copy()
+

    @DisplayName("Downloading K/N distribution in default .konan dir")
    @GradleTest
@@ -22,19 +22,20 @@ class NativeWithConfigurationCacheIT : KGPBaseTest() {
    @GradleTestVersions(minVersion = TestVersions.Gradle.G_8_1) // Since 8.1 Gradle on configuration cache it detects when the build logic accesses the "outside world" more strict https://docs.gradle.org/8.1.1/release-notes.html#configuration-inputs-detection-improvements
    @GradleTest
    fun testConfigurationCacheReusedSecondTime(gradleVersion: GradleVersion) {
-        nativeProject("native-with-configuration-cache", gradleVersion) {
-            // we need to download compiler on the first build, that is why we are setting custom konan home dir without any compiler inside
-            val localKonan = workingDir.resolve(".konan")
+        nativeProject(
+            "native-with-configuration-cache", gradleVersion, buildOptions = defaultBuildOptions.withBundledKotlinNative().copy(
+                // We need to download compiler on the first build, that is why we are setting custom konan home dir without any compiler inside
+                konanDataDir = workingDir.resolve(".konan"),
+            )
+        ) {
            build(
-                "help", buildOptions = defaultBuildOptions.copy(
-                    konanDataDir = localKonan
-                )
+                "help"
            ) {
                assertOutputContains("Configure project")
                assertOutputContains("Unpack Kotlin/Native compiler to")
            }

-            build("help", "-Pkonan.data.dir=$localKonan") {
+            build("help") {
                assertOutputContains("Reusing configuration cache.")
            }
        }
@@ -18,7 +18,6 @@ import org.jetbrains.kotlin.gradle.report.BuildReportType
 import org.junit.jupiter.api.condition.OS
 import java.nio.file.Path
 import java.util.*
-import kotlin.io.path.absolutePathString

 data class BuildOptions(
    val logLevel: LogLevel = LogLevel.INFO,
@@ -91,7 +90,7 @@ data class BuildOptions(
        val reinstall: Boolean? = null,
        val restrictedDistribution: Boolean? = null,
        val useXcodeMessageStyle: Boolean? = null,
-        val version: String? = null,
+        val version: String? = System.getProperty("kotlinNativeVersion"),
        val cacheOrchestration: String? = null,
        val incremental: Boolean? = null,
    )
@@ -286,4 +285,18 @@ fun BuildOptions.suppressDeprecationWarningsSinceGradleVersion(
    reason: String,
 ) = suppressDeprecationWarningsOn(reason) {
    currentGradleVersion >= GradleVersion.version(gradleVersion)
-}
+}
+
+/**
+ * This wrapper erases k/n version from passing parameters,
+ * because we should use Kotlin Native bundled in KGP instead of which built from current branch.
+ *
+ * In this case we will use k/n version, which declared in KGP.
+ *
+ * The most common case is when we override local konan dir for some reason.
+ */
+fun BuildOptions.withBundledKotlinNative() = copy(
+    nativeOptions = nativeOptions.copy(
+        version = null
+    )
+)
@@ -452,7 +452,17 @@ private fun TestProject.withBuildSummary(
    }
 }

-val konanDir get() = Paths.get(".").resolve("../../../build").resolve("konan-for-gradle-tests")
+/**
+ * This property is configured reade konan from specific directory, which in teamcity will be filled with k/n built from master.
+ * NOTE: On changing test konan dir location update related location in kotlin-teamcity-build repository
+ */
+val konanDir
+    get() =
+        System.getProperty("konanDataDirForIntegrationTests")?.let {
+            Paths.get(it)
+        } ?: Paths.get(".")
+            .resolve("../../../.kotlin")
+            .resolve("konan-for-gradle-tests")

 /**
 * On changing test kit dir location update related location in 'cleanTestKitCache' task.
@@ -13,7 +13,6 @@ enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: Bo

    // whether the build is executed from IDE or from console
    EXECUTED_FROM_IDEA(OVERRIDE, SAFE),
-
    // Build script

    //annotation processors