[Gradle IT] Added configuration for running IT with k/n from master

[Gradle] Updated kotlin-build-gradle-plugin to 0.0.40 #KT-45978 Ready for Review Merge-request: KT-MR-12509 Merged-by: Dmitrii Krasnov <Dmitrii.Krasnov@jetbrains.com>
2023-10-24 08:38:51 +00:00
parent 80485809d9
commit 10a6d8fd2c
22 changed files with 146 additions and 70 deletions
@@ -57,3 +57,4 @@ outTmp/
 /test.output
 /kotlin-native/dist
 kotlin-ide/
+/.kotlin/
@@ -29,7 +29,7 @@ cacheRedirectorEnabled=true
 defaultSnapshotVersion=2.0.255-SNAPSHOT
 kotlinLanguageVersion=2.0

-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40

 #maven.repository.mirror=http://repository.jetbrains.com/remote-repos/
 #bootstrap.kotlin.repo=https://maven.pkg.jetbrains.space/kotlin/p/kotlin/bootstrap
@@ -3770,10 +3770,10 @@
            <sha256 value="2324882ccb36833847b47fc397b50ad46bf96a8959b98ec68f6dc08b47dd7e57" origin="Generated by Gradle"/>
         </artifact>
      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-build-gradle-plugin" version="0.0.39">
-         <artifact name="kotlin-build-gradle-plugin-0.0.39.jar">
-            <md5 value="11833430f44f0ffbedbdd2a5bb9fa871" origin="Generated by Gradle"/>
-            <sha256 value="7e83e579356ef9f2e7df756b436b117a358c3ce542d9b535014f01a59b6fe49e" origin="Generated by Gradle"/>
+      <component group="org.jetbrains.kotlin" name="kotlin-build-gradle-plugin" version="0.0.40">
+         <artifact name="kotlin-build-gradle-plugin-0.0.40.jar">
+            <md5 value="051a39904e8285e78cbcd1dc8e2b221a" origin="Generated by Gradle"/>
+            <sha256 value="ea3644137bdf9006eecd1fa1e6771548de3c9b335ffac6387bff01400ab63bd9" origin="Generated by Gradle"/>
         </artifact>
      </component>
      <component group="org.jetbrains.kotlin" name="kotlin-gradle-plugin-annotations" version="1.8.20-dev-4242">
@@ -3840,12 +3840,6 @@
            <sha256 value="655825fa6511220fed0cae5aad4dfea2abf98fd10f0ae77eb8daf6abfcd0f027" origin="Generated by Gradle"/>
         </artifact>
      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib" version="1.5.31">
-         <artifact name="kotlin-stdlib-1.5.31.jar">
-            <md5 value="9bebda8bdc25a3a01c836b7616179f6c" origin="Generated by Gradle"/>
-            <sha256 value="4800ceacb2ec0bb9959a087154b8e35318ead1ea4eba32d4bb1b9734222a7e68" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
      <component group="org.jetbrains.kotlin" name="kotlin-stdlib" version="1.6.0">
         <artifact name="kotlin-stdlib-1.6.0.jar">
            <md5 value="74581955072dfff26c6928c6296b15b2" origin="Generated by Gradle"/>
@@ -3870,12 +3864,6 @@
            <sha256 value="4395647b1961d9fb730a34e8dbe56c293157bc0759004cca63d9b5ee6653e5c7" origin="Generated by Gradle"/>
         </artifact>
      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.5.31">
-         <artifact name="kotlin-stdlib-common-1.5.31.jar">
-            <md5 value="65557bf9158d26bb450352376584b0b8" origin="Generated by Gradle"/>
-            <sha256 value="dfa2a18e26b028388ee1968d199bf6f166f737ab7049c25a5e2da614404e22ad" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.6.0">
         <artifact name="kotlin-stdlib-common-1.6.0.jar">
            <md5 value="05517b4cdeff2fc7c359a6148ac5dc38" origin="Generated by Gradle"/>
@@ -3900,12 +3888,6 @@
            <sha256 value="fa20188abaa8ecf1d0035e93a969b071f10e45a1c8378c314521eade73f75fd5" origin="Generated by Gradle"/>
         </artifact>
      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.5.31">
-         <artifact name="kotlin-stdlib-jdk7-1.5.31.jar">
-            <md5 value="ce797b26bf08539083989677cdcdb957" origin="Generated by Gradle"/>
-            <sha256 value="a25bf47353ce899d843cbddee516d621a73473e7fba97f8d0301e7b4aed7c15f" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.7.10">
         <artifact name="kotlin-stdlib-jdk7-1.7.10.jar">
            <md5 value="16e9288ff1d39c8f983507e3cd72e8a8" origin="Generated by Gradle"/>
@@ -3930,12 +3912,6 @@
            <sha256 value="af1ec40c3b951afdcc0c2a0173c7b81763c5281c2d5bafbf0a8544a24c5dcc0c" origin="Generated by Gradle"/>
         </artifact>
      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.5.31">
-         <artifact name="kotlin-stdlib-jdk8-1.5.31.jar">
-            <md5 value="294e940f7b51c2c26611dc1f237eecc3" origin="Generated by Gradle"/>
-            <sha256 value="b548f7767aacf029d2417e47440742bd6d3ebede19b60386e23554ce5c4c5fdc" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.7.10">
         <artifact name="kotlin-stdlib-jdk8-1.7.10.jar">
            <md5 value="83947a5800c9fae4d5e0aa15ed5f186f" origin="Generated by Gradle"/>
@@ -78,13 +78,3 @@ tasks {
        environment["LIBCLANG_DISABLE_CRASH_RECOVERY"] = "1"
    }
 }
-
-sourceSets {
-    "main" {
-        kotlin {
-            srcDir(project.kotlinNativeVersionSrc())
-            srcDir("../../shared/src/library/kotlin")
-            srcDir("../../shared/src/main/kotlin")
-        }
-    }
-}
@@ -1,5 +1,5 @@
 cacheRedirectorEnabled=true
 bootstrap.kotlin.default.version=1.9.0-dev-877
-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40

 kotlin.native.enabled=true
@@ -41,7 +41,6 @@ def globalProperties = new java.util.Properties()
 ext.kotlin_root = project.file("../..").absolutePath
 project.logger.info("kotlin_root: $kotlin_root")
 globalProperties.load(new java.io.FileReader(project.file("$kotlin_root/gradle.properties")))
-ext.kotlinNativeVersionInResources = true

 def konanDataDir = project.hasProperty('konan.data.dir') ? project.property('konan.data.dir').toString() : null
 def platformManager = new PlatformManager(DistributionKt.buildDistribution(UtilsKt.getKotlinNativeDist(project).path, konanDataDir), false)
@@ -1 +1 @@
-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40
@@ -1 +1 @@
-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40
@@ -1 +1 @@
-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40
@@ -1 +1 @@
-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40
@@ -34,6 +34,19 @@ If you want to run only one test class, you need to append `--tests` flag with v
 ./gradlew :kotlin-gradle-plugin-integration-tests:kgpAllParallelTests --tests <class-name-with-package>
 ```

+#### How to Run Using Kotlin Native from Master
+
+Currently, Kotlin Native from master involves three configurations: `kgpMppTests`, `kgpNativeTests`, and `kgpOtherTests`. 
+Depending on your development environment, there are a few different ways you can run this.
+* **On Local Environment** In the case of Local Environment builds, you have two options, which you can add in your `local.properties` file:  
+  * `kotlin.native.enabled=true` - this property adds building Kotlin Native full bundle step before Integration Tests, then this bundle will be used in the Integration Tests.
+  * `kotlin.native.local.distribution.for.tests.enabled=false` - include this line if you need to disable running Integration Tests with Kotlin/Native from master, even when `kotlin.native.enabled` is set to true.
+* **On TeamCity** In the case of TeamCity builds, no extra setting is necessary. The mentioned configurations depend on the `full bundle`, which stores its artifacts in the `-DkonanDataDirForIntegrationTests` directory.
+Also, you can specify the konan directory for test by providing `-DkonanDataDirForIntegrationTests`, for example:
+```bash
+ ./gradlew :kotlin-gradle-plugin-integration-tests:kgpNativeTests -DkonanDataDirForIntegrationTests=/tmp/.konan
+```
+
 #### How to work with the tests

 Few rules you should follow while writing tests:
@@ -1,5 +1,6 @@
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
-import org.gradle.jvm.toolchain.internal.NoToolchainAvailableException
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.pill.PillExtension
 import java.nio.file.Paths

@@ -111,6 +112,11 @@ dependencies {
    testCompileOnly(commonDependency("org.jetbrains.intellij.deps:asm-all"))
 }

+val konanDataDir: String = System.getProperty("konanDataDirForIntegrationTests")
+    ?: project.rootDir
+        .resolve(".kotlin")
+        .resolve("konan-for-gradle-tests").absolutePath
+
 // Aapt2 from Android Gradle Plugin 3.2 and below does not handle long paths on Windows.
 val shortenTempRootName = project.providers.systemProperty("os.name").get().contains("Windows")

@@ -130,13 +136,52 @@ val cleanTestKitCacheTask = tasks.register<Delete>("cleanTestKitCache") {
 }

 tasks.register<Delete>("cleanUserHomeKonanDir") {
-    group = KGP_TEST_TASKS_GROUP
-    description = "Deletes ~/.konan dir before tests. This step is necessary to ensure that no test inadvertently creates this directory during execution."
+    description =
+        "Deletes ~/.konan dir before tests. This step is necessary to ensure that no test inadvertently creates this directory during execution."

    val userHomeKonanDir = Paths.get("${System.getProperty("user.home")}/.konan")
+
    delete(userHomeKonanDir)

-    println("Default .konan directory user's home has been deleted: $userHomeKonanDir")
+    doLast {
+        logger.info("Default .konan directory user's home has been deleted: $userHomeKonanDir")
+    }
+}
+
+tasks.register<Copy>("prepareNativeBundleForGradleIT") {
+
+    description = "This task adds depenecy on :kotlin-native:bundle and then copying built bundle into the tests' konan dir"
+
+    // 1. Build full Kotlin Native bundle
+    dependsOn(":kotlin-native:bundle")
+
+    // 2. Coping and extracting k/n artifacts from the 1st step to tests' konan data directory
+    val (extension, unzipFunction) = when (HostManager.host) {
+        KonanTarget.MINGW_X64 -> Pair("zip", ::zipTree)
+        else -> Pair("tar.gz", ::tarTree)
+    }
+
+    val kotlinNativeRootDir = rootProject.findProject(":kotlin-native")?.projectDir
+        ?: throw IllegalStateException("The path to kotlin-native module is undefined.")
+
+    from(
+        unzipFunction(
+            kotlinNativeRootDir.resolve("kotlin-native-${HostManager.platformName()}-${project.kotlinBuildProperties.defaultSnapshotVersion}.$extension")
+        )
+    )
+    from(
+        unzipFunction(
+            kotlinNativeRootDir.resolve("kotlin-native-prebuilt-${HostManager.platformName()}-${project.kotlinBuildProperties.defaultSnapshotVersion}.$extension")
+        )
+    )
+
+    into(
+        konanDataDir
+    )
+
+    doFirst {
+        delete(konanDataDir)
+    }
 }

 fun Test.includeMppAndAndroid(include: Boolean) = includeTestsWithPattern(include) {
@@ -147,6 +192,28 @@ fun Test.includeNative(include: Boolean) = includeTestsWithPattern(include) {
    addAll(listOf("org.jetbrains.kotlin.gradle.native.*", "*Commonizer*"))
 }

+fun Test.applyKotlinNativeFromCurrentBranchIfNeeded() {
+    val kotlinNativeFromMasterEnabled = project.kotlinBuildProperties.isKotlinNativeEnabled && project.kotlinBuildProperties.useKotlinNativeLocalDistributionForTests
+    if (kotlinNativeFromMasterEnabled && !project.kotlinBuildProperties.isTeamcityBuild) {
+        dependsOn(":kotlin-gradle-plugin-integration-tests:prepareNativeBundleForGradleIT")
+    }
+
+    // Providing necessary properties for running tests with k/n built from master on the local environment
+    val defaultSnapshotVersion = project.kotlinBuildProperties.defaultSnapshotVersion
+    if (kotlinNativeFromMasterEnabled && defaultSnapshotVersion != null) {
+        systemProperty("kotlinNativeVersion", defaultSnapshotVersion)
+        systemProperty("konanDataDirForIntegrationTests", konanDataDir)
+    }
+
+    // Providing necessary properties for running tests with k/n built from master on the TeamCity
+    if (project.kotlinBuildProperties.isTeamcityBuild) {
+        System.getProperty("kotlinNativeVersionForGradleIT")?.let {
+            systemProperty("kotlinNativeVersion", it)
+        }
+        systemProperty("konanDataDirForIntegrationTests", konanDataDir)
+    }
+}
+
 fun Test.includeTestsWithPattern(include: Boolean, patterns: (MutableSet<String>).() -> Unit) {
    if (splitGradleIntegrationTestTasks) {
        val filter = if (include)
@@ -250,6 +317,7 @@ val nativeTestsTask = tasks.register<Test>("kgpNativeTests") {
        excludeTags("JvmKGP", "JsKGP", "DaemonsKGP", "OtherKGP", "MppKGP", "AndroidKGP")
        includeEngines("junit-jupiter")
    }
+    applyKotlinNativeFromCurrentBranchIfNeeded()
 }

 // Daemon tests could run only sequentially as they could not be shared between parallel test builds
@@ -274,6 +342,7 @@ val otherPluginsTestTask = tasks.register<Test>("kgpOtherTests") {
        excludeTags("JvmKGP", "JsKGP", "NativeKGP", "DaemonsKGP", "MppKGP", "AndroidKGP")
        includeEngines("junit-jupiter")
    }
+    applyKotlinNativeFromCurrentBranchIfNeeded()
 }

 val mppTestsTask = tasks.register<Test>("kgpMppTests") {
@@ -285,6 +354,7 @@ val mppTestsTask = tasks.register<Test>("kgpMppTests") {
        excludeTags("JvmKGP", "JsKGP", "NativeKGP", "DaemonsKGP", "OtherKGP", "AndroidKGP")
        includeEngines("junit-jupiter")
    }
+    applyKotlinNativeFromCurrentBranchIfNeeded()
 }

 val androidTestsTask = tasks.register<Test>("kgpAndroidTests") {
@@ -66,7 +66,7 @@ open class CommonizerIT : KGPBaseTest() {
    @GradleTest
    fun testCleanNativeDistributionCommonization(gradleVersion: GradleVersion, @TempDir konanData: Path) {
        nativeProject("commonizeNativeDistributionWithIosLinuxWindows", gradleVersion) {
-            val buildOptions = defaultBuildOptions.copy(
+            val buildOptions = defaultBuildOptions.withBundledKotlinNative().copy(
                konanDataDir = konanData
            )

@@ -834,7 +834,11 @@ class GeneralNativeIT : KGPBaseTest() {
                }

                // Check that changing K/N version lead to tasks rerun
-                build(compileTasks, "-Porg.jetbrains.kotlin.native.version=${TestVersions.Kotlin.STABLE_RELEASE}") {
+                build(
+                    compileTasks, buildOptions = defaultBuildOptions.copy(
+                        nativeOptions = defaultBuildOptions.nativeOptions.copy(version = TestVersions.Kotlin.STABLE_RELEASE)
+                    )
+                ) {
                    assertTasksExecuted(compileTasks)
                }
            }
@@ -15,15 +15,12 @@ import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.presetName
 import org.junit.jupiter.api.Assumptions
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.DisplayName
 import org.junit.jupiter.api.condition.OS
 import org.junit.jupiter.api.io.TempDir
 import java.nio.file.Path
-import java.nio.file.Paths
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.appendText
-import kotlin.io.path.deleteRecursively

 // We temporarily disable it for windows until a proper fix is found for this issue:
 // https://youtrack.jetbrains.com/issue/KT-60138/NativeDownloadAndPlatformLibsIT-fails-on-Windows-OS
@@ -43,12 +40,12 @@ class NativeDownloadAndPlatformLibsIT : KGPBaseTest() {
    private val currentCompilerVersion = NativeCompilerDownloader.DEFAULT_KONAN_VERSION

    override val defaultBuildOptions: BuildOptions
-        get() = super.defaultBuildOptions.copy(
+        get() = super.defaultBuildOptions.withBundledKotlinNative().copy(
            // For each test in this class, we need to provide an isolated .konan directory,
            // so we create it within each test project folder
            konanDataDir = workingDir.resolve(".konan")
                .toFile()
-                .apply { mkdirs() }.toPath()
+                .apply { mkdirs() }.toPath(),
        )

    @OptIn(EnvironmentalVariablesOverride::class)
@@ -26,6 +26,9 @@ class NativeDownloadAndPlatformLibsNonParallelIT : KGPDaemonsBaseTest() {
    private val platformName: String = HostManager.platformName()
    private val currentCompilerVersion = NativeCompilerDownloader.DEFAULT_KONAN_VERSION

+    override val defaultBuildOptions: BuildOptions
+        get() = super.defaultBuildOptions.withBundledKotlinNative().copy()
+

    @DisplayName("Downloading K/N distribution in default .konan dir")
    @GradleTest
@@ -22,19 +22,20 @@ class NativeWithConfigurationCacheIT : KGPBaseTest() {
    @GradleTestVersions(minVersion = TestVersions.Gradle.G_8_1) // Since 8.1 Gradle on configuration cache it detects when the build logic accesses the "outside world" more strict https://docs.gradle.org/8.1.1/release-notes.html#configuration-inputs-detection-improvements
    @GradleTest
    fun testConfigurationCacheReusedSecondTime(gradleVersion: GradleVersion) {
-        nativeProject("native-with-configuration-cache", gradleVersion) {
-            // we need to download compiler on the first build, that is why we are setting custom konan home dir without any compiler inside
-            val localKonan = workingDir.resolve(".konan")
-            build(
-                "help", buildOptions = defaultBuildOptions.copy(
-                    konanDataDir = localKonan
+        nativeProject(
+            "native-with-configuration-cache", gradleVersion, buildOptions = defaultBuildOptions.withBundledKotlinNative().copy(
+                // We need to download compiler on the first build, that is why we are setting custom konan home dir without any compiler inside
+                konanDataDir = workingDir.resolve(".konan"),
            )
+        ) {
+            build(
+                "help"
            ) {
                assertOutputContains("Configure project")
                assertOutputContains("Unpack Kotlin/Native compiler to")
            }

-            build("help", "-Pkonan.data.dir=$localKonan") {
+            build("help") {
                assertOutputContains("Reusing configuration cache.")
            }
        }
@@ -18,7 +18,6 @@ import org.jetbrains.kotlin.gradle.report.BuildReportType
 import org.junit.jupiter.api.condition.OS
 import java.nio.file.Path
 import java.util.*
-import kotlin.io.path.absolutePathString

 data class BuildOptions(
    val logLevel: LogLevel = LogLevel.INFO,
@@ -91,7 +90,7 @@ data class BuildOptions(
        val reinstall: Boolean? = null,
        val restrictedDistribution: Boolean? = null,
        val useXcodeMessageStyle: Boolean? = null,
-        val version: String? = null,
+        val version: String? = System.getProperty("kotlinNativeVersion"),
        val cacheOrchestration: String? = null,
        val incremental: Boolean? = null,
    )
@@ -287,3 +286,17 @@ fun BuildOptions.suppressDeprecationWarningsSinceGradleVersion(
 ) = suppressDeprecationWarningsOn(reason) {
    currentGradleVersion >= GradleVersion.version(gradleVersion)
 }
+
+/**
+ * This wrapper erases k/n version from passing parameters,
+ * because we should use Kotlin Native bundled in KGP instead of which built from current branch.
+ *
+ * In this case we will use k/n version, which declared in KGP.
+ *
+ * The most common case is when we override local konan dir for some reason.
+ */
+fun BuildOptions.withBundledKotlinNative() = copy(
+    nativeOptions = nativeOptions.copy(
+        version = null
+    )
+)
@@ -452,7 +452,17 @@ private fun TestProject.withBuildSummary(
    }
 }

-val konanDir get() = Paths.get(".").resolve("../../../build").resolve("konan-for-gradle-tests")
+/**
+ * This property is configured reade konan from specific directory, which in teamcity will be filled with k/n built from master.
+ * NOTE: On changing test konan dir location update related location in kotlin-teamcity-build repository
+ */
+val konanDir
+    get() =
+        System.getProperty("konanDataDirForIntegrationTests")?.let {
+            Paths.get(it)
+        } ?: Paths.get(".")
+            .resolve("../../../.kotlin")
+            .resolve("konan-for-gradle-tests")

 /**
 * On changing test kit dir location update related location in 'cleanTestKitCache' task.
@@ -13,7 +13,6 @@ enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: Bo

    // whether the build is executed from IDE or from console
    EXECUTED_FROM_IDEA(OVERRIDE, SAFE),
-
    // Build script

    //annotation processors
@@ -3,7 +3,7 @@ org.gradle.kotlin.dsl.allWarningsAsErrors=true

 cacheRedirectorEnabled=true

-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40
 kotlin.options.suppressFreeCompilerArgsModificationWarning=true

 # Please keep it in sync with root gradle.properties.
@@ -1,4 +1,4 @@
-kotlin.build.gradlePlugin.version=0.0.39
+kotlin.build.gradlePlugin.version=0.0.40
 kotlin.options.suppressFreeCompilerArgsModificationWarning=true
 cacheRedirectorEnabled=true
 # Should be in sync with ":kotlin-daemon-config" project convention script configuration