[Gradle Native] Added Kotlin Native Provider to all native tasks

This provider is a first step of "Kotlin Native Toolchain".
This solves several problems:
1) Resolving K/N bundle during configuration phase ->
now it happens during execution phase
2) Downloading K/N bundle with internal mechanisms ->
now it uses Gradle dependency resolvers and Transform Artifact

^KT-58303
^KT-52567
^KT-49268

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheIT.kt

@@ -20,7 +20,12 @@ import org.gradle.api.logging.LogLevel
 import org.gradle.util.GradleVersion
 import org.jetbrains.kotlin.gradle.report.BuildReportType
 import org.jetbrains.kotlin.gradle.testbase.*
+import org.jetbrains.kotlin.gradle.util.capitalize
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.presetName
 import org.junit.jupiter.api.DisplayName
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Path
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.readText
 import kotlin.io.path.relativeTo
@@ -144,6 +149,33 @@ class BuildCacheIT : KGPBaseTest() {
         }
     }
 
+    @DisplayName("Changing native toolchain location should not break build cache")
+    @GradleTest
+    fun testNativeToolchainWithBuildCache(gradleVersion: GradleVersion, @TempDir customNativeHomePath: Path) {
+        nativeProject("native-simple-project", gradleVersion) {
+            enableLocalBuildCache(localBuildCacheDir)
+
+            val buildOptionsBeforeCaching = defaultBuildOptions.copy(
+                nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
+                    version = TestVersions.Kotlin.STABLE_RELEASE,
+                    distributionDownloadFromMaven = true
+                )
+            )
+            val nativeCompileTask = ":compileKotlin${HostManager.host.presetName.capitalize()}"
+            build(nativeCompileTask, buildOptions = buildOptionsBeforeCaching) {
+                assertTasksPackedToCache(nativeCompileTask)
+            }
+
+            val buildOptionsAfterCaching = buildOptionsBeforeCaching.copy(
+                konanDataDir = customNativeHomePath,
+            )
+
+            build("clean", nativeCompileTask, buildOptions = buildOptionsAfterCaching) {
+                assertTasksFromCache(nativeCompileTask)
+            }
+        }
+    }
+
     //doesn't work for build history files approach
     @DisplayName("Restore from build cache should not break incremental compilation")
     @GradleTest

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt

@@ -20,8 +20,13 @@ import org.gradle.api.logging.LogLevel
 import org.gradle.testkit.runner.BuildResult
 import org.gradle.util.GradleVersion
 import org.jetbrains.kotlin.gradle.testbase.*
+import org.jetbrains.kotlin.gradle.util.capitalize
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.presetName
 import org.junit.jupiter.api.DisplayName
+import org.junit.jupiter.api.io.TempDir
 import java.io.File
+import java.nio.file.Path
 import kotlin.io.path.*
 
 @DisplayName("Build cache relocation")
@@ -204,9 +209,16 @@ class BuildCacheRelocationIT : KGPBaseTest() {
     @DisplayName("with native project")
     @GradleTest
     fun testRelocationNative(gradleVersion: GradleVersion) {
+        val buildOptionsBeforeCaching = defaultBuildOptions.copy(
+            nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
+                version = TestVersions.Kotlin.STABLE_RELEASE,
+                distributionDownloadFromMaven = true
+            )
+        )
         val (firstProject, secondProject) = prepareTestProjects(
             "native-build-cache",
-            gradleVersion
+            gradleVersion,
+            buildOptions = buildOptionsBeforeCaching
         ) {
             val localRepoUri = it.projectPath.resolve("repo").toUri()
             it.subProject("build-cache-app").buildGradleKts.append(
@@ -276,12 +288,12 @@ class BuildCacheRelocationIT : KGPBaseTest() {
         buildJdk: File? = null,
         additionalConfiguration: (TestProject) -> Unit = {}
     ): Pair<TestProject, TestProject> {
-        val firstProject = project(projectName, gradleVersion, buildOptions, buildJdk = buildJdk) {
+        val firstProject = project(projectName, gradleVersion, buildOptions, buildJdk = buildJdk, forceOutput = true) {
             enableLocalBuildCache(localBuildCacheDir)
             additionalConfiguration(this)
         }
 
-        val secondProject = project(projectName, gradleVersion, buildOptions, buildJdk = buildJdk) {
+        val secondProject = project(projectName, gradleVersion, buildOptions, buildJdk = buildJdk, forceOutput = true) {
             enableLocalBuildCache(localBuildCacheDir)
             additionalConfiguration(this)
         }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/KotlinNativeCompilerDownloadIT.kt

@@ -0,0 +1,144 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.native
+
+import org.gradle.util.GradleVersion
+import org.jetbrains.kotlin.gradle.testbase.*
+import org.jetbrains.kotlin.gradle.testbase.TestVersions.Kotlin.STABLE_RELEASE
+import org.jetbrains.kotlin.gradle.util.capitalize
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.presetName
+import org.junit.jupiter.api.DisplayName
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Path
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.appendText
+import kotlin.test.assertContains
+
+
+@DisplayName("This test class contains different scenarios with downloading Kotlin Native Compiler during build.")
+@NativeGradlePluginTests
+class KotlinNativeCompilerDownloadIT : KGPBaseTest() {
+
+    private val currentPlatform = HostManager.platformName()
+
+    private val nativeHostTargetName = HostManager.host.presetName
+
+    private val UNPUCK_KONAN_FINISHED_LOG =
+        "Moving Kotlin/Native bundle from tmp directory"
+
+    private val STABLE_VERSION_DIR_NAME = "kotlin-native-prebuilt-$currentPlatform-$STABLE_RELEASE"
+
+    private val DOWNLOAD_KONAN_FINISHED_LOG =
+        "Download $STABLE_VERSION_DIR_NAME.${if (HostManager.hostIsMingw) "zip" else "tar.gz"} finished,"
+
+    override val defaultBuildOptions: BuildOptions
+        get() = super.defaultBuildOptions.copy(
+            nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
+                version = STABLE_RELEASE,
+                distributionDownloadFromMaven = true
+            )
+        )
+
+    @DisplayName("KT-58303: Kotlin Native must not be downloaded during configuration phase")
+    @GradleTest
+    fun shouldNotDownloadKotlinNativeOnConfigurationPhase(gradleVersion: GradleVersion, @TempDir konanTemp: Path) {
+        nativeProject(
+            "native-simple-project",
+            gradleVersion,
+            buildOptions = defaultBuildOptions.copy(
+                konanDataDir = konanTemp,
+            ),
+        ) {
+            build("help") {
+                assertOutputDoesNotContain(DOWNLOAD_KONAN_FINISHED_LOG)
+                assertOutputDoesNotContain(UNPUCK_KONAN_FINISHED_LOG)
+                assertOutputDoesNotContain("Please wait while Kotlin/Native")
+            }
+        }
+    }
+
+    @DisplayName("KT-58303: Kotlin Native must be downloaded during execution phase")
+    @GradleTest
+    fun shouldDownloadKotlinNativeOnExecutionPhase(gradleVersion: GradleVersion, @TempDir konanTemp: Path) {
+        nativeProject(
+            "native-simple-project",
+            gradleVersion,
+            buildOptions = defaultBuildOptions.copy(
+                konanDataDir = konanTemp,
+            ),
+        ) {
+            build("assemble") {
+                assertOutputDoesNotContain(DOWNLOAD_KONAN_FINISHED_LOG)
+                assertOutputContains(UNPUCK_KONAN_FINISHED_LOG)
+                assertOutputDoesNotContain("Please wait while Kotlin/Native")
+            }
+        }
+    }
+
+    @DisplayName("KT-58303: Downloading Kotlin Native on configuration phase(deprecated version)")
+    @GradleTest
+    fun shouldDownloadKotlinNativeOnConfigurationPhaseWithToolchainDisabled(gradleVersion: GradleVersion, @TempDir konanTemp: Path) {
+        nativeProject(
+            "native-simple-project",
+            gradleVersion,
+            buildOptions = defaultBuildOptions.copy(
+                konanDataDir = konanTemp,
+                freeArgs = listOf("-Pkotlin.native.toolchain.enabled=false"),
+            ),
+        ) {
+            build("assemble") {
+                assertOutputContains(DOWNLOAD_KONAN_FINISHED_LOG)
+                assertOutputContains("Please wait while Kotlin/Native")
+            }
+        }
+    }
+
+    @DisplayName("Checking multi-module native project with different path to kotlin native compiler in each module")
+    @GradleTest
+    fun multiModuleProjectWithDifferentKotlinNativeCompilers(gradleVersion: GradleVersion, @TempDir customNativeHomePath: Path) {
+        nativeProject("native-multi-module-project", gradleVersion, configureSubProjects = true) {
+            val defaultKotlinNativeHomePath =
+                defaultBuildOptions.konanDataDir?.toAbsolutePath()?.normalize()
+                    ?: error("Default konan data dir must be set in this test before overriding")
+
+            // setting different konan home in different subprojects
+            subProject("native1").gradleProperties.appendKonanToGradleProperties(customNativeHomePath.absolutePathString())
+            subProject("native2").gradleProperties.appendKonanToGradleProperties(defaultKotlinNativeHomePath.absolutePathString())
+
+            val buildOptions = defaultBuildOptions.copy(
+                konanDataDir = null,
+            )
+
+            build("assemble", buildOptions = buildOptions) {
+
+                // check that in first project we use k/n from custom konan location
+                assertNativeTasksClasspath(":native1:compileKotlin${nativeHostTargetName.capitalize()}") {
+                    val konanLibsPath = customNativeHomePath.resolve(STABLE_VERSION_DIR_NAME).resolve("konan").resolve("lib")
+                    assertContains(it, konanLibsPath.resolve("kotlin-native-compiler-embeddable.jar").absolutePathString())
+                    assertContains(it, konanLibsPath.resolve("trove4j.jar").absolutePathString())
+                }
+
+                // check that in second project we use k/n from default konan location
+                assertNativeTasksClasspath(":native2:compileKotlin${nativeHostTargetName.capitalize()}") {
+                    val konanLibsPath = defaultKotlinNativeHomePath.resolve(STABLE_VERSION_DIR_NAME).resolve("konan").resolve("lib")
+                    assertContains(it, konanLibsPath.resolve("kotlin-native-compiler-embeddable.jar").absolutePathString())
+                    assertContains(it, konanLibsPath.resolve("trove4j.jar").absolutePathString())
+                }
+            }
+        }
+    }
+
+    private fun Path.appendKonanToGradleProperties(konanAbsolutePathString: String) {
+        this.appendText(
+            """
+                    
+            kotlin.native.home=${konanAbsolutePathString}
+            konan.data.dir=${konanAbsolutePathString}
+            """.trimIndent()
+        )
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/NativeDownloadAndPlatformLibsIT.kt

@@ -27,6 +27,15 @@ import kotlin.io.path.appendText
 )
 @DisplayName("Tests for K/N builds with native downloading and platform libs")
 @NativeGradlePluginTests
+@Deprecated(
+    message =
+    """
+    This is deprecated test class with regression checks for old downloading logic.
+    We support it during migration to kotlin native toolchain.
+    If you want to add test here, be sure that you have added similar test with `-Pkotlin.native.toolchain.enabled=true`.
+    """,
+    ReplaceWith("NativeDownloadAndPlatformLibsIT")
+)
 class NativeDownloadAndPlatformLibsIT : KGPBaseTest() {
 
     private val platformName: String = HostManager.platformName()
@@ -34,6 +43,8 @@ class NativeDownloadAndPlatformLibsIT : KGPBaseTest() {
 
     override val defaultBuildOptions: BuildOptions
         get() = super.defaultBuildOptions.withBundledKotlinNative().copy(
+            // Disabling toolchain feature for checking stable logic with downloading kotlin native
+            freeArgs = listOf("-Pkotlin.native.toolchain.enabled=false"),
             // For each test in this class, we need to provide an isolated .konan directory,
             // so we create it within each test project folder
             konanDataDir = workingDir.resolve(".konan")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/TestVersions.kt

@@ -36,7 +36,7 @@ interface TestVersions {
     }
 
     object Kotlin {
-        const val STABLE_RELEASE = "1.9.20"
+        const val STABLE_RELEASE = "1.9.21"
 
         // Copied from KOTLIN_VERSION.kt file
         val CURRENT

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/outputHelpers.kt

@@ -62,17 +62,20 @@ fun getOutputForTask(taskPath: String, output: String, logLevel: LogLevel = LogL
             LogLevel.DEBUG -> taskOutputRegexForDebugLog(taskPath)
             else -> throw throw IllegalStateException("Unsupported log lever for task output was given: $logLevel")
         })
-    .find(output)
-    ?.let { it.groupValues[1] }
-    ?: error(
-        """
-        Could not find output for task $taskPath.
-        =================
-        Build output is:
-        $output 
-        =================     
-        """.trimIndent()
-    )
+    .findAll(output)
+    .map { it.groupValues[1] }
+    .joinToString(System.lineSeparator())
+    .ifEmpty {
+        error(
+            """
+            Could not find output for task $taskPath.
+            =================
+            Build output is:
+            $output 
+            =================     
+            """.trimIndent()
+        )
+    }
 
 class CommandLineArguments(
     val args: List<String>,

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-multi-module-project/build.gradle.kts

@@ -0,0 +1,8 @@
+plugins {
+    kotlin("multiplatform") apply false
+}
+
+repositories {
+    mavenCentral()
+    mavenLocal()
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-multi-module-project/native1/build.gradle.kts

@@ -0,0 +1,12 @@
+plugins {
+    kotlin("multiplatform")
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+kotlin {
+    <SingleNativeTarget>()
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-multi-module-project/native1/src/nativeMain/kotlin/Main.kt

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+fun main() {
+    println("Hello from first")
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-multi-module-project/native2/build.gradle.kts

@@ -0,0 +1,13 @@
+plugins {
+    kotlin("multiplatform")
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+
+kotlin {
+    <SingleNativeTarget>()
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-multi-module-project/native2/src/nativeMain/kotlin/Main.kt

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+fun main() {
+    println("Hello from second")
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-multi-module-project/settings.gradle.kts

@@ -0,0 +1,3 @@
+rootProject.name = "native-multi-module-project"
+include("native1")
+include("native2")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-simple-project/build.gradle.kts

@@ -0,0 +1,12 @@
+plugins {
+    kotlin("multiplatform")
+}
+
+repositories {
+    mavenCentral()
+    mavenLocal()
+}
+
+kotlin {
+    <SingleNativeTarget>()
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-simple-project/src/nativeMain/kotlin/main.kt

@@ -0,0 +1,3 @@
+fun main() {
+    println("Hello from simple native project")
+}