[Gradle] Moved downloading konan dependencies from compiler to Gradle

^KT-65823 Fixed

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/KotlinNativeDependenciesDownloadIT.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.native
+
+import org.gradle.util.GradleVersion
+import org.jetbrains.kotlin.gradle.testbase.*
+import org.junit.jupiter.api.DisplayName
+import org.junit.jupiter.api.condition.OS
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Path
+
+// We temporarily disable it for windows until a proper fix is found for this issue: KT-62761
+@OsCondition(
+    supportedOn = [OS.MAC, OS.LINUX], enabledOnCI = [OS.MAC, OS.LINUX]
+)
+@DisplayName("This test class contains different scenarios with downloading dependencies for Kotlin Native Compiler during build.")
+@NativeGradlePluginTests
+class KotlinNativeDependenciesDownloadIT : KGPBaseTest() {
+
+    @TempDir
+    lateinit var konanDataTempDir: Path
+
+    override val defaultBuildOptions: BuildOptions
+        get() = super.defaultBuildOptions.withBundledKotlinNative().copy(
+            // For each test in this class, we need to provide an isolated .konan directory,
+            // so we create it within each test project folder
+            konanDataDir = konanDataTempDir
+        )
+
+    @DisplayName("Gradle should download dependencies before compile execution")
+    @GradleTest
+    fun shouldDownloadDependenciesBeforeCompilerExecution(gradleVersion: GradleVersion) {
+        nativeProject("mpp-default-hierarchy", gradleVersion) {
+            build("assemble") {
+                assertOutputContains("Downloading dependency for Kotlin Native")
+                assertOutputDoesNotContain("(KonanProperties) Downloading dependency")
+            }
+        }
+    }
+
+    @DisplayName("Compiler should download dependencies when Kotlin Native Toolchain disabled ")
+    @GradleTest
+    fun checkCompilerDownloadsDependenciesWhenToochainDisabled(gradleVersion: GradleVersion) {
+        nativeProject("mpp-default-hierarchy", gradleVersion) {
+
+            build(
+                "assemble", buildOptions = defaultBuildOptions.copy(
+                    freeArgs = listOf("-Pkotlin.native.toolchain.enabled=false"),
+                )
+            ) {
+                assertOutputContains("(KonanProperties) Downloading dependency")
+                assertOutputDoesNotContain("Downloading dependency for Kotlin Native")
+            }
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt

@@ -22,9 +22,6 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
 import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
 import org.jetbrains.kotlin.commonizer.konanTargets
 import org.jetbrains.kotlin.compilerRunner.*
-import org.jetbrains.kotlin.compilerRunner.GradleCliCommonizer
-import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner
-import org.jetbrains.kotlin.compilerRunner.konanHome
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
 import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
@@ -104,7 +101,12 @@ internal abstract class NativeDistributionCommonizerTask
 
     @get:Nested
     internal val kotlinNativeProvider: Provider<KotlinNativeProvider> = project.provider {
-        KotlinNativeProvider(project, commonizerTargets.flatMap { target -> target.konanTargets }.toSet(), kotlinNativeBundleBuildService)
+        KotlinNativeProvider(
+            project,
+            commonizerTargets.flatMap { target -> target.konanTargets }.toSet(),
+            kotlinNativeBundleBuildService,
+            enableDependenciesDownloading = false
+        )
     }
 
     @TaskAction

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt

@@ -17,10 +17,14 @@ import org.jetbrains.kotlin.compilerRunner.konanHome
 import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled
 import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
 import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
 import org.jetbrains.kotlin.gradle.utils.filesProvider
 import org.jetbrains.kotlin.gradle.utils.property
+import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
+import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.konan.target.loadConfigurables
 
 /**
  * This is a nested provider for all native tasks
@@ -29,6 +33,7 @@ internal class KotlinNativeProvider(
     project: Project,
     konanTargets: Set<KonanTarget>,
     kotlinNativeBundleBuildService: Provider<KotlinNativeBundleBuildService>,
+    enableDependenciesDownloading: Boolean = true,
 ) {
     constructor(
         project: Project,
@@ -65,6 +70,32 @@ internal class KotlinNativeProvider(
         kotlinNativeVersion
     }
 
+    @get:Input
+    val kotlinNativeDependencies: Provider<Set<String>> =
+        kotlinNativeBundleVersion
+            .zip(bundleDirectory) { _, bundleDir ->
+                val requiredDependencies = mutableSetOf<String>()
+                if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) {
+                    val distribution = Distribution(bundleDir.asFile.absolutePath, konanDataDir = konanDataDir.orNull)
+                    konanTargets.forEach { konanTarget ->
+                        if (konanTarget.enabledOnCurrentHost) {
+                            val konanPropertiesLoader = loadConfigurables(
+                                konanTarget,
+                                distribution.properties,
+                                distribution.dependenciesDir,
+                                progressCallback = { url, currentBytes, totalBytes ->
+                                    project.logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ")
+                                }
+                            ) as KonanPropertiesLoader
+
+                            requiredDependencies.addAll(konanPropertiesLoader.dependencies)
+                            konanPropertiesLoader.downloadDependencies()
+                        }
+                    }
+                }
+                requiredDependencies
+            }
+
     // Gradle tries to evaluate this val during configuration cache,
     // which lead to resolving configuration, even if k/n bundle is in konan home directory.
     @Transient

native/utils/src/org/jetbrains/kotlin/konan/target/Apple.kt

@@ -19,12 +19,14 @@ package org.jetbrains.kotlin.konan.target
 import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.util.InternalServer
+import org.jetbrains.kotlin.konan.util.ProgressCallback
 
 class AppleConfigurablesImpl(
     target: KonanTarget,
     properties: Properties,
-    dependenciesDir: String?
+    dependenciesDir: String?,
-) : AppleConfigurables, KonanPropertiesLoader(target, properties, dependenciesDir) {
+    progressCallback: ProgressCallback,
+) : AppleConfigurables, KonanPropertiesLoader(target, properties, dependenciesDir, progressCallback = progressCallback) {
 
     private val sdkDependency = this.targetSysRoot!!
     private val toolchainDependency = this.targetToolchain!!
@@ -45,10 +47,13 @@ class AppleConfigurablesImpl(
         XcodePartsProvider.InternalServer -> absolute(additionalToolsDir)
     }
 
-    override val dependencies get() = super.dependencies + when (xcodePartsProvider) {
+    override val dependencies
-        is XcodePartsProvider.Local -> emptyList()
+        get() = super.dependencies +
-        XcodePartsProvider.InternalServer -> listOf(sdkDependency, toolchainDependency, xcodeAddonDependency)
+                if (InternalServer.isAvailable) listOf(
-    }
+                    sdkDependency,
+                    toolchainDependency,
+                    xcodeAddonDependency
+                ) else emptyList()
 
     private val xcodePartsProvider by lazy {
         if (InternalServer.isAvailable) {

native/utils/src/org/jetbrains/kotlin/konan/target/ConfigurablesImpl.kt

@@ -17,34 +17,42 @@
 package org.jetbrains.kotlin.konan.target
 
 import org.jetbrains.kotlin.konan.properties.*
+import org.jetbrains.kotlin.konan.util.ProgressCallback
 
-class GccConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?)
+class GccConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : GccConfigurables,
-    : GccConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot), ConfigurablesWithEmulator {
+    KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback), ConfigurablesWithEmulator {
     override val dependencies: List<String>
         get() = super.dependencies + listOfNotNull(emulatorDependency)
-    }
+}
-
+
-class AndroidConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?)
+class AndroidConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : AndroidConfigurables,
-    : AndroidConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot)
+    KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback)
-
+
-class WasmConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?)
+class WasmConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : WasmConfigurables,
-    : WasmConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot)
+    KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback)
-
+
-class ZephyrConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?)
+class ZephyrConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : ZephyrConfigurables,
-    : ZephyrConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot)
+    KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback)
-
+
-
+
-fun loadConfigurables(target: KonanTarget, properties: Properties, dependenciesRoot: String?): Configurables = when (target.family) {
+fun loadConfigurables(
-    Family.LINUX -> GccConfigurablesImpl(target, properties, dependenciesRoot)
+    target: KonanTarget,
-
+    properties: Properties,
-    Family.TVOS, Family.WATCHOS, Family.IOS, Family.OSX -> AppleConfigurablesImpl(target, properties, dependenciesRoot)
+    dependenciesRoot: String?,
-
+    progressCallback: ProgressCallback = { url, currentBytes, totalBytes ->
-    Family.ANDROID -> AndroidConfigurablesImpl(target, properties, dependenciesRoot)
+        print("\n(KonanProperties) Downloading dependency: $url (${currentBytes}/${totalBytes}). ")
-
+    },
-    Family.MINGW -> MingwConfigurablesImpl(target, properties, dependenciesRoot)
+): Configurables = when (target.family) {
-
+    Family.LINUX -> GccConfigurablesImpl(target, properties, dependenciesRoot, progressCallback)
-    Family.WASM -> WasmConfigurablesImpl(target, properties, dependenciesRoot)
+
-
+    Family.TVOS, Family.WATCHOS, Family.IOS, Family.OSX -> AppleConfigurablesImpl(target, properties, dependenciesRoot, progressCallback)
-    Family.ZEPHYR -> ZephyrConfigurablesImpl(target, properties, dependenciesRoot)
+
+    Family.ANDROID -> AndroidConfigurablesImpl(target, properties, dependenciesRoot, progressCallback)
+
+    Family.MINGW -> MingwConfigurablesImpl(target, properties, dependenciesRoot, progressCallback)
+
+    Family.WASM -> WasmConfigurablesImpl(target, properties, dependenciesRoot, progressCallback)
+
+    Family.ZEPHYR -> ZephyrConfigurablesImpl(target, properties, dependenciesRoot, progressCallback)
 }
 

native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt

@@ -21,15 +21,16 @@ import org.jetbrains.kotlin.konan.target.Configurables
 import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.util.ArchiveType
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
+import org.jetbrains.kotlin.konan.util.ProgressCallback
 import java.io.File
 
 interface TargetableExternalStorage {
-    fun targetString(key: String): String? 
+    fun targetString(key: String): String?
     fun targetList(key: String): List<String>
-    fun hostString(key: String): String? 
+    fun hostString(key: String): String?
-    fun hostList(key: String): List<String> 
+    fun hostList(key: String): List<String>
-    fun hostTargetString(key: String): String? 
+    fun hostTargetString(key: String): String?
-    fun hostTargetList(key: String): List<String> 
+    fun hostTargetList(key: String): List<String>
     fun absolute(value: String?): String
     fun downloadDependencies()
 }
@@ -39,21 +40,24 @@ abstract class KonanPropertiesLoader(
     val properties: Properties,
     private val dependenciesRoot: String?,
     private val host: KonanTarget = HostManager.host,
+    private val progressCallback: ProgressCallback,
 ) : Configurables {
+
+
     private val predefinedLlvmDistributions: Set<String> =
-            properties.propertyList("predefinedLlvmDistributions").toSet()
+        properties.propertyList("predefinedLlvmDistributions").toSet()
 
     private val predefinedLibffiVersions: Set<String> =
-            properties.propertyList("predefinedLibffiVersions").toSet()
+        properties.propertyList("predefinedLibffiVersions").toSet()
 
     private fun getPredefinedDependencyOrNull(
-            dependencyName: String,
+        dependencyName: String,
-            dependencyAccessor: () -> String?,
+        dependencyAccessor: () -> String?,
-            predefinedDependencies: Set<String>
+        predefinedDependencies: Set<String>,
     ): String? {
         // Store into variable to avoid repeated resolve.
         val dependency = dependencyAccessor()
-                ?: error("Undefined $dependencyName!")
+            ?: error("Undefined $dependencyName!")
         return when (dependency) {
             in predefinedDependencies -> dependency
             else -> null
@@ -61,8 +65,8 @@ abstract class KonanPropertiesLoader(
     }
 
     private fun compilerDependencies(): List<String> = listOfNotNull(
-            getPredefinedDependencyOrNull("LLVM home", this::llvmHome, predefinedLlvmDistributions),
+        getPredefinedDependencyOrNull("LLVM home", this::llvmHome, predefinedLlvmDistributions),
-            getPredefinedDependencyOrNull("libffi version", this::libffiDir, predefinedLibffiVersions)
+        getPredefinedDependencyOrNull("libffi version", this::libffiDir, predefinedLibffiVersions)
     )
 
     open val dependencies: List<String>
@@ -73,38 +77,30 @@ abstract class KonanPropertiesLoader(
     }
 
     // TODO: We may want to add caching to avoid repeated resolve.
-    override fun targetString(key: String): String? 
+    override fun targetString(key: String): String? = properties.targetString(key, target)
-        = properties.targetString(key, target)
+    override fun targetList(key: String): List<String> = properties.targetList(key, target)
-    override fun targetList(key: String): List<String>
+    override fun hostString(key: String): String? = properties.hostString(key, host)
-        = properties.targetList(key, target)
+    override fun hostList(key: String): List<String> = properties.hostList(key, host)
-    override fun hostString(key: String): String? 
+    override fun hostTargetString(key: String): String? = properties.hostTargetString(key, target, host)
-        = properties.hostString(key, host)
+    override fun hostTargetList(key: String): List<String> = properties.hostTargetList(key, target, host)
-    override fun hostList(key: String): List<String> 
-        = properties.hostList(key, host)
-    override fun hostTargetString(key: String): String? 
-        = properties.hostTargetString(key, target, host)
-    override fun hostTargetList(key: String): List<String> 
-        = properties.hostTargetList(key, target, host)
 
     override fun absolute(value: String?): String =
-            dependencyProcessor!!.resolve(value!!).absolutePath
+        dependencyProcessor!!.resolve(value!!).absolutePath
-    private val dependencyProcessor  by lazy {
+
+    private val dependencyProcessor by lazy {
         dependenciesRoot?.let {
             DependencyProcessor(
                 dependenciesRoot = File(dependenciesRoot),
                 properties = this,
-                archiveType = defaultArchiveTypeByHost(host)
+                archiveType = defaultArchiveTypeByHost(host),
-            ){ url, currentBytes, totalBytes ->
+                customProgressCallback = progressCallback
-                print("\n(KonanProperties) Downloading dependency: $url (${currentBytes}/${totalBytes}). ")
+            )
-            }
         }
     }
 }
 
 private fun defaultArchiveTypeByHost(host: KonanTarget): ArchiveType = when (host) {
-    KonanTarget.LINUX_X64,
+    KonanTarget.LINUX_X64, KonanTarget.MACOS_X64, KonanTarget.MACOS_ARM64 -> ArchiveType.TAR_GZ
-    KonanTarget.MACOS_X64,
-    KonanTarget.MACOS_ARM64 -> ArchiveType.TAR_GZ
     KonanTarget.MINGW_X64 -> ArchiveType.ZIP
     else -> error("$host can't be a host platform!")
 }

native/utils/src/org/jetbrains/kotlin/konan/target/Windows.kt

@@ -9,10 +9,11 @@ import org.jetbrains.kotlin.konan.util.InternalServer
 import java.nio.file.Path
 import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
 import org.jetbrains.kotlin.konan.properties.Properties
+import org.jetbrains.kotlin.konan.util.ProgressCallback
 import java.nio.file.Paths
 
-class MingwConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?) : MingwConfigurables,
+class MingwConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback : ProgressCallback) : MingwConfigurables,
-    KonanPropertiesLoader(target, properties, dependenciesRoot) {
+    KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback) {
     override val windowsKit: WindowsKit by lazy {
         when (windowsSdkPartsProvider) {
             WindowsSdkPartsProvider.InternalServer -> createCustomWindowsKitPath(Paths.get(absolute(windowsKitParts)))

native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt

@@ -65,9 +65,6 @@ private fun Properties.findCandidates(dependencies: List<String>): Map<String, L
 
 
 private val KonanPropertiesLoader.dependenciesUrl : String            get() = properties.dependenciesUrl
-private val KonanPropertiesLoader.airplaneMode : Boolean              get() = properties.airplaneMode
-private val KonanPropertiesLoader.downloadingAttempts : Int           get() = properties.downloadingAttempts
-private val KonanPropertiesLoader.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs
 
 sealed class DependencySource {
     data class Local(val path: File) : DependencySource()