From f525d03e6743d49f4840753cb169aa8c80e2626d Mon Sep 17 00:00:00 2001 From: Dmitrii Krasnov Date: Wed, 14 Feb 2024 15:57:02 +0100 Subject: [PATCH] [Gradle] Moved downloading konan dependencies from compiler to Gradle ^KT-65823 Fixed --- .../KotlinNativeDependenciesDownloadIT.kt | 59 +++++++++++++++++ .../NativeDistributionCommonizerTask.kt | 10 +-- .../native/toolchain/KotlinNativeProvider.kt | 31 +++++++++ .../jetbrains/kotlin/konan/target/Apple.kt | 17 +++-- .../kotlin/konan/target/ConfigurablesImpl.kt | 60 +++++++++-------- .../kotlin/konan/target/KonanProperties.kt | 64 +++++++++---------- .../jetbrains/kotlin/konan/target/Windows.kt | 5 +- .../kotlin/konan/util/DependencyProcessor.kt | 3 - 8 files changed, 174 insertions(+), 75 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/KotlinNativeDependenciesDownloadIT.kt diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/KotlinNativeDependenciesDownloadIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/KotlinNativeDependenciesDownloadIT.kt new file mode 100644 index 00000000000..d91962f7103 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/KotlinNativeDependenciesDownloadIT.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.native + +import org.gradle.util.GradleVersion +import org.jetbrains.kotlin.gradle.testbase.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.condition.OS +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +// We temporarily disable it for windows until a proper fix is found for this issue: KT-62761 +@OsCondition( + supportedOn = [OS.MAC, OS.LINUX], enabledOnCI = [OS.MAC, OS.LINUX] +) +@DisplayName("This test class contains different scenarios with downloading dependencies for Kotlin Native Compiler during build.") +@NativeGradlePluginTests +class KotlinNativeDependenciesDownloadIT : KGPBaseTest() { + + @TempDir + lateinit var konanDataTempDir: Path + + override val defaultBuildOptions: BuildOptions + get() = super.defaultBuildOptions.withBundledKotlinNative().copy( + // For each test in this class, we need to provide an isolated .konan directory, + // so we create it within each test project folder + konanDataDir = konanDataTempDir + ) + + @DisplayName("Gradle should download dependencies before compile execution") + @GradleTest + fun shouldDownloadDependenciesBeforeCompilerExecution(gradleVersion: GradleVersion) { + nativeProject("mpp-default-hierarchy", gradleVersion) { + build("assemble") { + assertOutputContains("Downloading dependency for Kotlin Native") + assertOutputDoesNotContain("(KonanProperties) Downloading dependency") + } + } + } + + @DisplayName("Compiler should download dependencies when Kotlin Native Toolchain disabled ") + @GradleTest + fun checkCompilerDownloadsDependenciesWhenToochainDisabled(gradleVersion: GradleVersion) { + nativeProject("mpp-default-hierarchy", gradleVersion) { + + build( + "assemble", buildOptions = defaultBuildOptions.copy( + freeArgs = listOf("-Pkotlin.native.toolchain.enabled=false"), + ) + ) { + assertOutputContains("(KonanProperties) Downloading dependency") + assertOutputDoesNotContain("Downloading dependency for Kotlin Native") + } + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt index cfbb124c46f..9caec6b3b10 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt @@ -22,9 +22,6 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.commonizer.konanTargets import org.jetbrains.kotlin.compilerRunner.* -import org.jetbrains.kotlin.compilerRunner.GradleCliCommonizer -import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner -import org.jetbrains.kotlin.compilerRunner.konanHome import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion @@ -104,7 +101,12 @@ internal abstract class NativeDistributionCommonizerTask @get:Nested internal val kotlinNativeProvider: Provider = project.provider { - KotlinNativeProvider(project, commonizerTargets.flatMap { target -> target.konanTargets }.toSet(), kotlinNativeBundleBuildService) + KotlinNativeProvider( + project, + commonizerTargets.flatMap { target -> target.konanTargets }.toSet(), + kotlinNativeBundleBuildService, + enableDependenciesDownloading = false + ) } @TaskAction diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt index 617d992383c..41bbcaabe1f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt @@ -17,10 +17,14 @@ import org.jetbrains.kotlin.compilerRunner.konanHome import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider +import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader import org.jetbrains.kotlin.gradle.utils.filesProvider import org.jetbrains.kotlin.gradle.utils.property +import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader +import org.jetbrains.kotlin.konan.target.Distribution import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.loadConfigurables /** * This is a nested provider for all native tasks @@ -29,6 +33,7 @@ internal class KotlinNativeProvider( project: Project, konanTargets: Set, kotlinNativeBundleBuildService: Provider, + enableDependenciesDownloading: Boolean = true, ) { constructor( project: Project, @@ -65,6 +70,32 @@ internal class KotlinNativeProvider( kotlinNativeVersion } + @get:Input + val kotlinNativeDependencies: Provider> = + kotlinNativeBundleVersion + .zip(bundleDirectory) { _, bundleDir -> + val requiredDependencies = mutableSetOf() + if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) { + val distribution = Distribution(bundleDir.asFile.absolutePath, konanDataDir = konanDataDir.orNull) + konanTargets.forEach { konanTarget -> + if (konanTarget.enabledOnCurrentHost) { + val konanPropertiesLoader = loadConfigurables( + konanTarget, + distribution.properties, + distribution.dependenciesDir, + progressCallback = { url, currentBytes, totalBytes -> + project.logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ") + } + ) as KonanPropertiesLoader + + requiredDependencies.addAll(konanPropertiesLoader.dependencies) + konanPropertiesLoader.downloadDependencies() + } + } + } + requiredDependencies + } + // Gradle tries to evaluate this val during configuration cache, // which lead to resolving configuration, even if k/n bundle is in konan home directory. @Transient diff --git a/native/utils/src/org/jetbrains/kotlin/konan/target/Apple.kt b/native/utils/src/org/jetbrains/kotlin/konan/target/Apple.kt index 110266b0c71..11b47499056 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/target/Apple.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/target/Apple.kt @@ -19,12 +19,14 @@ package org.jetbrains.kotlin.konan.target import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader import org.jetbrains.kotlin.konan.properties.Properties import org.jetbrains.kotlin.konan.util.InternalServer +import org.jetbrains.kotlin.konan.util.ProgressCallback class AppleConfigurablesImpl( target: KonanTarget, properties: Properties, - dependenciesDir: String? -) : AppleConfigurables, KonanPropertiesLoader(target, properties, dependenciesDir) { + dependenciesDir: String?, + progressCallback: ProgressCallback, +) : AppleConfigurables, KonanPropertiesLoader(target, properties, dependenciesDir, progressCallback = progressCallback) { private val sdkDependency = this.targetSysRoot!! private val toolchainDependency = this.targetToolchain!! @@ -45,10 +47,13 @@ class AppleConfigurablesImpl( XcodePartsProvider.InternalServer -> absolute(additionalToolsDir) } - override val dependencies get() = super.dependencies + when (xcodePartsProvider) { - is XcodePartsProvider.Local -> emptyList() - XcodePartsProvider.InternalServer -> listOf(sdkDependency, toolchainDependency, xcodeAddonDependency) - } + override val dependencies + get() = super.dependencies + + if (InternalServer.isAvailable) listOf( + sdkDependency, + toolchainDependency, + xcodeAddonDependency + ) else emptyList() private val xcodePartsProvider by lazy { if (InternalServer.isAvailable) { diff --git a/native/utils/src/org/jetbrains/kotlin/konan/target/ConfigurablesImpl.kt b/native/utils/src/org/jetbrains/kotlin/konan/target/ConfigurablesImpl.kt index 26a08a25d8c..af533f67ce0 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/target/ConfigurablesImpl.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/target/ConfigurablesImpl.kt @@ -17,34 +17,42 @@ package org.jetbrains.kotlin.konan.target import org.jetbrains.kotlin.konan.properties.* +import org.jetbrains.kotlin.konan.util.ProgressCallback -class GccConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?) - : GccConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot), ConfigurablesWithEmulator { +class GccConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : GccConfigurables, + KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback), ConfigurablesWithEmulator { override val dependencies: List get() = super.dependencies + listOfNotNull(emulatorDependency) - } - -class AndroidConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?) - : AndroidConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot) - -class WasmConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?) - : WasmConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot) - -class ZephyrConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?) - : ZephyrConfigurables, KonanPropertiesLoader(target, properties, dependenciesRoot) - - -fun loadConfigurables(target: KonanTarget, properties: Properties, dependenciesRoot: String?): Configurables = when (target.family) { - Family.LINUX -> GccConfigurablesImpl(target, properties, dependenciesRoot) - - Family.TVOS, Family.WATCHOS, Family.IOS, Family.OSX -> AppleConfigurablesImpl(target, properties, dependenciesRoot) - - Family.ANDROID -> AndroidConfigurablesImpl(target, properties, dependenciesRoot) - - Family.MINGW -> MingwConfigurablesImpl(target, properties, dependenciesRoot) - - Family.WASM -> WasmConfigurablesImpl(target, properties, dependenciesRoot) - - Family.ZEPHYR -> ZephyrConfigurablesImpl(target, properties, dependenciesRoot) +} + +class AndroidConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : AndroidConfigurables, + KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback) + +class WasmConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : WasmConfigurables, + KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback) + +class ZephyrConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback: ProgressCallback) : ZephyrConfigurables, + KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback) + + +fun loadConfigurables( + target: KonanTarget, + properties: Properties, + dependenciesRoot: String?, + progressCallback: ProgressCallback = { url, currentBytes, totalBytes -> + print("\n(KonanProperties) Downloading dependency: $url (${currentBytes}/${totalBytes}). ") + }, +): Configurables = when (target.family) { + Family.LINUX -> GccConfigurablesImpl(target, properties, dependenciesRoot, progressCallback) + + Family.TVOS, Family.WATCHOS, Family.IOS, Family.OSX -> AppleConfigurablesImpl(target, properties, dependenciesRoot, progressCallback) + + Family.ANDROID -> AndroidConfigurablesImpl(target, properties, dependenciesRoot, progressCallback) + + Family.MINGW -> MingwConfigurablesImpl(target, properties, dependenciesRoot, progressCallback) + + Family.WASM -> WasmConfigurablesImpl(target, properties, dependenciesRoot, progressCallback) + + Family.ZEPHYR -> ZephyrConfigurablesImpl(target, properties, dependenciesRoot, progressCallback) } diff --git a/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt b/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt index 7254048f4aa..492b85623fb 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt @@ -21,15 +21,16 @@ import org.jetbrains.kotlin.konan.target.Configurables import org.jetbrains.kotlin.konan.target.HostManager import org.jetbrains.kotlin.konan.util.ArchiveType import org.jetbrains.kotlin.konan.util.DependencyProcessor +import org.jetbrains.kotlin.konan.util.ProgressCallback import java.io.File interface TargetableExternalStorage { - fun targetString(key: String): String? + fun targetString(key: String): String? fun targetList(key: String): List - fun hostString(key: String): String? - fun hostList(key: String): List - fun hostTargetString(key: String): String? - fun hostTargetList(key: String): List + fun hostString(key: String): String? + fun hostList(key: String): List + fun hostTargetString(key: String): String? + fun hostTargetList(key: String): List fun absolute(value: String?): String fun downloadDependencies() } @@ -39,21 +40,24 @@ abstract class KonanPropertiesLoader( val properties: Properties, private val dependenciesRoot: String?, private val host: KonanTarget = HostManager.host, + private val progressCallback: ProgressCallback, ) : Configurables { + + private val predefinedLlvmDistributions: Set = - properties.propertyList("predefinedLlvmDistributions").toSet() + properties.propertyList("predefinedLlvmDistributions").toSet() private val predefinedLibffiVersions: Set = - properties.propertyList("predefinedLibffiVersions").toSet() + properties.propertyList("predefinedLibffiVersions").toSet() private fun getPredefinedDependencyOrNull( - dependencyName: String, - dependencyAccessor: () -> String?, - predefinedDependencies: Set + dependencyName: String, + dependencyAccessor: () -> String?, + predefinedDependencies: Set, ): String? { // Store into variable to avoid repeated resolve. val dependency = dependencyAccessor() - ?: error("Undefined $dependencyName!") + ?: error("Undefined $dependencyName!") return when (dependency) { in predefinedDependencies -> dependency else -> null @@ -61,8 +65,8 @@ abstract class KonanPropertiesLoader( } private fun compilerDependencies(): List = listOfNotNull( - getPredefinedDependencyOrNull("LLVM home", this::llvmHome, predefinedLlvmDistributions), - getPredefinedDependencyOrNull("libffi version", this::libffiDir, predefinedLibffiVersions) + getPredefinedDependencyOrNull("LLVM home", this::llvmHome, predefinedLlvmDistributions), + getPredefinedDependencyOrNull("libffi version", this::libffiDir, predefinedLibffiVersions) ) open val dependencies: List @@ -73,38 +77,30 @@ abstract class KonanPropertiesLoader( } // TODO: We may want to add caching to avoid repeated resolve. - override fun targetString(key: String): String? - = properties.targetString(key, target) - override fun targetList(key: String): List - = properties.targetList(key, target) - override fun hostString(key: String): String? - = properties.hostString(key, host) - override fun hostList(key: String): List - = properties.hostList(key, host) - override fun hostTargetString(key: String): String? - = properties.hostTargetString(key, target, host) - override fun hostTargetList(key: String): List - = properties.hostTargetList(key, target, host) + override fun targetString(key: String): String? = properties.targetString(key, target) + override fun targetList(key: String): List = properties.targetList(key, target) + override fun hostString(key: String): String? = properties.hostString(key, host) + override fun hostList(key: String): List = properties.hostList(key, host) + override fun hostTargetString(key: String): String? = properties.hostTargetString(key, target, host) + override fun hostTargetList(key: String): List = properties.hostTargetList(key, target, host) override fun absolute(value: String?): String = - dependencyProcessor!!.resolve(value!!).absolutePath - private val dependencyProcessor by lazy { + dependencyProcessor!!.resolve(value!!).absolutePath + + private val dependencyProcessor by lazy { dependenciesRoot?.let { DependencyProcessor( dependenciesRoot = File(dependenciesRoot), properties = this, - archiveType = defaultArchiveTypeByHost(host) - ){ url, currentBytes, totalBytes -> - print("\n(KonanProperties) Downloading dependency: $url (${currentBytes}/${totalBytes}). ") - } + archiveType = defaultArchiveTypeByHost(host), + customProgressCallback = progressCallback + ) } } } private fun defaultArchiveTypeByHost(host: KonanTarget): ArchiveType = when (host) { - KonanTarget.LINUX_X64, - KonanTarget.MACOS_X64, - KonanTarget.MACOS_ARM64 -> ArchiveType.TAR_GZ + KonanTarget.LINUX_X64, KonanTarget.MACOS_X64, KonanTarget.MACOS_ARM64 -> ArchiveType.TAR_GZ KonanTarget.MINGW_X64 -> ArchiveType.ZIP else -> error("$host can't be a host platform!") } \ No newline at end of file diff --git a/native/utils/src/org/jetbrains/kotlin/konan/target/Windows.kt b/native/utils/src/org/jetbrains/kotlin/konan/target/Windows.kt index cb6d5a2f557..5813da7eff5 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/target/Windows.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/target/Windows.kt @@ -9,10 +9,11 @@ import org.jetbrains.kotlin.konan.util.InternalServer import java.nio.file.Path import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader import org.jetbrains.kotlin.konan.properties.Properties +import org.jetbrains.kotlin.konan.util.ProgressCallback import java.nio.file.Paths -class MingwConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?) : MingwConfigurables, - KonanPropertiesLoader(target, properties, dependenciesRoot) { +class MingwConfigurablesImpl(target: KonanTarget, properties: Properties, dependenciesRoot: String?, progressCallback : ProgressCallback) : MingwConfigurables, + KonanPropertiesLoader(target, properties, dependenciesRoot, progressCallback = progressCallback) { override val windowsKit: WindowsKit by lazy { when (windowsSdkPartsProvider) { WindowsSdkPartsProvider.InternalServer -> createCustomWindowsKitPath(Paths.get(absolute(windowsKitParts))) diff --git a/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt b/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt index 383a46a67ef..45e6bf3f326 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt @@ -65,9 +65,6 @@ private fun Properties.findCandidates(dependencies: List): Map