azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Gradle] Changed the way of unpacking k/n dependencies

Browse Source 
Creating `tar` process is not supported by configuration cache.
Inner Gradle copy and archive operations don't work well with symlinks.
That is why we are using utils from org.apache.commons:commons-compress.

^KT-66422 Fixed
This commit is contained in:
Dmitrii Krasnov
2024-03-07 20:10:49 +01:00
committed by Space Team
parent fb3c1f1a2f
commit f18d00e6f0
10 changed files with 231 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt
+35 -6
View File
@@ -69,7 +69,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
)
@GradleTest
fun testMppWithMavenPublish(gradleVersion: GradleVersion) {
project("new-mpp-lib-and-app/sample-lib", gradleVersion) {
// with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) {
val publishedTargets = listOf("kotlinMultiplatform", "jvm6", "nodeJs", "linux64", "mingw64")
testConfigurationCacheOf(
*(publishedTargets.map { ":publish${it.replaceFirstChar { it.uppercaseChar() }}PublicationToMavenRepository" }
@@ -87,7 +89,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
)
@GradleTest
fun testAllMetadataJarWithConfigurationCache(gradleVersion: GradleVersion) {
project("new-mpp-lib-and-app/sample-lib", gradleVersion) {
// with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) {
testConfigurationCacheOf(":allMetadataJar")
}
}
@@ -100,7 +104,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
)
@GradleTest
fun testCommonizer(gradleVersion: GradleVersion) {
project("native-configuration-cache", gradleVersion) {
// with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
build(":cleanNativeDistributionCommonization")
build(":lib:compileCommonMainKotlinMetadata") {
@@ -129,7 +135,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
)
@GradleTest
fun testCInteropCommonizer(gradleVersion: GradleVersion) {
project("native-configuration-cache", gradleVersion) {
// with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
testConfigurationCacheOf(":lib:commonizeCInterop")
}
}
@@ -278,8 +286,19 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
}
abstract class AbstractConfigurationCacheIT : KGPBaseTest() {
override val defaultBuildOptions =
super.defaultBuildOptions.copy(configurationCache = true)
@TempDir
lateinit var konanDataTempDir: Path
override val defaultBuildOptions
get() = super.defaultBuildOptions.copy(
configurationCache = true,
konanDataDir = konanDataTempDir,
nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
// set the KGP's default Kotlin Native version, because in CI we don't have K/N versions in maven repo for each build
version = null
)
)
protected fun TestProject.testConfigurationCacheOf(
vararg taskNames: String,
@@ -294,4 +313,14 @@ abstract class AbstractConfigurationCacheIT : KGPBaseTest() {
buildOptions = buildOptions,
)
}
protected fun buildOptionsToAvoidKT66423(gradleVersion: GradleVersion) =
if (gradleVersion == GradleVersion.version(TestVersions.Gradle.G_8_6)) {
defaultBuildOptions.copy(
konanDataDir = konanDir,
nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
version = System.getProperty("kotlinNativeVersion")
)
)
} else defaultBuildOptions
}
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/MacosCapableConfigurationCacheIT.kt
+3 -1
View File
@@ -54,7 +54,9 @@ class MacosCapableConfigurationCacheIT : AbstractConfigurationCacheIT() {
)
}
project("native-configuration-cache", gradleVersion) {
// with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
testConfigurationCacheOf(
"build",
executedTaskNames = expectedTasks,
libraries/tools/kotlin-gradle-plugin/build.gradle.kts
+3
View File
@@ -103,6 +103,9 @@ dependencies {
exclude(group = "*")
}
commonCompileOnly("org.apache.commons:commons-compress:1.26.0")
embedded("org.apache.commons:commons-compress:1.26.0")
if (!kotlinBuildProperties.isInJpsBuildIdeaSync) {
// Adding workaround KT-57317 for Gradle versions where Kotlin runtime <1.8.0
"mainEmbedded"(project(":kotlin-build-tools-enum-compat"))
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeBundleBuildService.kt
+125 -4
View File
@@ -5,22 +5,37 @@
package org.jetbrains.kotlin.gradle.targets.native.toolchain
import org.apache.commons.compress.archivers.tar.TarArchiveEntry
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.ArchiveOperations
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.logging.Logger
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal
import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionCommonizerLock
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider
import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator
import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.loadConfigurables
import org.jetbrains.kotlin.konan.util.ArchiveExtractor
import org.jetbrains.kotlin.konan.util.ArchiveType
import java.io.BufferedInputStream
import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import java.nio.file.attribute.PosixFilePermission
import java.util.zip.GZIPInputStream
import javax.inject.Inject
private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan"
@@ -38,6 +53,9 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
@get:Inject
abstract val fso: FileSystemOperations
@get:Inject
abstract val archiveOperations: ArchiveOperations
private var canBeReinstalled: Boolean = true // we can reinstall a k/n bundle once during the build
companion object {
@@ -55,9 +73,20 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
}
}
/**
* This function downloads and installs a Kotlin Native bundle if needed
* and then prepares its platform libraries if needed.
*
* @param project The Gradle project object.
* @param kotlinNativeBundleConfiguration Gradle configuration for Kotlin Native Bundle
* @param kotlinNativeVersion The version of Kotlin/Native to install
* @param bundleDir The directory to store the Kotlin/Native bundle.
* @param reinstallFlag A flag indicating whether to reinstall the bundle.
* @param konanTargets The set of KonanTarget objects representing the targets for the Kotlin/Native bundle.
*/
internal fun prepareKotlinNativeBundle(
project: Project,
kotlinNativeCompilerConfiguration: ConfigurableFileCollection,
kotlinNativeBundleConfiguration: ConfigurableFileCollection,
kotlinNativeVersion: String,
bundleDir: File,
reinstallFlag: Boolean,
@@ -73,7 +102,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
if (!bundleDir.resolve(KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE).exists()) {
val gradleCachesKotlinNativeDir =
resolveKotlinNativeConfiguration(kotlinNativeVersion, kotlinNativeCompilerConfiguration)
resolveKotlinNativeConfiguration(kotlinNativeVersion, kotlinNativeBundleConfiguration)
project.logger.info("Moving Kotlin/Native bundle from tmp directory $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
fso.copy {
@@ -84,7 +113,37 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
}
}
project.setupKotlinNativeDependencies(konanTargets)
project.setupKotlinNativePlatformLibraries(konanTargets)
}
/**
* Downloads native dependencies for Kotlin Native based on the provided configuration.
* @return A set of required dependencies that were downloaded.
*/
internal fun downloadNativeDependencies(
bundleDir: File,
konanDataDir: String?,
konanTargets: Set<KonanTarget>,
logger: Logger,
): Set<String> {
val requiredDependencies = mutableSetOf<String>()
val distribution = Distribution(bundleDir.absolutePath, konanDataDir = konanDataDir)
konanTargets.forEach { konanTarget ->
if (konanTarget.enabledOnCurrentHost) {
val konanPropertiesLoader = loadConfigurables(
konanTarget,
distribution.properties,
distribution.dependenciesDir,
progressCallback = { url, currentBytes, totalBytes ->
logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ")
}
) as KonanPropertiesLoader
requiredDependencies.addAll(konanPropertiesLoader.dependencies)
konanPropertiesLoader.downloadDependencies(DependencyExtractor())
}
}
return requiredDependencies
}
private fun removeBundleIfNeeded(
@@ -115,7 +174,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
return gradleCachesKotlinNativeDir
}
private fun Project.setupKotlinNativeDependencies(konanTargets: Set<KonanTarget>) {
private fun Project.setupKotlinNativePlatformLibraries(konanTargets: Set<KonanTarget>) {
val distributionType = NativeDistributionTypeProvider(this).getDistributionType()
if (distributionType.mustGeneratePlatformLibs) {
konanTargets.forEach { konanTarget ->
@@ -123,4 +182,66 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
}
}
}
private inner class DependencyExtractor : ArchiveExtractor {
override fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) {
when (archiveType) {
ArchiveType.ZIP -> archiveOperations.zipTree(archive)
ArchiveType.TAR_GZ -> unzipTarGz(archive, targetDirectory)
else -> error("Unsupported format for unzipping $archive")
}
}
private fun unzipTarGz(archive: File, targetDir: File) {
GZIPInputStream(BufferedInputStream(archive.inputStream())).use { gzipInputStream ->
TarArchiveInputStream(gzipInputStream).use { tarInputStream ->
generateSequence {
tarInputStream.nextEntry
}.forEach { entry: TarArchiveEntry ->
val outputFile = File("$targetDir/${entry.name}")
if (entry.isDirectory) {
outputFile.mkdirs()
} else {
if (entry.isSymbolicLink) {
Files.createSymbolicLink(outputFile.toPath(), Paths.get(entry.linkName))
} else {
outputFile.outputStream().use {
tarInputStream.copyTo(it)
}
Files.setPosixFilePermissions(outputFile.toPath(), getPosixFilePermissions(entry.mode))
}
}
}
}
}
}
private fun getPosixFilePermissions(mode: Int): Set<PosixFilePermission> {
val permissions: MutableSet<PosixFilePermission> = mutableSetOf()
// adding owner permissions
permissions.addPermission(mode, 0b100_000_000, PosixFilePermission.OWNER_READ)
permissions.addPermission(mode, 0b010_000_000, PosixFilePermission.OWNER_WRITE)
permissions.addPermission(mode, 0b001_000_000, PosixFilePermission.OWNER_EXECUTE)
// adding group permissions
permissions.addPermission(mode, 0b000_100_000, PosixFilePermission.GROUP_READ)
permissions.addPermission(mode, 0b000_010_000, PosixFilePermission.GROUP_WRITE)
permissions.addPermission(mode, 0b000_001_000, PosixFilePermission.GROUP_EXECUTE)
// adding other permissions
permissions.addPermission(mode, 0b000_000_100, PosixFilePermission.OTHERS_READ)
permissions.addPermission(mode, 0b000_000_010, PosixFilePermission.OTHERS_WRITE)
permissions.addPermission(mode, 0b000_000_001, PosixFilePermission.OTHERS_EXECUTE)
return permissions
}
private fun MutableSet<PosixFilePermission>.addPermission(mode: Int, permissionBitMask: Int, permission: PosixFilePermission) {
if ((mode and permissionBitMask) > 0) {
add(permission)
}
}
}
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt
+9 -22
View File
@@ -17,14 +17,10 @@ import org.jetbrains.kotlin.compilerRunner.konanHome
import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled
import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
import org.jetbrains.kotlin.gradle.utils.filesProvider
import org.jetbrains.kotlin.gradle.utils.property
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.loadConfigurables
/**
* This is a nested provider for all native tasks
@@ -74,26 +70,17 @@ internal class KotlinNativeProvider(
val kotlinNativeDependencies: Provider<Set<String>> =
kotlinNativeBundleVersion
.zip(bundleDirectory) { _, bundleDir ->
val requiredDependencies = mutableSetOf<String>()
if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) {
val distribution = Distribution(bundleDir.asFile.absolutePath, konanDataDir = konanDataDir.orNull)
konanTargets.forEach { konanTarget ->
if (konanTarget.enabledOnCurrentHost) {
val konanPropertiesLoader = loadConfigurables(
konanTarget,
distribution.properties,
distribution.dependenciesDir,
progressCallback = { url, currentBytes, totalBytes ->
project.logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ")
}
) as KonanPropertiesLoader
requiredDependencies.addAll(konanPropertiesLoader.dependencies)
konanPropertiesLoader.downloadDependencies()
}
}
kotlinNativeBundleBuildService.get()
.downloadNativeDependencies(
bundleDir.asFile,
konanDataDir.orNull,
konanTargets,
project.logger
)
} else {
emptySet()
}
requiredDependencies
}
// Gradle tries to evaluate this val during configuration cache,