[Gradle] Changed the way of unpacking k/n dependencies

Creating `tar` process is not supported by configuration cache. Inner Gradle copy and archive operations don't work well with symlinks. That is why we are using utils from org.apache.commons:commons-compress. ^KT-66422 Fixed
2024-03-07 20:10:49 +01:00
parent fb3c1f1a2f
commit f18d00e6f0
10 changed files with 231 additions and 47 deletions
@@ -1501,6 +1501,12 @@
            <sha256 value="961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908" origin="Generated by Gradle"/>
         </artifact>
      </component>
+      <component group="commons-io" name="commons-io" version="2.15.1">
+         <artifact name="commons-io-2.15.1.jar">
+            <md5 value="84351f7991a0e6722f00e96a4ccc376f" origin="Generated by Gradle"/>
+            <sha256 value="a58af12ee1b68cfd2ebb0c27caef164f084381a00ec81a48cc275fd7ea54e154" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
      <component group="commons-io" name="commons-io" version="2.4">
         <artifact name="commons-io-2.4.jar">
            <md5 value="7f97854dc04c119d461fed14f5d8bb96" origin="Generated by Gradle"/>
@@ -2603,6 +2609,12 @@
            <sha256 value="6aecfd5459728a595601cfa07258d131972ffc39b492eb48bdd596577a2f244a" origin="Generated by Gradle"/>
         </artifact>
      </component>
+      <component group="org.apache.commons" name="commons-compress" version="1.26.0">
+         <artifact name="commons-compress-1.26.0.jar">
+            <md5 value="cbc0bb8eca12b747ca21bf998d736e60" origin="Generated by Gradle"/>
+            <sha256 value="051aceb8bbcc62d0f5b2b8ac72c53767f9c59bfbd050151e65bef6f51c8ed9c9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
      <component group="org.apache.commons" name="commons-csv" version="1.8">
         <artifact name="commons-csv-1.8.jar">
            <md5 value="9019d99d6072f48fd9b1f909f0c45d24" origin="Generated by Gradle"/>
@@ -2615,6 +2627,12 @@
            <sha256 value="d919d904486c037f8d193412da0c92e22a9fa24230b9d67a57855c5c31c7e94e" origin="Generated by Gradle"/>
         </artifact>
      </component>
+      <component group="org.apache.commons" name="commons-lang3" version="3.14.0">
+         <artifact name="commons-lang3-3.14.0.jar">
+            <md5 value="4e5c3f5e6b0b965ef241d7d72ac8971f" origin="Generated by Gradle"/>
+            <sha256 value="7b96bf3ee68949abb5bc465559ac270e0551596fa34523fddf890ec418dde13c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
      <component group="org.apache.commons" name="commons-lang3" version="3.8.1">
         <artifact name="commons-lang3-3.8.1.jar">
            <md5 value="540b1256d887a6993ecbef23371a3302" origin="Generated by Gradle"/>
@@ -69,7 +69,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
    )
    @GradleTest
    fun testMppWithMavenPublish(gradleVersion: GradleVersion) {
-        project("new-mpp-lib-and-app/sample-lib", gradleVersion) {
+        // with Configuration Cache we currently have such problem KT-66423
+        val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
+        project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) {
            val publishedTargets = listOf("kotlinMultiplatform", "jvm6", "nodeJs", "linux64", "mingw64")
            testConfigurationCacheOf(
                *(publishedTargets.map { ":publish${it.replaceFirstChar { it.uppercaseChar() }}PublicationToMavenRepository" }
@@ -87,7 +89,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
    )
    @GradleTest
    fun testAllMetadataJarWithConfigurationCache(gradleVersion: GradleVersion) {
-        project("new-mpp-lib-and-app/sample-lib", gradleVersion) {
+        // with Configuration Cache we currently have such problem KT-66423
+        val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
+        project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) {
            testConfigurationCacheOf(":allMetadataJar")
        }
    }
@@ -100,7 +104,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
    )
    @GradleTest
    fun testCommonizer(gradleVersion: GradleVersion) {
-        project("native-configuration-cache", gradleVersion) {
+        // with Configuration Cache we currently have such problem KT-66423
+        val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
+        project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
            build(":cleanNativeDistributionCommonization")

            build(":lib:compileCommonMainKotlinMetadata") {
@@ -129,7 +135,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
    )
    @GradleTest
    fun testCInteropCommonizer(gradleVersion: GradleVersion) {
-        project("native-configuration-cache", gradleVersion) {
+        // with Configuration Cache we currently have such problem KT-66423
+        val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
+        project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
            testConfigurationCacheOf(":lib:commonizeCInterop")
        }
    }
@@ -278,8 +286,19 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
 }

 abstract class AbstractConfigurationCacheIT : KGPBaseTest() {
-    override val defaultBuildOptions =
-        super.defaultBuildOptions.copy(configurationCache = true)
+
+    @TempDir
+    lateinit var konanDataTempDir: Path
+
+    override val defaultBuildOptions
+        get() = super.defaultBuildOptions.copy(
+            configurationCache = true,
+            konanDataDir = konanDataTempDir,
+            nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
+                // set the KGP's default Kotlin Native version, because in CI we don't have K/N versions in maven repo for each build
+                version = null
+            )
+        )

    protected fun TestProject.testConfigurationCacheOf(
        vararg taskNames: String,
@@ -294,4 +313,14 @@ abstract class AbstractConfigurationCacheIT : KGPBaseTest() {
            buildOptions = buildOptions,
        )
    }
+
+    protected fun buildOptionsToAvoidKT66423(gradleVersion: GradleVersion) =
+        if (gradleVersion == GradleVersion.version(TestVersions.Gradle.G_8_6)) {
+            defaultBuildOptions.copy(
+                konanDataDir = konanDir,
+                nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
+                    version = System.getProperty("kotlinNativeVersion")
+                )
+            )
+        } else defaultBuildOptions
 }
@@ -54,7 +54,9 @@ class MacosCapableConfigurationCacheIT : AbstractConfigurationCacheIT() {
            )
        }

-        project("native-configuration-cache", gradleVersion) {
+        // with Configuration Cache we currently have such problem KT-66423
+        val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
+        project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
            testConfigurationCacheOf(
                "build",
                executedTaskNames = expectedTasks,
@@ -103,6 +103,9 @@ dependencies {
        exclude(group = "*")
    }

+    commonCompileOnly("org.apache.commons:commons-compress:1.26.0")
+    embedded("org.apache.commons:commons-compress:1.26.0")
+
    if (!kotlinBuildProperties.isInJpsBuildIdeaSync) {
        // Adding workaround KT-57317 for Gradle versions where Kotlin runtime <1.8.0
        "mainEmbedded"(project(":kotlin-build-tools-enum-compat"))
@@ -5,22 +5,37 @@

 package org.jetbrains.kotlin.gradle.targets.native.toolchain

+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.file.ArchiveOperations
 import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.FileSystemOperations
+import org.gradle.api.logging.Logger
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.Provider
 import org.gradle.api.services.BuildService
 import org.gradle.api.services.BuildServiceParameters
 import org.gradle.api.tasks.Internal
+import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
 import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionCommonizerLock
 import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider
 import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator
 import org.jetbrains.kotlin.gradle.tasks.withType
 import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
+import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
+import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.konan.target.loadConfigurables
+import org.jetbrains.kotlin.konan.util.ArchiveExtractor
+import org.jetbrains.kotlin.konan.util.ArchiveType
+import java.io.BufferedInputStream
 import java.io.File
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.file.attribute.PosixFilePermission
+import java.util.zip.GZIPInputStream
 import javax.inject.Inject

 private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan"
@@ -38,6 +53,9 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
    @get:Inject
    abstract val fso: FileSystemOperations

+    @get:Inject
+    abstract val archiveOperations: ArchiveOperations
+
    private var canBeReinstalled: Boolean = true // we can reinstall a k/n bundle once during the build

    companion object {
@@ -55,9 +73,20 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
            }
    }

+    /**
+     * This function downloads and installs a Kotlin Native bundle if needed
+     * and then prepares its platform libraries if needed.
+     *
+     * @param project The Gradle project object.
+     * @param kotlinNativeBundleConfiguration Gradle configuration for Kotlin Native Bundle
+     * @param kotlinNativeVersion The version of Kotlin/Native to install
+     * @param bundleDir The directory to store the Kotlin/Native bundle.
+     * @param reinstallFlag A flag indicating whether to reinstall the bundle.
+     * @param konanTargets The set of KonanTarget objects representing the targets for the Kotlin/Native bundle.
+     */
    internal fun prepareKotlinNativeBundle(
        project: Project,
-        kotlinNativeCompilerConfiguration: ConfigurableFileCollection,
+        kotlinNativeBundleConfiguration: ConfigurableFileCollection,
        kotlinNativeVersion: String,
        bundleDir: File,
        reinstallFlag: Boolean,
@@ -73,7 +102,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi

            if (!bundleDir.resolve(KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE).exists()) {
                val gradleCachesKotlinNativeDir =
-                    resolveKotlinNativeConfiguration(kotlinNativeVersion, kotlinNativeCompilerConfiguration)
+                    resolveKotlinNativeConfiguration(kotlinNativeVersion, kotlinNativeBundleConfiguration)

                project.logger.info("Moving Kotlin/Native bundle from tmp directory $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
                fso.copy {
@@ -84,7 +113,37 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
            }
        }

-        project.setupKotlinNativeDependencies(konanTargets)
+        project.setupKotlinNativePlatformLibraries(konanTargets)
+    }
+
+    /**
+     * Downloads native dependencies for Kotlin Native based on the provided configuration.
+     * @return A set of required dependencies that were downloaded.
+     */
+    internal fun downloadNativeDependencies(
+        bundleDir: File,
+        konanDataDir: String?,
+        konanTargets: Set<KonanTarget>,
+        logger: Logger,
+    ): Set<String> {
+        val requiredDependencies = mutableSetOf<String>()
+        val distribution = Distribution(bundleDir.absolutePath, konanDataDir = konanDataDir)
+        konanTargets.forEach { konanTarget ->
+            if (konanTarget.enabledOnCurrentHost) {
+                val konanPropertiesLoader = loadConfigurables(
+                    konanTarget,
+                    distribution.properties,
+                    distribution.dependenciesDir,
+                    progressCallback = { url, currentBytes, totalBytes ->
+                        logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ")
+                    }
+                ) as KonanPropertiesLoader
+
+                requiredDependencies.addAll(konanPropertiesLoader.dependencies)
+                konanPropertiesLoader.downloadDependencies(DependencyExtractor())
+            }
+        }
+        return requiredDependencies
    }

    private fun removeBundleIfNeeded(
@@ -115,7 +174,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
        return gradleCachesKotlinNativeDir
    }

-    private fun Project.setupKotlinNativeDependencies(konanTargets: Set<KonanTarget>) {
+    private fun Project.setupKotlinNativePlatformLibraries(konanTargets: Set<KonanTarget>) {
        val distributionType = NativeDistributionTypeProvider(this).getDistributionType()
        if (distributionType.mustGeneratePlatformLibs) {
            konanTargets.forEach { konanTarget ->
@@ -123,4 +182,66 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
            }
        }
    }
+
+    private inner class DependencyExtractor : ArchiveExtractor {
+
+        override fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) {
+            when (archiveType) {
+                ArchiveType.ZIP -> archiveOperations.zipTree(archive)
+                ArchiveType.TAR_GZ -> unzipTarGz(archive, targetDirectory)
+                else -> error("Unsupported format for unzipping $archive")
+            }
+        }
+
+        private fun unzipTarGz(archive: File, targetDir: File) {
+            GZIPInputStream(BufferedInputStream(archive.inputStream())).use { gzipInputStream ->
+                TarArchiveInputStream(gzipInputStream).use { tarInputStream ->
+                    generateSequence {
+                        tarInputStream.nextEntry
+                    }.forEach { entry: TarArchiveEntry ->
+                        val outputFile = File("$targetDir/${entry.name}")
+                        if (entry.isDirectory) {
+                            outputFile.mkdirs()
+                        } else {
+                            if (entry.isSymbolicLink) {
+                                Files.createSymbolicLink(outputFile.toPath(), Paths.get(entry.linkName))
+                            } else {
+                                outputFile.outputStream().use {
+                                    tarInputStream.copyTo(it)
+                                }
+                                Files.setPosixFilePermissions(outputFile.toPath(), getPosixFilePermissions(entry.mode))
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        private fun getPosixFilePermissions(mode: Int): Set<PosixFilePermission> {
+            val permissions: MutableSet<PosixFilePermission> = mutableSetOf()
+
+            // adding owner permissions
+            permissions.addPermission(mode, 0b100_000_000, PosixFilePermission.OWNER_READ)
+            permissions.addPermission(mode, 0b010_000_000, PosixFilePermission.OWNER_WRITE)
+            permissions.addPermission(mode, 0b001_000_000, PosixFilePermission.OWNER_EXECUTE)
+
+            // adding group permissions
+            permissions.addPermission(mode, 0b000_100_000, PosixFilePermission.GROUP_READ)
+            permissions.addPermission(mode, 0b000_010_000, PosixFilePermission.GROUP_WRITE)
+            permissions.addPermission(mode, 0b000_001_000, PosixFilePermission.GROUP_EXECUTE)
+
+            // adding other permissions
+            permissions.addPermission(mode, 0b000_000_100, PosixFilePermission.OTHERS_READ)
+            permissions.addPermission(mode, 0b000_000_010, PosixFilePermission.OTHERS_WRITE)
+            permissions.addPermission(mode, 0b000_000_001, PosixFilePermission.OTHERS_EXECUTE)
+
+            return permissions
+        }
+
+        private fun MutableSet<PosixFilePermission>.addPermission(mode: Int, permissionBitMask: Int, permission: PosixFilePermission) {
+            if ((mode and permissionBitMask) > 0) {
+                add(permission)
+            }
+        }
+    }
 }
@@ -17,14 +17,10 @@ import org.jetbrains.kotlin.compilerRunner.konanHome
 import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled
 import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
-import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
 import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
 import org.jetbrains.kotlin.gradle.utils.filesProvider
 import org.jetbrains.kotlin.gradle.utils.property
-import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
-import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.KonanTarget
-import org.jetbrains.kotlin.konan.target.loadConfigurables

 /**
 * This is a nested provider for all native tasks
@@ -74,26 +70,17 @@ internal class KotlinNativeProvider(
    val kotlinNativeDependencies: Provider<Set<String>> =
        kotlinNativeBundleVersion
            .zip(bundleDirectory) { _, bundleDir ->
-                val requiredDependencies = mutableSetOf<String>()
                if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) {
-                    val distribution = Distribution(bundleDir.asFile.absolutePath, konanDataDir = konanDataDir.orNull)
-                    konanTargets.forEach { konanTarget ->
-                        if (konanTarget.enabledOnCurrentHost) {
-                            val konanPropertiesLoader = loadConfigurables(
-                                konanTarget,
-                                distribution.properties,
-                                distribution.dependenciesDir,
-                                progressCallback = { url, currentBytes, totalBytes ->
-                                    project.logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ")
+                    kotlinNativeBundleBuildService.get()
+                        .downloadNativeDependencies(
+                            bundleDir.asFile,
+                            konanDataDir.orNull,
+                            konanTargets,
+                            project.logger
+                        )
+                } else {
+                    emptySet()
                }
-                            ) as KonanPropertiesLoader
-
-                            requiredDependencies.addAll(konanPropertiesLoader.dependencies)
-                            konanPropertiesLoader.downloadDependencies()
-                        }
-                    }
-                }
-                requiredDependencies
            }

    // Gradle tries to evaluate this val during configuration cache,
@@ -19,9 +19,7 @@ package org.jetbrains.kotlin.konan.properties
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.Configurables
 import org.jetbrains.kotlin.konan.target.HostManager
-import org.jetbrains.kotlin.konan.util.ArchiveType
-import org.jetbrains.kotlin.konan.util.DependencyProcessor
-import org.jetbrains.kotlin.konan.util.ProgressCallback
+import org.jetbrains.kotlin.konan.util.*
 import java.io.File

 interface TargetableExternalStorage {
@@ -76,6 +74,10 @@ abstract class KonanPropertiesLoader(
        dependencyProcessor!!.run()
    }

+    fun downloadDependencies(archiveExtractor: ArchiveExtractor) {
+        dependencyProcessor!!.run(archiveExtractor)
+    }
+
    // TODO: We may want to add caching to avoid repeated resolve.
    override fun targetString(key: String): String? = properties.targetString(key, target)
    override fun targetList(key: String): List<String> = properties.targetList(key, target)
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.util
+
+import java.io.File
+
+/**
+ * An interface for extracting archive files.
+ */
+interface ArchiveExtractor {
+
+    /**
+     * Extracts the contents of the specified archive file to the target directory.
+     *
+     * @param archive The archive file to extract.
+     * @param targetDirectory The directory where the contents of the archive will be extracted to.
+     * @param archiveType The type of the archive file.
+     */
+    fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType)
+}
@@ -33,9 +33,8 @@ enum class ArchiveType(val fileExtension: String) {
    }
 }

-class DependencyExtractor(
-        private val archiveType: ArchiveType
-) {
+class DependencyExtractor : ArchiveExtractor {
+
    private fun extractTarGz(tarGz: File, targetDirectory: File) {
        val tarProcess = ProcessBuilder().apply {
            command("tar", "-xzf", tarGz.canonicalPath)
@@ -53,12 +52,13 @@ class DependencyExtractor(
                tarProcess.destroy()
                throw RuntimeException(
                    "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
-                    "Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec.")
+                            "Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec."
+                )
            }
        }
    }

-    fun extract(archive: File, targetDirectory: File) {
+    override fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) {
        when (archiveType) {
            ArchiveType.ZIP -> archive.toPath().unzipTo(targetDirectory.toPath())
            ArchiveType.TAR_GZ -> extractTarGz(archive, targetDirectory)
@@ -109,7 +109,6 @@ class DependencyProcessor(
    private var isInfoShown = false

    private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback)
-    private val extractor = DependencyExtractor(archiveType)

    constructor(dependenciesRoot: File,
                properties: KonanPropertiesLoader,
@@ -172,7 +171,7 @@ class DependencyProcessor(
        }
    }

-    private fun downloadDependency(dependency: String, baseUrl: String) {
+    private fun downloadDependency(dependency: String, baseUrl: String, archiveExtractor: ArchiveExtractor) {
        val depDir = File(dependenciesDirectory, dependency)
        val depName = depDir.name

@@ -211,7 +210,7 @@ class DependencyProcessor(
            downloader.download(url, archive)
        }
        println("Extracting dependency: $archive into $dependenciesDirectory")
-        extractor.extract(archive, dependenciesDirectory)
+        archiveExtractor.extract(archive, dependenciesDirectory, archiveType)
        if (deleteArchives) {
            archive.delete()
        }
@@ -275,7 +274,7 @@ class DependencyProcessor(
        }
    }

-    fun run() {
+    fun run(archiveExtractor: ArchiveExtractor = DependencyExtractor()) {
        // We need a lock that can be shared between different classloaders (KT-39781).
        // TODO: Rework dependencies downloading to avoid storing the lock in the system properties.
        val lock = System.getProperties().computeIfAbsent("kotlin.native.dependencies.lock") {
@@ -300,7 +299,7 @@ class DependencyProcessor(
                            DependencySource.Remote.Internal -> InternalServer.url
                        }
                        // TODO: consider using different caches for different remotes.
-                        downloadDependency(dependency, baseUrl)
+                        downloadDependency(dependency, baseUrl, archiveExtractor)
                    }
                }
            }