[Gradle] Changed the way of unpacking k/n dependencies

Creating `tar` process is not supported by configuration cache.
Inner Gradle copy and archive operations don't work well with symlinks.
That is why we are using utils from org.apache.commons:commons-compress.

^KT-66422 Fixed
This commit is contained in:
Dmitrii Krasnov
2024-03-07 20:10:49 +01:00
committed by Space Team
parent fb3c1f1a2f
commit f18d00e6f0
10 changed files with 231 additions and 47 deletions
+18
View File
@@ -1501,6 +1501,12 @@
<sha256 value="961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908" origin="Generated by Gradle"/> <sha256 value="961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="commons-io" name="commons-io" version="2.15.1">
<artifact name="commons-io-2.15.1.jar">
<md5 value="84351f7991a0e6722f00e96a4ccc376f" origin="Generated by Gradle"/>
<sha256 value="a58af12ee1b68cfd2ebb0c27caef164f084381a00ec81a48cc275fd7ea54e154" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="commons-io" name="commons-io" version="2.4"> <component group="commons-io" name="commons-io" version="2.4">
<artifact name="commons-io-2.4.jar"> <artifact name="commons-io-2.4.jar">
<md5 value="7f97854dc04c119d461fed14f5d8bb96" origin="Generated by Gradle"/> <md5 value="7f97854dc04c119d461fed14f5d8bb96" origin="Generated by Gradle"/>
@@ -2603,6 +2609,12 @@
<sha256 value="6aecfd5459728a595601cfa07258d131972ffc39b492eb48bdd596577a2f244a" origin="Generated by Gradle"/> <sha256 value="6aecfd5459728a595601cfa07258d131972ffc39b492eb48bdd596577a2f244a" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="org.apache.commons" name="commons-compress" version="1.26.0">
<artifact name="commons-compress-1.26.0.jar">
<md5 value="cbc0bb8eca12b747ca21bf998d736e60" origin="Generated by Gradle"/>
<sha256 value="051aceb8bbcc62d0f5b2b8ac72c53767f9c59bfbd050151e65bef6f51c8ed9c9" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.commons" name="commons-csv" version="1.8"> <component group="org.apache.commons" name="commons-csv" version="1.8">
<artifact name="commons-csv-1.8.jar"> <artifact name="commons-csv-1.8.jar">
<md5 value="9019d99d6072f48fd9b1f909f0c45d24" origin="Generated by Gradle"/> <md5 value="9019d99d6072f48fd9b1f909f0c45d24" origin="Generated by Gradle"/>
@@ -2615,6 +2627,12 @@
<sha256 value="d919d904486c037f8d193412da0c92e22a9fa24230b9d67a57855c5c31c7e94e" origin="Generated by Gradle"/> <sha256 value="d919d904486c037f8d193412da0c92e22a9fa24230b9d67a57855c5c31c7e94e" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="org.apache.commons" name="commons-lang3" version="3.14.0">
<artifact name="commons-lang3-3.14.0.jar">
<md5 value="4e5c3f5e6b0b965ef241d7d72ac8971f" origin="Generated by Gradle"/>
<sha256 value="7b96bf3ee68949abb5bc465559ac270e0551596fa34523fddf890ec418dde13c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.commons" name="commons-lang3" version="3.8.1"> <component group="org.apache.commons" name="commons-lang3" version="3.8.1">
<artifact name="commons-lang3-3.8.1.jar"> <artifact name="commons-lang3-3.8.1.jar">
<md5 value="540b1256d887a6993ecbef23371a3302" origin="Generated by Gradle"/> <md5 value="540b1256d887a6993ecbef23371a3302" origin="Generated by Gradle"/>
@@ -69,7 +69,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
) )
@GradleTest @GradleTest
fun testMppWithMavenPublish(gradleVersion: GradleVersion) { fun testMppWithMavenPublish(gradleVersion: GradleVersion) {
project("new-mpp-lib-and-app/sample-lib", gradleVersion) { // with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) {
val publishedTargets = listOf("kotlinMultiplatform", "jvm6", "nodeJs", "linux64", "mingw64") val publishedTargets = listOf("kotlinMultiplatform", "jvm6", "nodeJs", "linux64", "mingw64")
testConfigurationCacheOf( testConfigurationCacheOf(
*(publishedTargets.map { ":publish${it.replaceFirstChar { it.uppercaseChar() }}PublicationToMavenRepository" } *(publishedTargets.map { ":publish${it.replaceFirstChar { it.uppercaseChar() }}PublicationToMavenRepository" }
@@ -87,7 +89,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
) )
@GradleTest @GradleTest
fun testAllMetadataJarWithConfigurationCache(gradleVersion: GradleVersion) { fun testAllMetadataJarWithConfigurationCache(gradleVersion: GradleVersion) {
project("new-mpp-lib-and-app/sample-lib", gradleVersion) { // with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) {
testConfigurationCacheOf(":allMetadataJar") testConfigurationCacheOf(":allMetadataJar")
} }
} }
@@ -100,7 +104,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
) )
@GradleTest @GradleTest
fun testCommonizer(gradleVersion: GradleVersion) { fun testCommonizer(gradleVersion: GradleVersion) {
project("native-configuration-cache", gradleVersion) { // with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
build(":cleanNativeDistributionCommonization") build(":cleanNativeDistributionCommonization")
build(":lib:compileCommonMainKotlinMetadata") { build(":lib:compileCommonMainKotlinMetadata") {
@@ -129,7 +135,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
) )
@GradleTest @GradleTest
fun testCInteropCommonizer(gradleVersion: GradleVersion) { fun testCInteropCommonizer(gradleVersion: GradleVersion) {
project("native-configuration-cache", gradleVersion) { // with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
testConfigurationCacheOf(":lib:commonizeCInterop") testConfigurationCacheOf(":lib:commonizeCInterop")
} }
} }
@@ -278,8 +286,19 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() {
} }
abstract class AbstractConfigurationCacheIT : KGPBaseTest() { abstract class AbstractConfigurationCacheIT : KGPBaseTest() {
override val defaultBuildOptions =
super.defaultBuildOptions.copy(configurationCache = true) @TempDir
lateinit var konanDataTempDir: Path
override val defaultBuildOptions
get() = super.defaultBuildOptions.copy(
configurationCache = true,
konanDataDir = konanDataTempDir,
nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
// set the KGP's default Kotlin Native version, because in CI we don't have K/N versions in maven repo for each build
version = null
)
)
protected fun TestProject.testConfigurationCacheOf( protected fun TestProject.testConfigurationCacheOf(
vararg taskNames: String, vararg taskNames: String,
@@ -294,4 +313,14 @@ abstract class AbstractConfigurationCacheIT : KGPBaseTest() {
buildOptions = buildOptions, buildOptions = buildOptions,
) )
} }
protected fun buildOptionsToAvoidKT66423(gradleVersion: GradleVersion) =
if (gradleVersion == GradleVersion.version(TestVersions.Gradle.G_8_6)) {
defaultBuildOptions.copy(
konanDataDir = konanDir,
nativeOptions = super.defaultBuildOptions.nativeOptions.copy(
version = System.getProperty("kotlinNativeVersion")
)
)
} else defaultBuildOptions
} }
@@ -54,7 +54,9 @@ class MacosCapableConfigurationCacheIT : AbstractConfigurationCacheIT() {
) )
} }
project("native-configuration-cache", gradleVersion) { // with Configuration Cache we currently have such problem KT-66423
val buildOptions = buildOptionsToAvoidKT66423(gradleVersion)
project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) {
testConfigurationCacheOf( testConfigurationCacheOf(
"build", "build",
executedTaskNames = expectedTasks, executedTaskNames = expectedTasks,
@@ -103,6 +103,9 @@ dependencies {
exclude(group = "*") exclude(group = "*")
} }
commonCompileOnly("org.apache.commons:commons-compress:1.26.0")
embedded("org.apache.commons:commons-compress:1.26.0")
if (!kotlinBuildProperties.isInJpsBuildIdeaSync) { if (!kotlinBuildProperties.isInJpsBuildIdeaSync) {
// Adding workaround KT-57317 for Gradle versions where Kotlin runtime <1.8.0 // Adding workaround KT-57317 for Gradle versions where Kotlin runtime <1.8.0
"mainEmbedded"(project(":kotlin-build-tools-enum-compat")) "mainEmbedded"(project(":kotlin-build-tools-enum-compat"))
@@ -5,22 +5,37 @@
package org.jetbrains.kotlin.gradle.targets.native.toolchain package org.jetbrains.kotlin.gradle.targets.native.toolchain
import org.apache.commons.compress.archivers.tar.TarArchiveEntry
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.api.Task import org.gradle.api.Task
import org.gradle.api.file.ArchiveOperations
import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.FileSystemOperations import org.gradle.api.file.FileSystemOperations
import org.gradle.api.logging.Logger
import org.gradle.api.provider.Property import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal import org.gradle.api.tasks.Internal
import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionCommonizerLock import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionCommonizerLock
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider
import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator
import org.jetbrains.kotlin.gradle.tasks.withType import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.loadConfigurables
import org.jetbrains.kotlin.konan.util.ArchiveExtractor
import org.jetbrains.kotlin.konan.util.ArchiveType
import java.io.BufferedInputStream
import java.io.File import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import java.nio.file.attribute.PosixFilePermission
import java.util.zip.GZIPInputStream
import javax.inject.Inject import javax.inject.Inject
private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan" private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan"
@@ -38,6 +53,9 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
@get:Inject @get:Inject
abstract val fso: FileSystemOperations abstract val fso: FileSystemOperations
@get:Inject
abstract val archiveOperations: ArchiveOperations
private var canBeReinstalled: Boolean = true // we can reinstall a k/n bundle once during the build private var canBeReinstalled: Boolean = true // we can reinstall a k/n bundle once during the build
companion object { companion object {
@@ -55,9 +73,20 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
} }
} }
/**
* This function downloads and installs a Kotlin Native bundle if needed
* and then prepares its platform libraries if needed.
*
* @param project The Gradle project object.
* @param kotlinNativeBundleConfiguration Gradle configuration for Kotlin Native Bundle
* @param kotlinNativeVersion The version of Kotlin/Native to install
* @param bundleDir The directory to store the Kotlin/Native bundle.
* @param reinstallFlag A flag indicating whether to reinstall the bundle.
* @param konanTargets The set of KonanTarget objects representing the targets for the Kotlin/Native bundle.
*/
internal fun prepareKotlinNativeBundle( internal fun prepareKotlinNativeBundle(
project: Project, project: Project,
kotlinNativeCompilerConfiguration: ConfigurableFileCollection, kotlinNativeBundleConfiguration: ConfigurableFileCollection,
kotlinNativeVersion: String, kotlinNativeVersion: String,
bundleDir: File, bundleDir: File,
reinstallFlag: Boolean, reinstallFlag: Boolean,
@@ -73,7 +102,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
if (!bundleDir.resolve(KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE).exists()) { if (!bundleDir.resolve(KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE).exists()) {
val gradleCachesKotlinNativeDir = val gradleCachesKotlinNativeDir =
resolveKotlinNativeConfiguration(kotlinNativeVersion, kotlinNativeCompilerConfiguration) resolveKotlinNativeConfiguration(kotlinNativeVersion, kotlinNativeBundleConfiguration)
project.logger.info("Moving Kotlin/Native bundle from tmp directory $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}") project.logger.info("Moving Kotlin/Native bundle from tmp directory $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
fso.copy { fso.copy {
@@ -84,7 +113,37 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
} }
} }
project.setupKotlinNativeDependencies(konanTargets) project.setupKotlinNativePlatformLibraries(konanTargets)
}
/**
* Downloads native dependencies for Kotlin Native based on the provided configuration.
* @return A set of required dependencies that were downloaded.
*/
internal fun downloadNativeDependencies(
bundleDir: File,
konanDataDir: String?,
konanTargets: Set<KonanTarget>,
logger: Logger,
): Set<String> {
val requiredDependencies = mutableSetOf<String>()
val distribution = Distribution(bundleDir.absolutePath, konanDataDir = konanDataDir)
konanTargets.forEach { konanTarget ->
if (konanTarget.enabledOnCurrentHost) {
val konanPropertiesLoader = loadConfigurables(
konanTarget,
distribution.properties,
distribution.dependenciesDir,
progressCallback = { url, currentBytes, totalBytes ->
logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ")
}
) as KonanPropertiesLoader
requiredDependencies.addAll(konanPropertiesLoader.dependencies)
konanPropertiesLoader.downloadDependencies(DependencyExtractor())
}
}
return requiredDependencies
} }
private fun removeBundleIfNeeded( private fun removeBundleIfNeeded(
@@ -115,7 +174,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
return gradleCachesKotlinNativeDir return gradleCachesKotlinNativeDir
} }
private fun Project.setupKotlinNativeDependencies(konanTargets: Set<KonanTarget>) { private fun Project.setupKotlinNativePlatformLibraries(konanTargets: Set<KonanTarget>) {
val distributionType = NativeDistributionTypeProvider(this).getDistributionType() val distributionType = NativeDistributionTypeProvider(this).getDistributionType()
if (distributionType.mustGeneratePlatformLibs) { if (distributionType.mustGeneratePlatformLibs) {
konanTargets.forEach { konanTarget -> konanTargets.forEach { konanTarget ->
@@ -123,4 +182,66 @@ internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServi
} }
} }
} }
private inner class DependencyExtractor : ArchiveExtractor {
override fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) {
when (archiveType) {
ArchiveType.ZIP -> archiveOperations.zipTree(archive)
ArchiveType.TAR_GZ -> unzipTarGz(archive, targetDirectory)
else -> error("Unsupported format for unzipping $archive")
}
}
private fun unzipTarGz(archive: File, targetDir: File) {
GZIPInputStream(BufferedInputStream(archive.inputStream())).use { gzipInputStream ->
TarArchiveInputStream(gzipInputStream).use { tarInputStream ->
generateSequence {
tarInputStream.nextEntry
}.forEach { entry: TarArchiveEntry ->
val outputFile = File("$targetDir/${entry.name}")
if (entry.isDirectory) {
outputFile.mkdirs()
} else {
if (entry.isSymbolicLink) {
Files.createSymbolicLink(outputFile.toPath(), Paths.get(entry.linkName))
} else {
outputFile.outputStream().use {
tarInputStream.copyTo(it)
}
Files.setPosixFilePermissions(outputFile.toPath(), getPosixFilePermissions(entry.mode))
}
}
}
}
}
}
private fun getPosixFilePermissions(mode: Int): Set<PosixFilePermission> {
val permissions: MutableSet<PosixFilePermission> = mutableSetOf()
// adding owner permissions
permissions.addPermission(mode, 0b100_000_000, PosixFilePermission.OWNER_READ)
permissions.addPermission(mode, 0b010_000_000, PosixFilePermission.OWNER_WRITE)
permissions.addPermission(mode, 0b001_000_000, PosixFilePermission.OWNER_EXECUTE)
// adding group permissions
permissions.addPermission(mode, 0b000_100_000, PosixFilePermission.GROUP_READ)
permissions.addPermission(mode, 0b000_010_000, PosixFilePermission.GROUP_WRITE)
permissions.addPermission(mode, 0b000_001_000, PosixFilePermission.GROUP_EXECUTE)
// adding other permissions
permissions.addPermission(mode, 0b000_000_100, PosixFilePermission.OTHERS_READ)
permissions.addPermission(mode, 0b000_000_010, PosixFilePermission.OTHERS_WRITE)
permissions.addPermission(mode, 0b000_000_001, PosixFilePermission.OTHERS_EXECUTE)
return permissions
}
private fun MutableSet<PosixFilePermission>.addPermission(mode: Int, permissionBitMask: Int, permission: PosixFilePermission) {
if ((mode and permissionBitMask) > 0) {
add(permission)
}
}
}
} }
@@ -17,14 +17,10 @@ import org.jetbrains.kotlin.compilerRunner.konanHome
import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled
import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost
import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
import org.jetbrains.kotlin.gradle.utils.filesProvider import org.jetbrains.kotlin.gradle.utils.filesProvider
import org.jetbrains.kotlin.gradle.utils.property import org.jetbrains.kotlin.gradle.utils.property
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.loadConfigurables
/** /**
* This is a nested provider for all native tasks * This is a nested provider for all native tasks
@@ -74,26 +70,17 @@ internal class KotlinNativeProvider(
val kotlinNativeDependencies: Provider<Set<String>> = val kotlinNativeDependencies: Provider<Set<String>> =
kotlinNativeBundleVersion kotlinNativeBundleVersion
.zip(bundleDirectory) { _, bundleDir -> .zip(bundleDirectory) { _, bundleDir ->
val requiredDependencies = mutableSetOf<String>()
if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) { if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) {
val distribution = Distribution(bundleDir.asFile.absolutePath, konanDataDir = konanDataDir.orNull) kotlinNativeBundleBuildService.get()
konanTargets.forEach { konanTarget -> .downloadNativeDependencies(
if (konanTarget.enabledOnCurrentHost) { bundleDir.asFile,
val konanPropertiesLoader = loadConfigurables( konanDataDir.orNull,
konanTarget, konanTargets,
distribution.properties, project.logger
distribution.dependenciesDir, )
progressCallback = { url, currentBytes, totalBytes -> } else {
project.logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ") emptySet()
}
) as KonanPropertiesLoader
requiredDependencies.addAll(konanPropertiesLoader.dependencies)
konanPropertiesLoader.downloadDependencies()
}
}
} }
requiredDependencies
} }
// Gradle tries to evaluate this val during configuration cache, // Gradle tries to evaluate this val during configuration cache,
@@ -19,9 +19,7 @@ package org.jetbrains.kotlin.konan.properties
import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.Configurables import org.jetbrains.kotlin.konan.target.Configurables
import org.jetbrains.kotlin.konan.target.HostManager import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.util.ArchiveType import org.jetbrains.kotlin.konan.util.*
import org.jetbrains.kotlin.konan.util.DependencyProcessor
import org.jetbrains.kotlin.konan.util.ProgressCallback
import java.io.File import java.io.File
interface TargetableExternalStorage { interface TargetableExternalStorage {
@@ -76,6 +74,10 @@ abstract class KonanPropertiesLoader(
dependencyProcessor!!.run() dependencyProcessor!!.run()
} }
fun downloadDependencies(archiveExtractor: ArchiveExtractor) {
dependencyProcessor!!.run(archiveExtractor)
}
// TODO: We may want to add caching to avoid repeated resolve. // TODO: We may want to add caching to avoid repeated resolve.
override fun targetString(key: String): String? = properties.targetString(key, target) override fun targetString(key: String): String? = properties.targetString(key, target)
override fun targetList(key: String): List<String> = properties.targetList(key, target) override fun targetList(key: String): List<String> = properties.targetList(key, target)
@@ -0,0 +1,23 @@
/*
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.util
import java.io.File
/**
* An interface for extracting archive files.
*/
interface ArchiveExtractor {
/**
* Extracts the contents of the specified archive file to the target directory.
*
* @param archive The archive file to extract.
* @param targetDirectory The directory where the contents of the archive will be extracted to.
* @param archiveType The type of the archive file.
*/
fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType)
}
@@ -33,9 +33,8 @@ enum class ArchiveType(val fileExtension: String) {
} }
} }
class DependencyExtractor( class DependencyExtractor : ArchiveExtractor {
private val archiveType: ArchiveType
) {
private fun extractTarGz(tarGz: File, targetDirectory: File) { private fun extractTarGz(tarGz: File, targetDirectory: File) {
val tarProcess = ProcessBuilder().apply { val tarProcess = ProcessBuilder().apply {
command("tar", "-xzf", tarGz.canonicalPath) command("tar", "-xzf", tarGz.canonicalPath)
@@ -47,18 +46,19 @@ class DependencyExtractor(
finished && tarProcess.exitValue() != 0 -> finished && tarProcess.exitValue() != 0 ->
throw RuntimeException( throw RuntimeException(
"Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" + "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
"Tar exit code: ${tarProcess.exitValue()}." "Tar exit code: ${tarProcess.exitValue()}."
) )
!finished -> { !finished -> {
tarProcess.destroy() tarProcess.destroy()
throw RuntimeException( throw RuntimeException(
"Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" + "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
"Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec.") "Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec."
)
} }
} }
} }
fun extract(archive: File, targetDirectory: File) { override fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) {
when (archiveType) { when (archiveType) {
ArchiveType.ZIP -> archive.toPath().unzipTo(targetDirectory.toPath()) ArchiveType.ZIP -> archive.toPath().unzipTo(targetDirectory.toPath())
ArchiveType.TAR_GZ -> extractTarGz(archive, targetDirectory) ArchiveType.TAR_GZ -> extractTarGz(archive, targetDirectory)
@@ -109,7 +109,6 @@ class DependencyProcessor(
private var isInfoShown = false private var isInfoShown = false
private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback) private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback)
private val extractor = DependencyExtractor(archiveType)
constructor(dependenciesRoot: File, constructor(dependenciesRoot: File,
properties: KonanPropertiesLoader, properties: KonanPropertiesLoader,
@@ -172,7 +171,7 @@ class DependencyProcessor(
} }
} }
private fun downloadDependency(dependency: String, baseUrl: String) { private fun downloadDependency(dependency: String, baseUrl: String, archiveExtractor: ArchiveExtractor) {
val depDir = File(dependenciesDirectory, dependency) val depDir = File(dependenciesDirectory, dependency)
val depName = depDir.name val depName = depDir.name
@@ -211,7 +210,7 @@ class DependencyProcessor(
downloader.download(url, archive) downloader.download(url, archive)
} }
println("Extracting dependency: $archive into $dependenciesDirectory") println("Extracting dependency: $archive into $dependenciesDirectory")
extractor.extract(archive, dependenciesDirectory) archiveExtractor.extract(archive, dependenciesDirectory, archiveType)
if (deleteArchives) { if (deleteArchives) {
archive.delete() archive.delete()
} }
@@ -275,7 +274,7 @@ class DependencyProcessor(
} }
} }
fun run() { fun run(archiveExtractor: ArchiveExtractor = DependencyExtractor()) {
// We need a lock that can be shared between different classloaders (KT-39781). // We need a lock that can be shared between different classloaders (KT-39781).
// TODO: Rework dependencies downloading to avoid storing the lock in the system properties. // TODO: Rework dependencies downloading to avoid storing the lock in the system properties.
val lock = System.getProperties().computeIfAbsent("kotlin.native.dependencies.lock") { val lock = System.getProperties().computeIfAbsent("kotlin.native.dependencies.lock") {
@@ -300,7 +299,7 @@ class DependencyProcessor(
DependencySource.Remote.Internal -> InternalServer.url DependencySource.Remote.Internal -> InternalServer.url
} }
// TODO: consider using different caches for different remotes. // TODO: consider using different caches for different remotes.
downloadDependency(dependency, baseUrl) downloadDependency(dependency, baseUrl, archiveExtractor)
} }
} }
} }