diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 4db3977f789..bd6da337ae5 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1501,6 +1501,12 @@ + + + + + + @@ -2603,6 +2609,12 @@ + + + + + + @@ -2615,6 +2627,12 @@ + + + + + + diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt index 97f50c8f7d1..e95b0900630 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt @@ -69,7 +69,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() { ) @GradleTest fun testMppWithMavenPublish(gradleVersion: GradleVersion) { - project("new-mpp-lib-and-app/sample-lib", gradleVersion) { + // with Configuration Cache we currently have such problem KT-66423 + val buildOptions = buildOptionsToAvoidKT66423(gradleVersion) + project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) { val publishedTargets = listOf("kotlinMultiplatform", "jvm6", "nodeJs", "linux64", "mingw64") testConfigurationCacheOf( *(publishedTargets.map { ":publish${it.replaceFirstChar { it.uppercaseChar() }}PublicationToMavenRepository" } @@ -87,7 +89,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() { ) @GradleTest fun testAllMetadataJarWithConfigurationCache(gradleVersion: GradleVersion) { - project("new-mpp-lib-and-app/sample-lib", gradleVersion) { + // with Configuration Cache we currently have such problem KT-66423 + val buildOptions = buildOptionsToAvoidKT66423(gradleVersion) + project("new-mpp-lib-and-app/sample-lib", gradleVersion, buildOptions = buildOptions) { testConfigurationCacheOf(":allMetadataJar") } } @@ -100,7 +104,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() { ) @GradleTest fun testCommonizer(gradleVersion: GradleVersion) { - project("native-configuration-cache", gradleVersion) { + // with Configuration Cache we currently have such problem KT-66423 + val buildOptions = buildOptionsToAvoidKT66423(gradleVersion) + project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) { build(":cleanNativeDistributionCommonization") build(":lib:compileCommonMainKotlinMetadata") { @@ -129,7 +135,9 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() { ) @GradleTest fun testCInteropCommonizer(gradleVersion: GradleVersion) { - project("native-configuration-cache", gradleVersion) { + // with Configuration Cache we currently have such problem KT-66423 + val buildOptions = buildOptionsToAvoidKT66423(gradleVersion) + project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) { testConfigurationCacheOf(":lib:commonizeCInterop") } } @@ -278,8 +286,19 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() { } abstract class AbstractConfigurationCacheIT : KGPBaseTest() { - override val defaultBuildOptions = - super.defaultBuildOptions.copy(configurationCache = true) + + @TempDir + lateinit var konanDataTempDir: Path + + override val defaultBuildOptions + get() = super.defaultBuildOptions.copy( + configurationCache = true, + konanDataDir = konanDataTempDir, + nativeOptions = super.defaultBuildOptions.nativeOptions.copy( + // set the KGP's default Kotlin Native version, because in CI we don't have K/N versions in maven repo for each build + version = null + ) + ) protected fun TestProject.testConfigurationCacheOf( vararg taskNames: String, @@ -294,4 +313,14 @@ abstract class AbstractConfigurationCacheIT : KGPBaseTest() { buildOptions = buildOptions, ) } + + protected fun buildOptionsToAvoidKT66423(gradleVersion: GradleVersion) = + if (gradleVersion == GradleVersion.version(TestVersions.Gradle.G_8_6)) { + defaultBuildOptions.copy( + konanDataDir = konanDir, + nativeOptions = super.defaultBuildOptions.nativeOptions.copy( + version = System.getProperty("kotlinNativeVersion") + ) + ) + } else defaultBuildOptions } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/MacosCapableConfigurationCacheIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/MacosCapableConfigurationCacheIT.kt index e6db5789f51..eb3c22acdea 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/MacosCapableConfigurationCacheIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/MacosCapableConfigurationCacheIT.kt @@ -54,7 +54,9 @@ class MacosCapableConfigurationCacheIT : AbstractConfigurationCacheIT() { ) } - project("native-configuration-cache", gradleVersion) { + // with Configuration Cache we currently have such problem KT-66423 + val buildOptions = buildOptionsToAvoidKT66423(gradleVersion) + project("native-configuration-cache", gradleVersion, buildOptions = buildOptions) { testConfigurationCacheOf( "build", executedTaskNames = expectedTasks, diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index d3867e8d9a3..ea8e92c4940 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -103,6 +103,9 @@ dependencies { exclude(group = "*") } + commonCompileOnly("org.apache.commons:commons-compress:1.26.0") + embedded("org.apache.commons:commons-compress:1.26.0") + if (!kotlinBuildProperties.isInJpsBuildIdeaSync) { // Adding workaround KT-57317 for Gradle versions where Kotlin runtime <1.8.0 "mainEmbedded"(project(":kotlin-build-tools-enum-compat")) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeBundleBuildService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeBundleBuildService.kt index 75eaf92a4bf..8af3771d6d1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeBundleBuildService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeBundleBuildService.kt @@ -5,22 +5,37 @@ package org.jetbrains.kotlin.gradle.targets.native.toolchain +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.file.ArchiveOperations import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.FileSystemOperations +import org.gradle.api.logging.Logger import org.gradle.api.provider.Property import org.gradle.api.provider.Provider import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters import org.gradle.api.tasks.Internal +import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionCommonizerLock import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator import org.jetbrains.kotlin.gradle.tasks.withType import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject +import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader +import org.jetbrains.kotlin.konan.target.Distribution import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.loadConfigurables +import org.jetbrains.kotlin.konan.util.ArchiveExtractor +import org.jetbrains.kotlin.konan.util.ArchiveType +import java.io.BufferedInputStream import java.io.File +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.attribute.PosixFilePermission +import java.util.zip.GZIPInputStream import javax.inject.Inject private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan" @@ -38,6 +53,9 @@ internal abstract class KotlinNativeBundleBuildService : BuildService, + logger: Logger, + ): Set { + val requiredDependencies = mutableSetOf() + val distribution = Distribution(bundleDir.absolutePath, konanDataDir = konanDataDir) + konanTargets.forEach { konanTarget -> + if (konanTarget.enabledOnCurrentHost) { + val konanPropertiesLoader = loadConfigurables( + konanTarget, + distribution.properties, + distribution.dependenciesDir, + progressCallback = { url, currentBytes, totalBytes -> + logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ") + } + ) as KonanPropertiesLoader + + requiredDependencies.addAll(konanPropertiesLoader.dependencies) + konanPropertiesLoader.downloadDependencies(DependencyExtractor()) + } + } + return requiredDependencies } private fun removeBundleIfNeeded( @@ -115,7 +174,7 @@ internal abstract class KotlinNativeBundleBuildService : BuildService) { + private fun Project.setupKotlinNativePlatformLibraries(konanTargets: Set) { val distributionType = NativeDistributionTypeProvider(this).getDistributionType() if (distributionType.mustGeneratePlatformLibs) { konanTargets.forEach { konanTarget -> @@ -123,4 +182,66 @@ internal abstract class KotlinNativeBundleBuildService : BuildService archiveOperations.zipTree(archive) + ArchiveType.TAR_GZ -> unzipTarGz(archive, targetDirectory) + else -> error("Unsupported format for unzipping $archive") + } + } + + private fun unzipTarGz(archive: File, targetDir: File) { + GZIPInputStream(BufferedInputStream(archive.inputStream())).use { gzipInputStream -> + TarArchiveInputStream(gzipInputStream).use { tarInputStream -> + generateSequence { + tarInputStream.nextEntry + }.forEach { entry: TarArchiveEntry -> + val outputFile = File("$targetDir/${entry.name}") + if (entry.isDirectory) { + outputFile.mkdirs() + } else { + if (entry.isSymbolicLink) { + Files.createSymbolicLink(outputFile.toPath(), Paths.get(entry.linkName)) + } else { + outputFile.outputStream().use { + tarInputStream.copyTo(it) + } + Files.setPosixFilePermissions(outputFile.toPath(), getPosixFilePermissions(entry.mode)) + } + } + } + } + } + } + + private fun getPosixFilePermissions(mode: Int): Set { + val permissions: MutableSet = mutableSetOf() + + // adding owner permissions + permissions.addPermission(mode, 0b100_000_000, PosixFilePermission.OWNER_READ) + permissions.addPermission(mode, 0b010_000_000, PosixFilePermission.OWNER_WRITE) + permissions.addPermission(mode, 0b001_000_000, PosixFilePermission.OWNER_EXECUTE) + + // adding group permissions + permissions.addPermission(mode, 0b000_100_000, PosixFilePermission.GROUP_READ) + permissions.addPermission(mode, 0b000_010_000, PosixFilePermission.GROUP_WRITE) + permissions.addPermission(mode, 0b000_001_000, PosixFilePermission.GROUP_EXECUTE) + + // adding other permissions + permissions.addPermission(mode, 0b000_000_100, PosixFilePermission.OTHERS_READ) + permissions.addPermission(mode, 0b000_000_010, PosixFilePermission.OTHERS_WRITE) + permissions.addPermission(mode, 0b000_000_001, PosixFilePermission.OTHERS_EXECUTE) + + return permissions + } + + private fun MutableSet.addPermission(mode: Int, permissionBitMask: Int, permission: PosixFilePermission) { + if ((mode and permissionBitMask) > 0) { + add(permission) + } + } + } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt index 41bbcaabe1f..32b2d2dd375 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/toolchain/KotlinNativeProvider.kt @@ -17,14 +17,10 @@ import org.jetbrains.kotlin.compilerRunner.konanHome import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider -import org.jetbrains.kotlin.gradle.plugin.mpp.enabledOnCurrentHost import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader import org.jetbrains.kotlin.gradle.utils.filesProvider import org.jetbrains.kotlin.gradle.utils.property -import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader -import org.jetbrains.kotlin.konan.target.Distribution import org.jetbrains.kotlin.konan.target.KonanTarget -import org.jetbrains.kotlin.konan.target.loadConfigurables /** * This is a nested provider for all native tasks @@ -74,26 +70,17 @@ internal class KotlinNativeProvider( val kotlinNativeDependencies: Provider> = kotlinNativeBundleVersion .zip(bundleDirectory) { _, bundleDir -> - val requiredDependencies = mutableSetOf() if (project.kotlinNativeToolchainEnabled && enableDependenciesDownloading) { - val distribution = Distribution(bundleDir.asFile.absolutePath, konanDataDir = konanDataDir.orNull) - konanTargets.forEach { konanTarget -> - if (konanTarget.enabledOnCurrentHost) { - val konanPropertiesLoader = loadConfigurables( - konanTarget, - distribution.properties, - distribution.dependenciesDir, - progressCallback = { url, currentBytes, totalBytes -> - project.logger.info("Downloading dependency for Kotlin Native: $url (${currentBytes}/${totalBytes}). ") - } - ) as KonanPropertiesLoader - - requiredDependencies.addAll(konanPropertiesLoader.dependencies) - konanPropertiesLoader.downloadDependencies() - } - } + kotlinNativeBundleBuildService.get() + .downloadNativeDependencies( + bundleDir.asFile, + konanDataDir.orNull, + konanTargets, + project.logger + ) + } else { + emptySet() } - requiredDependencies } // Gradle tries to evaluate this val during configuration cache, diff --git a/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt b/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt index 492b85623fb..606e0d088d5 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/target/KonanProperties.kt @@ -19,9 +19,7 @@ package org.jetbrains.kotlin.konan.properties import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.Configurables import org.jetbrains.kotlin.konan.target.HostManager -import org.jetbrains.kotlin.konan.util.ArchiveType -import org.jetbrains.kotlin.konan.util.DependencyProcessor -import org.jetbrains.kotlin.konan.util.ProgressCallback +import org.jetbrains.kotlin.konan.util.* import java.io.File interface TargetableExternalStorage { @@ -76,6 +74,10 @@ abstract class KonanPropertiesLoader( dependencyProcessor!!.run() } + fun downloadDependencies(archiveExtractor: ArchiveExtractor) { + dependencyProcessor!!.run(archiveExtractor) + } + // TODO: We may want to add caching to avoid repeated resolve. override fun targetString(key: String): String? = properties.targetString(key, target) override fun targetList(key: String): List = properties.targetList(key, target) diff --git a/native/utils/src/org/jetbrains/kotlin/konan/util/ArchiveExtractor.kt b/native/utils/src/org/jetbrains/kotlin/konan/util/ArchiveExtractor.kt new file mode 100644 index 00000000000..0eea5825527 --- /dev/null +++ b/native/utils/src/org/jetbrains/kotlin/konan/util/ArchiveExtractor.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.util + +import java.io.File + +/** + * An interface for extracting archive files. + */ +interface ArchiveExtractor { + + /** + * Extracts the contents of the specified archive file to the target directory. + * + * @param archive The archive file to extract. + * @param targetDirectory The directory where the contents of the archive will be extracted to. + * @param archiveType The type of the archive file. + */ + fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) +} \ No newline at end of file diff --git a/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt b/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt index 105becd6e7f..b29d0f23725 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt @@ -33,9 +33,8 @@ enum class ArchiveType(val fileExtension: String) { } } -class DependencyExtractor( - private val archiveType: ArchiveType -) { +class DependencyExtractor : ArchiveExtractor { + private fun extractTarGz(tarGz: File, targetDirectory: File) { val tarProcess = ProcessBuilder().apply { command("tar", "-xzf", tarGz.canonicalPath) @@ -47,18 +46,19 @@ class DependencyExtractor( finished && tarProcess.exitValue() != 0 -> throw RuntimeException( "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" + - "Tar exit code: ${tarProcess.exitValue()}." + "Tar exit code: ${tarProcess.exitValue()}." ) !finished -> { tarProcess.destroy() throw RuntimeException( "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" + - "Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec.") + "Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec." + ) } } } - fun extract(archive: File, targetDirectory: File) { + override fun extract(archive: File, targetDirectory: File, archiveType: ArchiveType) { when (archiveType) { ArchiveType.ZIP -> archive.toPath().unzipTo(targetDirectory.toPath()) ArchiveType.TAR_GZ -> extractTarGz(archive, targetDirectory) diff --git a/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt b/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt index 45e6bf3f326..6017353bfe1 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt @@ -109,7 +109,6 @@ class DependencyProcessor( private var isInfoShown = false private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback) - private val extractor = DependencyExtractor(archiveType) constructor(dependenciesRoot: File, properties: KonanPropertiesLoader, @@ -172,7 +171,7 @@ class DependencyProcessor( } } - private fun downloadDependency(dependency: String, baseUrl: String) { + private fun downloadDependency(dependency: String, baseUrl: String, archiveExtractor: ArchiveExtractor) { val depDir = File(dependenciesDirectory, dependency) val depName = depDir.name @@ -211,7 +210,7 @@ class DependencyProcessor( downloader.download(url, archive) } println("Extracting dependency: $archive into $dependenciesDirectory") - extractor.extract(archive, dependenciesDirectory) + archiveExtractor.extract(archive, dependenciesDirectory, archiveType) if (deleteArchives) { archive.delete() } @@ -275,7 +274,7 @@ class DependencyProcessor( } } - fun run() { + fun run(archiveExtractor: ArchiveExtractor = DependencyExtractor()) { // We need a lock that can be shared between different classloaders (KT-39781). // TODO: Rework dependencies downloading to avoid storing the lock in the system properties. val lock = System.getProperties().computeIfAbsent("kotlin.native.dependencies.lock") { @@ -300,7 +299,7 @@ class DependencyProcessor( DependencySource.Remote.Internal -> InternalServer.url } // TODO: consider using different caches for different remotes. - downloadDependency(dependency, baseUrl) + downloadDependency(dependency, baseUrl, archiveExtractor) } } }