From 91259569d7ae4f488729a1c243eb829ae7c1871f Mon Sep 17 00:00:00 2001 From: "sebastian.sellmair" Date: Thu, 17 Jun 2021 16:45:45 +0200 Subject: [PATCH] [Gradle] Remove old NativeDistributionCommonizerTask.kt - Remove kotlin.mpp.enableHierarchicalCommonization gradle property: This had to be done, since newer optimizations are not compatible with non-hierarchical commonization ^KT-47301 --- .../kotlin/gradle/plugin/KotlinProperties.kt | 6 - .../native/KotlinNativeTargetPreset.kt | 6 +- .../native/internal/CommonizerTasks.kt | 29 +- ...rchicalKotlinNativePlatformDependencies.kt | 70 ---- .../KotlinNativePlatformDependencies.kt | 390 +++--------------- .../internal/MissingNativeStdlibWarning.kt | 30 ++ .../NativeDistributionCommonizerTask.kt | 298 ------------- .../internals/ExposedForIntegrationTests.kt | 5 +- 8 files changed, 102 insertions(+), 732 deletions(-) delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/MissingNativeStdlibWarning.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt index 8c2bb6a10fb..cd52c1ad844 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt @@ -260,12 +260,6 @@ internal class PropertiesProvider private constructor(private val project: Proje val commonizerLogLevel: String? get() = property("kotlin.mpp.commonizerLogLevel") - /** - * Enables experimental commonization of 'higher level' shared native source sets - */ - val enableHierarchicalCommonization: Boolean - get() = booleanProperty("kotlin.mpp.enableHierarchicalCommonization") ?: true - val enableNativeDistributionCommonizationCache: Boolean get() = booleanProperty("kotlin.mpp.enableNativeDistributionCommonizationCache") ?: true diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt index 08d8c185a43..061eb3579a6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt @@ -16,7 +16,7 @@ import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import org.jetbrains.kotlin.gradle.targets.native.DisabledNativeTargetsReporter import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator -import org.jetbrains.kotlin.gradle.targets.native.internal.setUpKotlinNativePlatformDependencies +import org.jetbrains.kotlin.gradle.targets.native.internal.setupKotlinNativePlatformDependencies import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject import org.jetbrains.kotlin.konan.CompilerVersion @@ -87,7 +87,7 @@ abstract class AbstractKotlinNativeTargetPreset( SingleActionPerProject.run(project, "setUpKotlinNativePlatformDependencies") { project.gradle.projectsEvaluated { - project.setUpKotlinNativePlatformDependencies() + project.setupKotlinNativePlatformDependencies() } } @@ -148,4 +148,4 @@ internal val KonanTarget.enabledOnCurrentHost // so we have to transform it to KotlinVersion first. // Note: this check doesn't take into account the meta version (release, eap, dev). internal fun CompilerVersion.isAtLeast(major: Int, minor: Int, patch: Int): Boolean = - KotlinVersion(this.major, this.minor, this.maintenance).isAtLeast(major, minor, patch) \ No newline at end of file + KotlinVersion(this.major, this.minor, this.maintenance).isAtLeast(major, minor, patch) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt index cfe8b017a8a..b5089170511 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt @@ -17,8 +17,6 @@ import org.jetbrains.kotlin.gradle.tasks.registerTask internal val Project.isCInteropCommonizationEnabled: Boolean get() = PropertiesProvider(this).enableCInteropCommonization -internal val Project.isHierarchicalCommonizationEnabled: Boolean get() = PropertiesProvider(this).enableHierarchicalCommonization - internal val Project.isIntransitiveMetadataConfigurationEnabled: Boolean get() = PropertiesProvider(this).enableIntransitiveMetadataConfiguration @@ -48,12 +46,12 @@ internal val Project.runCommonizerTask: TaskProvider internal val Project.commonizeCInteropTask: TaskProvider? get() { - if (isHierarchicalCommonizationEnabled && isCInteropCommonizationEnabled) { + if (isCInteropCommonizationEnabled) { return locateOrRegisterTask( "commonizeCInterop", invokeWhenRegistered = { commonizeTask.dependsOn(this) - commonizeNativeDistributionHierarchicallyTask?.let(this::dependsOn) + commonizeNativeDistributionTask?.let(this::dependsOn) }, configureTask = { group = "interop" @@ -81,25 +79,12 @@ internal val Project.copyCommonizeCInteropForIdeTask: TaskProvider? +internal val Project.commonizeNativeDistributionTask: TaskProvider? get() { - if (isHierarchicalCommonizationEnabled) return null - return locateOrRegisterTask( - "commonizeNativeDistribution", - invokeWhenRegistered = { commonizeTask.dependsOn(this) }, - configureTask = { - group = "interop" - description = "Invokes the commonizer on the platform libraries provided by the Kotlin/Native distribution" - } - ) - } - -internal val Project.commonizeNativeDistributionHierarchicallyTask: TaskProvider? - get() { - if (!isHierarchicalCommonizationEnabled) return null + if (!isAllowCommonizer()) return null return rootProject.locateOrRegisterTask( "commonizeNativeDistribution", - invokeWhenRegistered = { rootProject.commonizeTask.dependsOn(this); cleanNativeDistributionCommonizationTask }, + invokeWhenRegistered = { rootProject.commonizeTask.dependsOn(this); cleanNativeDistributionCommonizerTask }, configureTask = { group = "interop" description = "Invokes the commonizer on platform libraries provided by the Kotlin/Native distribution" @@ -107,9 +92,9 @@ internal val Project.commonizeNativeDistributionHierarchicallyTask: TaskProvider ) } -internal val Project.cleanNativeDistributionCommonizationTask: TaskProvider? +internal val Project.cleanNativeDistributionCommonizerTask: TaskProvider? get() { - val commonizeNativeDistributionTask = commonizeNativeDistributionHierarchicallyTask ?: return null + val commonizeNativeDistributionTask = commonizeNativeDistributionTask ?: return null return rootProject.locateOrRegisterTask( "cleanNativeDistributionCommonization", configureTask = { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt deleted file mode 100644 index 16488d8b725..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.targets.native.internal - -import org.gradle.api.Project -import org.gradle.api.file.FileCollection -import org.jetbrains.kotlin.commonizer.* -import org.jetbrains.kotlin.compilerRunner.konanHome -import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull -import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet -import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet -import org.jetbrains.kotlin.gradle.targets.metadata.getMetadataCompilationForSourceSet -import org.jetbrains.kotlin.gradle.utils.filesProvider -import java.io.File - -internal fun Project.setUpHierarchicalKotlinNativePlatformDependencies() { - val kotlin = multiplatformExtensionOrNull ?: return - kotlin.sourceSets.forEach { sourceSet -> - val target = getCommonizerTarget(sourceSet) ?: return@forEach - val stdlib = project.filesProvider { listOf(konanDistribution.stdlib) } - addDependencies(sourceSet, getNativeDistributionDependencies(target)) - addDependencies(sourceSet, stdlib) - } -} - -internal fun Project.getNativeDistributionDependencies(target: CommonizerTarget): FileCollection { - val commonizerTask = commonizeNativeDistributionHierarchicallyTask?.get() ?: return project.files() - - return when (target) { - is SharedCommonizerTarget -> commonizerTask.getCommonizedDependenciesFor(target) - is LeafCommonizerTarget -> getOriginalPlatformLibrariesFor(target) - } -} - -private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget): FileCollection { - return project.filesProvider { konanDistribution.platformLibsDir.resolve(target.konanTarget.name).listLibraryFiles().toSet() } -} - -private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection { - val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target) - return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this) -} - -private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedDependenciesFor(target: CommonizerTarget): FileCollection { - return when (target) { - is LeafCommonizerTarget -> project.getOriginalPlatformLibrariesFor(target) - is SharedCommonizerTarget -> getCommonizedPlatformLibrariesFor(target) - } -} - -private fun Project.addDependencies(sourceSet: KotlinSourceSet, libraries: FileCollection) { - getMetadataCompilationForSourceSet(sourceSet)?.let { compilation -> - compilation.compileDependencyFiles += libraries - } - if (sourceSet is DefaultKotlinSourceSet) { - val metadataConfigurationName = - if (project.isIntransitiveMetadataConfigurationEnabled) sourceSet.intransitiveMetadataConfigurationName - else sourceSet.implementationMetadataConfigurationName - dependencies.add(metadataConfigurationName, libraries) - } -} - -private val Project.konanDistribution: KonanDistribution - get() = KonanDistribution(project.file(konanHome)) - -private fun File.listLibraryFiles(): List = listFiles().orEmpty() - .filter { it.isDirectory || it.extension == "klib" } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt index e895afb4086..072d7bec28a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt @@ -7,27 +7,76 @@ package org.jetbrains.kotlin.gradle.targets.native.internal import org.gradle.api.Project import org.gradle.api.file.FileCollection +import org.jetbrains.kotlin.commonizer.* import org.jetbrains.kotlin.compilerRunner.konanHome import org.jetbrains.kotlin.gradle.dsl.multiplatformExtension import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull -import org.jetbrains.kotlin.gradle.plugin.* -import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.KOTLIN_NATIVE_HOME -import org.jetbrains.kotlin.gradle.plugin.mpp.CompilationSourceSetUtil -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeCompilation +import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType +import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet +import org.jetbrains.kotlin.gradle.plugin.compareVersionNumbers +import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion +import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet import org.jetbrains.kotlin.gradle.targets.metadata.getMetadataCompilationForSourceSet import org.jetbrains.kotlin.gradle.targets.metadata.isKotlinGranularMetadataEnabled -import org.jetbrains.kotlin.gradle.targets.native.internal.NativePlatformDependency.* -import org.jetbrains.kotlin.gradle.utils.SingleWarningPerBuild -import org.jetbrains.kotlin.konan.library.* -import org.jetbrains.kotlin.konan.target.KonanTarget -import org.jetbrains.kotlin.utils.addToStdlib.flattenTo +import org.jetbrains.kotlin.gradle.targets.native.internal.MissingNativeStdlibWarning.showMissingNativeStdlibWarning +import org.jetbrains.kotlin.gradle.utils.filesProvider import java.io.File -import java.util.concurrent.Callable + +internal fun Project.setupKotlinNativePlatformDependencies() { + val kotlin = multiplatformExtensionOrNull ?: return + + if (!konanDistribution.stdlib.exists()) { + showMissingNativeStdlibWarning() + } + + kotlin.sourceSets.forEach { sourceSet -> + val target = getCommonizerTarget(sourceSet) ?: return@forEach + addDependencies(sourceSet, getNativeDistributionDependencies(target)) + addDependencies(sourceSet, project.filesProvider { setOf(konanDistribution.stdlib) }) + } +} + +internal fun Project.getNativeDistributionDependencies(target: CommonizerTarget): FileCollection { + return when (target) { + is LeafCommonizerTarget -> getOriginalPlatformLibrariesFor(target) + is SharedCommonizerTarget -> commonizeNativeDistributionTask?.get()?.getCommonizedPlatformLibrariesFor(target) ?: project.files() + } +} + +private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget): FileCollection = project.filesProvider { + konanDistribution.platformLibsDir.resolve(target.konanTarget.name).listLibraryFiles().toSet() +} + +private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection { + val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target) + return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this) +} + +private fun Project.addDependencies(sourceSet: KotlinSourceSet, libraries: FileCollection) { + getMetadataCompilationForSourceSet(sourceSet)?.let { compilation -> + compilation.compileDependencyFiles += libraries + } + if (sourceSet is DefaultKotlinSourceSet) { + val metadataConfigurationName = + if (project.isIntransitiveMetadataConfigurationEnabled) sourceSet.intransitiveMetadataConfigurationName + else sourceSet.implementationMetadataConfigurationName + dependencies.add(metadataConfigurationName, libraries) + } +} + +private val Project.konanDistribution: KonanDistribution + get() = KonanDistribution(project.file(konanHome)) + +private fun File.listLibraryFiles(): List = listFiles().orEmpty() + .filter { it.isDirectory || it.extension == "klib" } + private val Project.isNativeDependencyPropagationEnabled: Boolean get() = (findProperty("kotlin.native.enableDependencyPropagation") as? String)?.toBoolean() ?: true //for reflection call from KotlinCommonizerModelBuilder +// DO NOT REFACTOR THIS FUNCTION! +// TODO SELLMAIR: Resolve fragile reflection call from IDE plugin @JvmOverloads @JvmName("isAllowCommonizer") internal fun Project.isAllowCommonizer( @@ -41,324 +90,3 @@ internal fun Project.isAllowCommonizer( && isKotlinGranularMetadataEnabled && !isNativeDependencyPropagationEnabled // temporary fix: turn on commonizer only when native deps propagation is disabled } - -internal fun Project.setUpKotlinNativePlatformDependencies() { - if (multiplatformExtensionOrNull == null) { - // not a multiplatform project, nothing to set up - return - } - val kotlinVersion = getKotlinPluginVersion() - val allowCommonizer = isAllowCommonizer(kotlinVersion) - if (allowCommonizer && isHierarchicalCommonizationEnabled) { - return setUpHierarchicalKotlinNativePlatformDependencies() - } - - val dependencyResolver = NativePlatformDependencyResolver(this, kotlinVersion) - findSourceSetsToAddDependencies(allowCommonizer).forEach { (sourceSet: KotlinSourceSet, sourceSetDeps: Set) -> - sourceSetDeps.forEach { sourceSetDep: NativePlatformDependency -> - dependencyResolver.addForResolve(sourceSetDep) { resolvedFiles: FileCollection -> - //add commonized klibs to metadata compilation - if (sourceSetDep is CommonizedCommon) { - getMetadataCompilationForSourceSet(sourceSet)?.let { compilation -> - compilation.compileDependencyFiles += resolvedFiles - } - } - dependencies.add(sourceSet.implementationMetadataConfigurationName, dependencies.create(resolvedFiles)) - } - } - } - - dependencyResolver.resolveAll() -} - -private sealed class NativePlatformDependency { - /* Non-commonized target-neutral libraries taken directly from the Kotlin/Native distribution */ - data class OutOfDistributionCommon(val includeEndorsedLibs: Boolean) : NativePlatformDependency() - - /* Non-commonized libraries for the specific target taken directly from the Kotlin/Native distribution */ - data class OutOfDistributionPlatform(val target: KonanTarget) : NativePlatformDependency() - - /* Commonized (common) libraries */ - data class CommonizedCommon(val targets: Set) : NativePlatformDependency() - - /* Commonized libraries for a specific target platform */ - data class CommonizedPlatform(val target: KonanTarget, val common: CommonizedCommon) : NativePlatformDependency() -} - -private class NativePlatformDependencyResolver(val project: Project, val kotlinVersion: String) { - private val distributionDir = File(project.konanHome) - private val distributionLibsDir = distributionDir.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - - private var alreadyResolved = false - private val dependencies = mutableMapOf Unit>>() - - fun addForResolve(dependency: NativePlatformDependency, whenResolved: (FileCollection) -> Unit) { - check(!alreadyResolved) - dependencies.computeIfAbsent(dependency) { mutableListOf() }.add(whenResolved) - } - - fun resolveAll() { - check(!alreadyResolved) - alreadyResolved = true - - project.commonizeNativeDistributionTask?.configure { commonizerTask -> - commonizerTask.targetGroups = dependencies.keys.filterIsInstance().map { it.targets }.toSet() - } - - // then, resolve dependencies one by one - dependencies.forEach { (dependency, actions) -> - val fileCollection = when (dependency) { - is OutOfDistributionCommon -> { - /* stdlib, endorsed libs */ - var hasStdlib = false - - val libs = libsInCommonDir(distributionLibsDir) { dir -> - val isStdlib = dir.endsWith(KONAN_STDLIB_NAME) - hasStdlib = hasStdlib || isStdlib - - return@libsInCommonDir isStdlib || dependency.includeEndorsedLibs - } - - if (!hasStdlib) warnAboutMissingNativeStdlib() - - project.files(libs) - } - - is OutOfDistributionPlatform -> { - /* platform libs for a specific target */ - project.files(libsInPlatformDir(distributionLibsDir, dependency.target)) - } - - is CommonizedCommon -> { - /* commonized platform libs with expect declarations */ - val commonizedLibsDir = project.nativeDistributionCommonizerOutputDirectory(dependency.targets) - project.files(Callable { - libsInCommonDir(commonizedLibsDir) - }).builtBy(project.commonizeNativeDistributionTask) - - } - - is CommonizedPlatform -> { - /* commonized platform libs with actual declarations */ - val commonizedLibsDir = project.nativeDistributionCommonizerOutputDirectory(dependency.common.targets) - project.files(Callable { - libsInPlatformDir(commonizedLibsDir, dependency.target) + libsInCommonDir(commonizedLibsDir) - }).builtBy(project.commonizeNativeDistributionTask) - } - } - - actions.forEach { it(fileCollection) } - } - } - - private fun warnAboutMissingNativeStdlib() { - if (!project.hasProperty("kotlin.native.nostdlib")) { - SingleWarningPerBuild.show( - project, - buildString { - append(NO_NATIVE_STDLIB_WARNING) - if (PropertiesProvider(project).nativeHome != null) - append(NO_NATIVE_STDLIB_PROPERTY_WARNING) - } - ) - } - } - - companion object { - private fun libsInCommonDir(basePath: File, predicate: (File) -> Boolean = { true }) = - basePath.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).listFiles()?.filter { predicate(it) }?.toSet() ?: emptySet() - - private fun libsInPlatformDir(basePath: File, target: KonanTarget) = - basePath.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(target.name).listFiles()?.toSet() ?: emptySet() - } -} - -private fun Project.findSourceSetsToAddDependencies(allowCommonizer: Boolean): Map> { - val sourceSetsToAddDeps = mutableMapOf>() - if (allowCommonizer) { - sourceSetsToAddDeps += findSourceSetsToAddCommonizedPlatformDependencies() - } - - val compilationsBySourceSets = CompilationSourceSetUtil.compilationsBySourceSets(this) - val nativeCompilations = compilationsBySourceSets.values.flattenTo(mutableSetOf()).filterIsInstance() - - nativeCompilations.associate { it.defaultSourceSet to NativeSourceSetDetails(it.konanTarget, it.enableEndorsedLibs) } - .forEach { (defaultSourceSet, details) -> - if (defaultSourceSet !in sourceSetsToAddDeps) { - sourceSetsToAddDeps[defaultSourceSet] = setOf( - OutOfDistributionCommon(details.includeEndorsedLibs), - OutOfDistributionPlatform(details.target) - ) - } - } - - return sourceSetsToAddDeps -} - -private fun Project.findSourceSetsToAddCommonizedPlatformDependencies(): Map> { - val sourceSetsToAddDeps = mutableMapOf>() - - val compilationsBySourceSets = CompilationSourceSetUtil.compilationsBySourceSets(this) - val nativeCompilations = compilationsBySourceSets.values.flattenTo(mutableSetOf()).filterIsInstance() - - nativeCompilations.forEach { nativeCompilation -> - // consider source sets in compilation only one step above the default source set - // TODO: reconsider this restriction - val commonSourceSetCandidates = nativeCompilation.defaultSourceSet.dependsOn - nativeCompilation.defaultSourceSet - - commonSourceSetCandidates.forEach sourceSet@{ sourceSet -> - if (sourceSet.name == KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME - || sourceSet.name == KotlinSourceSet.COMMON_TEST_SOURCE_SET_NAME - ) { - // exclude the most common source sets - return@sourceSet - } - - if (sourceSet in sourceSetsToAddDeps) { - // already processed - return@sourceSet - } - - val leafSourceSets = mutableMapOf() - - for (compilation in compilationsBySourceSets.getValue(sourceSet)) { - if (compilation !is KotlinNativeCompilation) { - // the source set participates in non-native compilation - return@sourceSet - } - - val defaultSourceSet = compilation.defaultSourceSet - if (defaultSourceSet == sourceSet) { - // there is a compilation where the source set is the default source set - return@sourceSet - } - - leafSourceSets[defaultSourceSet] = NativeSourceSetDetails( - target = compilation.konanTarget, - includeEndorsedLibs = compilation.enableEndorsedLibs - ) - } - - val allTargets = leafSourceSets.values.mapTo(mutableSetOf()) { it.target } - if (allTargets.isEmpty()) - return@sourceSet - - val commonizedCommonDep = CommonizedCommon(allTargets) - val includeEndorsedLibsToCommonSourceSet = leafSourceSets.values.all { it.includeEndorsedLibs } - - // save the dependencies for the current common native source set - sourceSetsToAddDeps[sourceSet] = setOf( - OutOfDistributionCommon(includeEndorsedLibsToCommonSourceSet), - commonizedCommonDep - ) - - // now set the dependencies for leaf native source sets (those that dependsOn the current common native source set) - leafSourceSets.forEach { (leafSourceSet, details) -> - val newLeafSourceSetDeps = setOf( - OutOfDistributionCommon(details.includeEndorsedLibs), - CommonizedPlatform(details.target, commonizedCommonDep) - ) - - /* - * It may happen that the leaf source set has already been processed with another common native source set, and - * its dependencies have already been evaluated. Then, it's necessary to merge earlier and newly evaluated dependencies - * to narrow combination of commonized targets. - * - * Why? Consider this example: There is `watchos()` shortcut in Gradle DSL that creates few native targets - * in HMPP project with the corresponding source set hierarchy: - * - * watchosMain [commonized targets: watchosX64, watchosArm32, watchosArm64] - * / \ - * watchosX64Main watchosDeviceMain [commonized targets: watchosArm32, watchosArm64] - * / \ - * watchosArm32Main watchosArm64Main - * - * There are two common native source sets that participate in commonization process: - * - * 1. `watchosMain`. This source set is included into three native compilations for different native targets. - * Thus, it has three commonized targets: watchosX64, watchosArm32 and watchosArm64. - * - * 2. `watchosDeviceMain`. Two native compilations -> two commonized targets: watchosArm32 and watchosArm64. - * - * Let's consider some leaf source set that depends on both of them, for instance, `watchosArm64Main`: - * - * - When `watchosArm64Main` is processed with `watchosMain`, the dependencies are evaluated from `watchosMain` - * viewpoint. This means the following: the leaf source set should get the libraries with `actual` declarations - * produced as a result of commonization of the three targets: watchosX64, watchosArm32 and watchosArm64. - * - * - When `watchosArm64Main` is processed with `watchosDeviceMain`, which is immediate parent according to the hierarchy, - * the dependencies are evaluated from `watchosDeviceMain` viewpoint. Assuming libraries with `actual` declarations - * produced for tho targets: watchosArm32, watchosArm64. - */ - val existingLeafSourceSetDeps = sourceSetsToAddDeps[leafSourceSet] - sourceSetsToAddDeps[leafSourceSet] = existingLeafSourceSetDeps?.let { - mergeLeafSourceSetDeps(leafSourceSet, it, newLeafSourceSetDeps) - } ?: newLeafSourceSetDeps - } - } - } - - return sourceSetsToAddDeps -} - -private fun Project.mergeLeafSourceSetDeps( - leafSourceSet: KotlinSourceSet, - existingDeps: Set, - newDeps: Set -): Set { - - fun merge(dependencies: List): CommonizedPlatform? = when (dependencies.size) { - 1 -> dependencies[0] - 2 -> { - val (first, second) = dependencies - when { - first.target != second.target -> { - // leaf source set is included into several native compilations for different targets - // this is an extremely rare case, though, possible - let's just fallback to no-merge - null - } - first.common.targets.containsAll(second.common.targets) -> second - second.common.targets.containsAll(first.common.targets) -> first - else -> null - } - } - else -> error("Unexpected number of dependencies: $dependencies") - } - - fun merge(dependencies: List): OutOfDistributionCommon = when (dependencies.size) { - 1 -> dependencies[0] - 2 -> { - val (first, second) = dependencies - if (!first.includeEndorsedLibs) first else second - } - else -> error("Unexpected number of dependencies: $dependencies") - } - - val allDeps = existingDeps + newDeps - - val allDepTypes = allDeps.mapTo(HashSet()) { it::class } - check(allDepTypes.size == 2 && OutOfDistributionCommon::class in allDepTypes && CommonizedPlatform::class in allDepTypes) { - // sanity check, such case should not happen normally - "Unexpected combination of dependency types in leaf $leafSourceSet while merging $existingDeps and $newDeps" - } - - val result = HashSet() - result += merge(allDeps.filterIsInstance()) ?: run { - logger.warn("Leaf $leafSourceSet is probably included into several native compilations with different targets.") - return existingDeps - } - result += merge(allDeps.filterIsInstance()) - - return result -} - -private class NativeSourceSetDetails( - val target: KonanTarget, - val includeEndorsedLibs: Boolean -) - -internal const val NO_NATIVE_STDLIB_WARNING = - "The Kotlin/Native distribution used in this build does not provide the standard library. " - -internal const val NO_NATIVE_STDLIB_PROPERTY_WARNING = - "Make sure that the '$KOTLIN_NATIVE_HOME' property points to a valid Kotlin/Native distribution." diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/MissingNativeStdlibWarning.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/MissingNativeStdlibWarning.kt new file mode 100644 index 00000000000..5193f799a80 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/MissingNativeStdlibWarning.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.targets.native.internal + +import org.gradle.api.Project +import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider +import org.jetbrains.kotlin.gradle.utils.SingleWarningPerBuild + +internal object MissingNativeStdlibWarning { + const val NO_NATIVE_STDLIB_WARNING = + "The Kotlin/Native distribution used in this build does not provide the standard library. " + + const val NO_NATIVE_STDLIB_PROPERTY_WARNING = + "Make sure that the '${PropertiesProvider.KOTLIN_NATIVE_HOME}' property points to a valid Kotlin/Native distribution." + + fun Project.showMissingNativeStdlibWarning() { + if (!project.hasProperty("kotlin.native.nostdlib")) { + SingleWarningPerBuild.show( + project, buildString { + append(NO_NATIVE_STDLIB_WARNING) + if (PropertiesProvider(project).nativeHome != null) + append(NO_NATIVE_STDLIB_PROPERTY_WARNING) + } + ) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt deleted file mode 100644 index c700f081f33..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.targets.native.internal - -import org.gradle.api.DefaultTask -import org.gradle.api.Project -import org.gradle.api.tasks.* -import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner -import org.jetbrains.kotlin.compilerRunner.konanHome -import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion -import org.jetbrains.kotlin.gradle.targets.native.internal.SuccessMarker.Companion.getSuccessMarker -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR -import org.jetbrains.kotlin.konan.target.KonanTarget -import java.io.File -import java.io.IOException -import java.nio.charset.StandardCharsets -import java.nio.file.* -import java.nio.file.attribute.* -import java.time.* -import java.util.* - - -internal typealias KonanTargetGroup = Set - -internal open class NativeDistributionCommonizerTask : DefaultTask() { - - private val konanHome = project.file(project.konanHome) - - @get:Input - var targetGroups: Set = emptySet() - - @get:PathSensitive(PathSensitivity.ABSOLUTE) - @get:InputDirectory - @Suppress("unused") // Only for up-to-date checker. The directory with the original common libs. - val originalCommonLibrariesDirectory = konanHome - .resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) - - @get:PathSensitive(PathSensitivity.ABSOLUTE) - @get:InputDirectory - @Suppress("unused") // Only for up-to-date checker. The directory with the original platform libs. - val originalPlatformLibrariesDirectory = konanHome - .resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) - - @get:OutputDirectories - val commonizerTargetOutputDirectories - get() = targetGroups.map { targets -> project.nativeDistributionCommonizerOutputDirectory(targets) } - - @get:PathSensitive(PathSensitivity.ABSOLUTE) - @get:InputFiles - @Suppress("unused") // Only for up-to-date checker. - val successMarkers - get() = targetGroups.map { targets -> project.getSuccessMarker(targets).file } - - /* - Ensures that only one CommonizerTask can run at a time. - This is necessary because of the sucess-marker mechansim of this task. - This is a phantom file: No one has the intention to actually create this output file. - However, telling Gradle that all those tasks rely on the same output file will enforce - non-parallel execution. - */ - @get:OutputFile - @Suppress("unused") - val taskMutex: File = project.rootProject.file(".commonizer-phantom-output") - - @TaskAction - fun run() { - // first of all remove directories with unused commonized libraries plus temporary directories with commonized libraries - // that accidentally were not cleaned up before - cleanUp( - baseDirectory = konanHome.resolve(KONAN_DISTRIBUTION_KLIB_DIR).resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR), - excludedDirectories = commonizerTargetOutputDirectories - ) - - val executionEnvironment = createExecutionEnvironment() - - try { - callCommonizerCLI(project, executionEnvironment.commandLineArguments) - executionEnvironment.stagedDirectories.forEach { stagedDirectory -> stagedDirectory.onSuccess() } - executionEnvironment.successMarkers.forEach { successMarker -> successMarker.writeSuccess() } - } catch (e: Throwable) { - executionEnvironment.stagedDirectories.forEach { stagedDirectory -> stagedDirectory.onFailure() } - executionEnvironment.successMarkers.forEach { successMarker -> successMarker.delete() } - throw e - } - } - - private fun createExecutionEnvironment(): CommonizerExecutionEnvironment { - val stagedDirectories = mutableListOf() - val successMarkers = mutableListOf() - val arguments = mutableListOf() - - targetGroups.forEach { targets -> - // no need to commonize, just use the libraries from the distribution - if (targets.size <= 1) return@forEach - val orderedTargetNames = targets.map { it.name }.sorted() - val successMarker = project.getSuccessMarker(targets) - if (successMarker.isSuccess) return@forEach - - val stagedDirectory = TemporaryStagedDirectory( - temporaryDirectoryFile = project.createTempNativeDistributionCommonizerOutputDirectory(targets), - targetDirectoryFile = project.nativeDistributionCommonizerOutputDirectory(targets) - ) - - stagedDirectories += stagedDirectory - successMarkers += successMarker - arguments += "native-dist-commonize" - arguments += "-distribution-path" - arguments += konanHome.absolutePath - arguments += "-output-path" - arguments += stagedDirectory.temporaryDirectoryFile.absolutePath - arguments += "-targets" - arguments += orderedTargetNames.joinToString(separator = ",") - } - - return CommonizerExecutionEnvironment( - commandLineArguments = arguments, - successMarkers = successMarkers, - stagedDirectories = stagedDirectories - ) - } -} - -private class CommonizerExecutionEnvironment( - val commandLineArguments: List, - val successMarkers: List, - val stagedDirectories: List -) - -private class SuccessMarker private constructor(val file: File) { - companion object { - private const val SUCCESS_MARKER = ".commonized" - private const val SUCCESS_MARKER_CONTENT = "1" - - fun Project.getSuccessMarker(targets: KonanTargetGroup): SuccessMarker { - return SuccessMarker(nativeDistributionCommonizerOutputDirectory(targets).resolve(SUCCESS_MARKER)) - } - } - - val isSuccess get() = file.isFile && file.readText() == SUCCESS_MARKER_CONTENT - - fun delete(): Boolean = file.delete() - - fun writeSuccess() { - if (isSuccess) return - if (!file.parentFile.exists()) { - file.parentFile.mkdirs() - } - if (file.isDirectory) { - renameToTempAndDelete(file) - } - file.writeText(SUCCESS_MARKER_CONTENT) - } -} - -private class TemporaryStagedDirectory(val temporaryDirectoryFile: File, private val targetDirectoryFile: File) { - fun onFailure() = renameToTempAndDelete(temporaryDirectoryFile) - fun onSuccess() = renameDirectory(temporaryDirectoryFile, targetDirectoryFile) -} - -internal fun Project.nativeDistributionCommonizerOutputDirectory(targets: KonanTargetGroup): File { - val kotlinVersion = project.getKotlinPluginVersion() - val orderedTargetNames = targets.map { it.name }.sorted() - val discriminator = buildString { - orderedTargetNames.joinTo(this, separator = "-") - append("-") - append(kotlinVersion.toLowerCase().base64) - } - return project.file(konanHome) - .resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR) - .resolve(discriminator) -} - -internal fun Project.createTempNativeDistributionCommonizerOutputDirectory(targets: KonanTargetGroup): File { - val outputDirectory = nativeDistributionCommonizerOutputDirectory(targets) - outputDirectory.parentFile.mkdirs() - return Files.createTempDirectory( - /* dir = */ outputDirectory.parentFile.toPath(), - /* prefix = */ "tmp-new-${outputDirectory.name}" - ).toFile() -} - -internal fun callCommonizerCLI(project: Project, commandLineArguments: List) { - if (commandLineArguments.isEmpty()) return - - KotlinNativeCommonizerToolRunner(project).run(commandLineArguments) -} - -private fun renameDirectory(source: File, destination: File) { - val sourcePath = source.toPath() - val destinationPath = destination.toPath() - - val suppressedExceptions = mutableListOf() - - for (it in 0 until 3) { - try { - renameToTempAndDelete(destination) - Files.move(sourcePath, destinationPath, StandardCopyOption.ATOMIC_MOVE) - return - } catch (e: IOException) { - suppressedExceptions += e - - if (e is AtomicMoveNotSupportedException) { - // new attempts have no more sense - break - } - } - } - - throw IllegalStateException("Failed to rename $source to $destination").apply { suppressedExceptions.forEach(::addSuppressed) } -} - -private fun renameToTempAndDelete(directory: File) { - if (!directory.exists()) return - - val dirToRemove = if (directory.name.startsWith("tmp-")) { - // already temp directory, return as is - directory - } else { - // first, rename the directory to some temp directory - val tempDir = Files.createTempFile( - /* dir = */ directory.parentFile.toPath(), - /* prefix = */ "tmp-old-" + directory.name, - /* suffix = */ null - ) - Files.delete(tempDir) - - Files.move(directory.toPath(), tempDir, StandardCopyOption.ATOMIC_MOVE) - - tempDir.toFile() - } - - dirToRemove.deleteRecursively() -} - -private fun cleanUp(baseDirectory: File, excludedDirectories: List) { - fun File.getAttributes(): BasicFileAttributes? = - try { - Files.readAttributes(toPath(), BasicFileAttributes::class.java) - } catch (_: IOException) { - null - } - - fun FileTime.isSameOrAfter(targetInstant: Instant): Boolean { - val fileInstant = toInstant() - - if (fileInstant.atZone(ZoneOffset.UTC).toLocalDate().year <= 1970) { - // file time represents the epoch (or even a time point before it) - // such instant can't be used for reliable comparison - return false - } - - return fileInstant >= targetInstant - } - - val now = Instant.now() - val oneHourAgo = now.minus(Duration.ofHours(1)) - val oneMonthAgo = now.minus(Duration.ofDays(31)) - - val excludedPaths = excludedDirectories.map { it.absolutePath }.toSet() - - baseDirectory.listFiles() - ?.forEach { file -> - if (file.absolutePath in excludedPaths) return@forEach - - val attributes = file.getAttributes() ?: return@forEach - if (attributes.isDirectory) { - if (file.name.startsWith("tmp-")) { - // temp directories created more than 1 hour ago are stale and should be GCed - if (attributes.creationTime().isSameOrAfter(oneHourAgo)) return@forEach - } else { - // clean up other directories which were not accesses within the last month - if (attributes.lastAccessTime().isSameOrAfter(oneMonthAgo)) return@forEach - } - } /*else { - // clean up everything that is not a directory - }*/ - - try { - renameToTempAndDelete(file) - } catch (_: IOException) { - // do nothing - } - } -} - -private val String.base64 - get() = base64Encoder.encodeToString(toByteArray(StandardCharsets.UTF_8)) - -private val base64Encoder = Base64.getEncoder().withoutPadding() diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internals/ExposedForIntegrationTests.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internals/ExposedForIntegrationTests.kt index ac223b63d86..211de7d65ec 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internals/ExposedForIntegrationTests.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internals/ExposedForIntegrationTests.kt @@ -6,12 +6,13 @@ package org.jetbrains.kotlin.gradle.internals import org.jetbrains.kotlin.gradle.plugin.KOTLIN_12X_MPP_DEPRECATION_WARNING -import org.jetbrains.kotlin.gradle.targets.native.internal.NO_NATIVE_STDLIB_PROPERTY_WARNING -import org.jetbrains.kotlin.gradle.targets.native.internal.NO_NATIVE_STDLIB_WARNING + import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinProjectStructureMetadata import org.jetbrains.kotlin.gradle.plugin.mpp.MULTIPLATFORM_PROJECT_METADATA_JSON_FILE_NAME import org.jetbrains.kotlin.gradle.plugin.mpp.parseKotlinSourceSetMetadataFromJson import org.jetbrains.kotlin.gradle.targets.native.DisabledNativeTargetsReporter +import org.jetbrains.kotlin.gradle.targets.native.internal.MissingNativeStdlibWarning.NO_NATIVE_STDLIB_PROPERTY_WARNING +import org.jetbrains.kotlin.gradle.targets.native.internal.MissingNativeStdlibWarning.NO_NATIVE_STDLIB_WARNING fun parseKotlinSourceSetMetadataFromJson(json: String): KotlinProjectStructureMetadata? = parseKotlinSourceSetMetadataFromJson(json)