From 3cee2ca40907e7d7f870d5d3945f4e61917652dd Mon Sep 17 00:00:00 2001 From: Sebastian Sellmair Date: Wed, 14 Sep 2022 12:26:22 +0200 Subject: [PATCH] [Gradle][MPP] Flatten transformed metadata library directories The main purpose of this commit is to fix KT-48135 and to de-duplicate libraries for the IDE whilst also minimizing zip operations during IDE import. However, with this commit the metadata library directories layout also gets changed for CLI builds. The overall layout is now more shared with cinterops and the IDE. ^KT-48135 Verification Pending --- .../jetbrains/kotlin/gradle/KlibBasedMppIT.kt | 11 +- ...IdeaKpmMetadataBinaryDependencyResolver.kt | 55 ++-- .../gradle/plugin/mpp/CompositeMetadataJar.kt | 97 ------- .../mpp/GetAllCompiledSourceSetMetadata.kt | 47 --- .../mpp/GranularMetadataTransformation.kt | 26 +- .../mpp/KotlinMetadataLibraryDirectories.kt | 24 ++ .../mpp/TransformKotlinGranularMetadata.kt | 19 +- ...adleKpmFragmentGranularMetadataResolver.kt | 22 +- ...nsformKotlinGranularMetadataForFragment.kt | 9 +- .../plugin/mpp/transformMetadataLibraries.kt | 97 +++++++ .../plugin/sources/DefaultKotlinSourceSet.kt | 22 +- .../sources/SourceSetMetadataStorageForIde.kt | 34 --- ...ropMetadataDependencyTransformationTask.kt | 97 +++++-- .../kotlin/gradle/utils/providerApiUtils.kt | 4 + .../kotlin/gradle/CompositeMetadataJarTest.kt | 271 ------------------ 15 files changed, 270 insertions(+), 565 deletions(-) delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/CompositeMetadataJar.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GetAllCompiledSourceSetMetadata.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/SourceSetMetadataStorageForIde.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/CompositeMetadataJarTest.kt diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KlibBasedMppIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KlibBasedMppIT.kt index e82ec433d6f..6ae3c32c834 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KlibBasedMppIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KlibBasedMppIT.kt @@ -10,8 +10,8 @@ import org.jetbrains.kotlin.konan.target.HostManager import java.io.File import java.util.* import java.util.zip.ZipFile -import kotlin.test.Test import kotlin.test.Ignore +import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue @@ -92,10 +92,10 @@ class KlibBasedMppIT : BaseGradleIT() { checkTaskCompileClasspath( "compile${hostSpecificSourceSet.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.getDefault()) else it.toString() }}KotlinMetadata", listOf( - "published-producer-$hostSpecificSourceSet.klib", - "published-producer-commonMain.klib", - "published-dependency-$hostSpecificSourceSet.klib", - "published-dependency-commonMain.klib" + "published-producer-1.0-$hostSpecificSourceSet.klib", + "published-producer-1.0-commonMain.klib", + "published-dependency-1.0-$hostSpecificSourceSet.klib", + "published-dependency-1.0-commonMain.klib" ) ) } @@ -357,6 +357,7 @@ class KlibBasedMppIT : BaseGradleIT() { build("${subproject?.prependIndent(":").orEmpty()}:$printingTaskName") { assertSuccessful() val itemsLine = output.lines().single { "###$printingTaskName" in it }.substringAfter(printingTaskName) + // NOTE: This does not work for commonized libraries, they may contain the ',' naturally val items = itemsLine.removeSurrounding("[", "]").split(", ").toSet() checkAnyItemsContains.forEach { pattern -> assertTrue { items.any { pattern in it } } } checkNoItemContains.forEach { pattern -> assertFalse { items.any { pattern in it } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/kpm/idea/IdeaKpmMetadataBinaryDependencyResolver.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/kpm/idea/IdeaKpmMetadataBinaryDependencyResolver.kt index bd19b3f6b49..6fa0f7baf09 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/kpm/idea/IdeaKpmMetadataBinaryDependencyResolver.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/kpm/idea/IdeaKpmMetadataBinaryDependencyResolver.kt @@ -9,11 +9,11 @@ import org.gradle.api.artifacts.component.ModuleComponentIdentifier import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider -import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.GradleKpmFragmentGranularMetadataResolverFactory +import org.jetbrains.kotlin.gradle.plugin.mpp.kotlinTransformedMetadataLibraryDirectoryForIde import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.GradleKpmFragment +import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.GradleKpmFragmentGranularMetadataResolverFactory import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.GradleKpmModule.Companion.moduleName import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.toKpmModuleDependency -import org.jetbrains.kotlin.gradle.utils.withTemporaryDirectory internal class IdeaKpmMetadataBinaryDependencyResolver( private val fragmentGranularMetadataResolverFactory: GradleKpmFragmentGranularMetadataResolverFactory @@ -34,39 +34,28 @@ internal class IdeaKpmMetadataBinaryDependencyResolver( is ProjectMetadataProvider -> return emptySet() is JarMetadataProvider -> resolution.metadataProvider } - return resolution.allVisibleSourceSetNames.mapNotNull { visibleFragmentName -> - val binaryFile = withTemporaryDirectory("metadataBinaryDependencyResolver") { temporaryDirectory -> - val sourceBinaryFile = metadataProvider.getSourceSetCompiledMetadata( - sourceSetName = visibleFragmentName, - outputDirectory = temporaryDirectory, - materializeFile = true + + return metadataProvider.read { artifactHandle -> + resolution.allVisibleSourceSetNames.mapNotNull { visibleFragmentName -> + val sourceSet = artifactHandle.findSourceSet(visibleFragmentName) ?: return@mapNotNull null + val metadataLibrary = sourceSet.metadataLibrary ?: return@mapNotNull null + + val libraryFile = fragment.project.kotlinTransformedMetadataLibraryDirectoryForIde + .resolve(metadataLibrary.relativeFile) + .apply { if (!isFile) metadataLibrary.copyTo(this) } + + IdeaKpmResolvedBinaryDependencyImpl( + binaryType = IdeaKpmDependency.CLASSPATH_BINARY_TYPE, + binaryFile = libraryFile, + coordinates = IdeaKpmBinaryCoordinatesImpl( + group = gradleModuleIdentifier.group, + module = gradleModuleIdentifier.module, + version = gradleModuleIdentifier.version, + kotlinModuleName = kotlinModuleIdentifier.moduleName, + kotlinFragmentName = visibleFragmentName + ) ) - - if (!sourceBinaryFile.isFile) - return@mapNotNull null - - fragment.project.rootDir - .resolve(".gradle").resolve("kotlin").resolve("transformedKotlinMetadata") - .resolve(gradleModuleIdentifier.group) - .resolve(gradleModuleIdentifier.module) - .resolve(gradleModuleIdentifier.version) - .resolve(kotlinModuleIdentifier.moduleName) - .resolve(visibleFragmentName) - .resolve(sourceBinaryFile.name) - .apply { if (!isFile) sourceBinaryFile.copyTo(this) } } - - IdeaKpmResolvedBinaryDependencyImpl( - binaryType = IdeaKpmDependency.CLASSPATH_BINARY_TYPE, - binaryFile = binaryFile, - coordinates = IdeaKpmBinaryCoordinatesImpl( - group = gradleModuleIdentifier.group, - module = gradleModuleIdentifier.module, - version = gradleModuleIdentifier.version, - kotlinModuleName = kotlinModuleIdentifier.moduleName, - kotlinFragmentName = visibleFragmentName - ) - ) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/CompositeMetadataJar.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/CompositeMetadataJar.kt deleted file mode 100644 index 626dcb55689..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/CompositeMetadataJar.kt +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.mpp - -import org.jetbrains.kotlin.gradle.utils.copyZipFilePartially -import org.jetbrains.kotlin.gradle.utils.ensureValidZipDirectoryPath -import org.jetbrains.kotlin.gradle.utils.listDescendants -import java.io.File -import java.util.zip.ZipFile - -internal fun CompositeMetadataJar( - moduleIdentifier: String, - projectStructureMetadata: KotlinProjectStructureMetadata, - primaryArtifactFile: File, - hostSpecificArtifactsBySourceSet: Map, -): CompositeMetadataJar = CompositeMetadataJarImpl( - moduleIdentifier, projectStructureMetadata, primaryArtifactFile, hostSpecificArtifactsBySourceSet -) - -internal interface CompositeMetadataJar { - fun getSourceSetCompiledMetadata( - sourceSetName: String, outputDirectory: File, materializeFile: Boolean - ): File - - fun getSourceSetCInteropMetadata( - sourceSetName: String, outputDirectory: File, materializeFiles: Boolean - ): Set -} - -private class CompositeMetadataJarImpl( - private val moduleIdentifier: String, - private val projectStructureMetadata: KotlinProjectStructureMetadata, - private val primaryArtifactFile: File, - private val hostSpecificArtifactsBySourceSet: Map, -) : CompositeMetadataJar { - - override fun getSourceSetCompiledMetadata( - sourceSetName: String, outputDirectory: File, materializeFile: Boolean - ): File { - val artifactFile = getArtifactFile(sourceSetName) - val moduleOutputDirectory = outputDirectory.resolve(moduleIdentifier).also { it.mkdirs() } - - val extension = projectStructureMetadata.sourceSetBinaryLayout[sourceSetName]?.archiveExtension - ?: SourceSetMetadataLayout.METADATA.archiveExtension - - val metadataOutputFile = moduleOutputDirectory.resolve("$moduleIdentifier-$sourceSetName.$extension") - - /** In composite builds, we don't really need tro process the file in IDE import, so ignore it if it's missing */ - // refactor: allow only included builds to provide no artifacts, and allow this only in IDE import - if (!artifactFile.isFile) { - return metadataOutputFile - } - - if (!materializeFile) { - return metadataOutputFile - } - - if (metadataOutputFile.exists()) { - metadataOutputFile.delete() - } - - copyZipFilePartially(artifactFile, metadataOutputFile, "$sourceSetName/") - return metadataOutputFile - } - - override fun getSourceSetCInteropMetadata( - sourceSetName: String, outputDirectory: File, materializeFiles: Boolean - ): Set { - val artifactFile = getArtifactFile(sourceSetName) - val moduleOutputDirectory = outputDirectory.resolve(moduleIdentifier).also(File::mkdirs) - - ZipFile(getArtifactFile(sourceSetName)).use { artifactZipFile -> - val cinteropMetadataDirectory = projectStructureMetadata.sourceSetCInteropMetadataDirectory[sourceSetName] ?: return emptySet() - val cinteropMetadataDirectoryPath = ensureValidZipDirectoryPath(cinteropMetadataDirectory) - val cinteropEntries = artifactZipFile.listDescendants(cinteropMetadataDirectoryPath) - val cinteropNames = cinteropEntries.map { entry -> - entry.name.removePrefix(cinteropMetadataDirectoryPath).split("/", limit = 2).first() - }.toSet() - - val cinteropsByOutputFile = cinteropNames.associateBy { cinteropName -> moduleOutputDirectory.resolve("$cinteropName.klib") } - if (materializeFiles) { - cinteropsByOutputFile.forEach { (cinteropOutputFile, cinteropName) -> - copyZipFilePartially(artifactFile, cinteropOutputFile, "$cinteropMetadataDirectoryPath$cinteropName/") - } - } - - return cinteropsByOutputFile.keys - } - } - - private fun getArtifactFile(sourceSetName: String): File = - hostSpecificArtifactsBySourceSet[sourceSetName] ?: primaryArtifactFile -} - diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GetAllCompiledSourceSetMetadata.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GetAllCompiledSourceSetMetadata.kt deleted file mode 100644 index 619ce7b14e2..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GetAllCompiledSourceSetMetadata.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.mpp - -import org.gradle.api.Project -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.FileCollection -import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets -import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider -import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider -import java.io.File - -/** - * @param outputDirectoryWhenMaterialised: Root output directory to put files if [materializeFilesIfNecessary] is set - * and the artifact is indeed a [CompositeMetadataJar] - * - * @param materializeFilesIfNecessary: If the artifact is a [CompositeMetadataJar] then this flag will tell if - * compiled source set metadata should be extracted from this jar file into the given [outputDirectoryWhenMaterialised] - */ -internal fun ChooseVisibleSourceSets.getAllCompiledSourceSetMetadata( - project: Project, - outputDirectoryWhenMaterialised: File, - materializeFilesIfNecessary: Boolean -): ConfigurableFileCollection = project.files( - visibleSourceSetNamesExcludingDependsOn.map { visibleSourceSetName -> - metadataProvider.getSourceSetCompiledMetadata( - project, visibleSourceSetName, outputDirectoryWhenMaterialised, materializeFilesIfNecessary - ) - } -) - -internal fun ChooseVisibleSourceSets.MetadataProvider.getSourceSetCompiledMetadata( - project: Project, - sourceSetName: String, - outputDirectoryWhenMaterialised: File, - materializeFilesIfNecessary: Boolean -): FileCollection = when (this) { - is ProjectMetadataProvider -> getSourceSetCompiledMetadata(sourceSetName) - is JarMetadataProvider -> project.files( - getSourceSetCompiledMetadata( - sourceSetName, outputDirectoryWhenMaterialised, materializeFilesIfNecessary - ) - ) -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt index f35e5ea75e6..a1f9009759b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt @@ -16,7 +16,7 @@ import org.gradle.api.file.FileCollection import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.dsl.multiplatformExtension import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet -import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.Companion.asMetadataProvider +import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope import org.jetbrains.kotlin.gradle.plugin.sources.sourceSetDependencyConfigurationByScope import org.jetbrains.kotlin.gradle.targets.metadata.ALL_COMPILE_METADATA_CONFIGURATION_NAME @@ -80,8 +80,8 @@ internal sealed class MetadataDependencyResolution( ) : MetadataDependencyResolution(dependency, projectDependency) { internal sealed class MetadataProvider { - class JarMetadataProvider(private val compositeMetadataArtifact: CompositeMetadataJar) : - MetadataProvider(), CompositeMetadataJar by compositeMetadataArtifact + class JarMetadataProvider(private val compositeMetadataArtifact: CompositeMetadataArtifact) : + MetadataProvider(), CompositeMetadataArtifact by compositeMetadataArtifact abstract class ProjectMetadataProvider : MetadataProvider() { enum class MetadataConsumer { Ide, Cli } @@ -89,10 +89,6 @@ internal sealed class MetadataDependencyResolution( abstract fun getSourceSetCompiledMetadata(sourceSetName: String): FileCollection abstract fun getSourceSetCInteropMetadata(sourceSetName: String, consumer: MetadataConsumer): FileCollection } - - companion object { - fun CompositeMetadataJar.asMetadataProvider() = JarMetadataProvider(this) - } } override fun toString(): String = @@ -178,6 +174,7 @@ internal class GranularMetadataTransformation( val transitiveDependenciesToVisit = when (dependencyResult) { is MetadataDependencyResolution.KeepOriginalDependency -> resolvedDependency.dependencies.filterIsInstance() + is MetadataDependencyResolution.ChooseVisibleSourceSets -> dependencyResult.visibleTransitiveDependencies is MetadataDependencyResolution.Exclude.PublishedPlatformSourceSetDependency -> dependencyResult.visibleTransitiveDependencies is MetadataDependencyResolution.Exclude.Unrequested -> error("a visited dependency is erroneously considered unrequested") @@ -277,12 +274,15 @@ internal class GranularMetadataTransformation( moduleIdentifier = mppDependencyMetadataExtractor.moduleIdentifier ) - is JarMppDependencyProjectStructureMetadataExtractor -> CompositeMetadataJar( - moduleIdentifier = ModuleIds.fromComponent(project, module).toString(), - projectStructureMetadata = projectStructureMetadata, - primaryArtifactFile = mppDependencyMetadataExtractor.primaryArtifactFile, - hostSpecificArtifactsBySourceSet = sourceSetVisibility.hostSpecificMetadataArtifactBySourceSet, - ).asMetadataProvider() + is JarMppDependencyProjectStructureMetadataExtractor -> JarMetadataProvider( + CompositeMetadataArtifactImpl( + moduleDependencyIdentifier = ModuleIds.fromComponent(project, module), + moduleDependencyVersion = module.moduleVersion?.version ?: "unspecified", + kotlinProjectStructureMetadata = projectStructureMetadata, + primaryArtifactFile = mppDependencyMetadataExtractor.primaryArtifactFile, + hostSpecificArtifactFilesBySourceSetName = sourceSetVisibility.hostSpecificMetadataArtifactBySourceSet + ) + ) } return MetadataDependencyResolution.ChooseVisibleSourceSets( diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt new file mode 100644 index 00000000000..66455291a83 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.mpp + +import org.gradle.api.Project +import java.io.File + +private const val kotlinTransformedMetadataLibraries = "kotlinTransformedMetadataLibraries" +private const val kotlinTransformedCInteropMetadataLibraries = "kotlinTransformedCInteropMetadataLibraries" + +internal fun Project.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File = + buildDir.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName) + +internal val Project.kotlinTransformedMetadataLibraryDirectoryForIde: File + get() = rootDir.resolve(".gradle").resolve("kotlin").resolve(kotlinTransformedMetadataLibraries) + +internal fun Project.kotlinTransformedCInteropMetadataLibraryDirectoryForBuild(sourceSetName: String): File = + buildDir.resolve(kotlinTransformedCInteropMetadataLibraries).resolve(sourceSetName) + +internal val Project.kotlinTransformedCInteropMetadataLibraryDirectoryForIde: File + get() = rootDir.resolve(".gradle").resolve("kotlin").resolve(kotlinTransformedCInteropMetadataLibraries) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/TransformKotlinGranularMetadata.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/TransformKotlinGranularMetadata.kt index 8792f11a585..505b188753d 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/TransformKotlinGranularMetadata.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/TransformKotlinGranularMetadata.kt @@ -14,7 +14,6 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope.* import org.jetbrains.kotlin.gradle.plugin.sources.internal -import org.jetbrains.kotlin.gradle.plugin.sources.withDependsOnClosure import org.jetbrains.kotlin.gradle.targets.metadata.ALL_COMPILE_METADATA_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.targets.metadata.KotlinMetadataTargetConfigurator import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider @@ -37,7 +36,7 @@ open class TransformKotlinGranularMetadata @get:OutputDirectory val outputsDir: File by project.provider { - project.buildDir.resolve("kotlinSourceSetMetadata/${kotlinSourceSet.name}") + project.kotlinTransformedMetadataLibraryDirectoryForBuild(kotlinSourceSet.name) } @Suppress("unused") // Gradle input @@ -98,14 +97,20 @@ open class TransformKotlinGranularMetadata transformation.metadataDependencyResolutions } + /** + * Contains "actually transformed" metadata libraries (extracted from any [CompositeMetadataArtifact]) as well as + * [FileCollection]s pointing to project dependency klibs. + * + * Project dependencies are still required to be listed here (despite not being produced by this task), since + * this [filesByResolution] will be used to build the compile-classpath for metadata compilations. + */ @get:Internal internal val filesByResolution: Map get() = metadataDependencyResolutions .filterIsInstance() .associateWith { chooseVisibleSourceSets -> - chooseVisibleSourceSets.getAllCompiledSourceSetMetadata( - project, outputDirectoryWhenMaterialised = outputsDir, materializeFilesIfNecessary = false - ).builtBy(this) + val files = project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false) + project.files(files).builtBy(this) } @TaskAction @@ -118,9 +123,7 @@ open class TransformKotlinGranularMetadata metadataDependencyResolutions .filterIsInstance() .forEach { chooseVisibleSourceSets -> - chooseVisibleSourceSets.getAllCompiledSourceSetMetadata( - project, outputDirectoryWhenMaterialised = outputsDir, materializeFilesIfNecessary = true - ) + project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmFragmentGranularMetadataResolver.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmFragmentGranularMetadataResolver.kt index 8b71d79c855..a55dc087715 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmFragmentGranularMetadataResolver.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmFragmentGranularMetadataResolver.kt @@ -8,7 +8,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20 import org.gradle.api.Project import org.gradle.api.artifacts.result.ResolvedDependencyResult import org.jetbrains.kotlin.gradle.plugin.mpp.* -import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.Companion.asMetadataProvider +import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider import org.jetbrains.kotlin.project.model.* import org.jetbrains.kotlin.utils.addToStdlib.flattenTo import java.io.File @@ -79,6 +79,7 @@ internal class GradleKpmFragmentGranularMetadataResolver( is GradleKpmExternalPlainModule -> { MetadataDependencyResolution.KeepOriginalDependency(resolvedComponentResult, isResolvedAsProject) } + else -> run { val metadataSourceComponent = dependencyNode.run { metadataSourceComponent ?: selectedComponent } @@ -108,14 +109,17 @@ internal class GradleKpmFragmentGranularMetadataResolver( moduleIdentifier = projectStructureMetadataExtractor.moduleIdentifier ) - is JarMppDependencyProjectStructureMetadataExtractor -> CompositeMetadataJar( - moduleIdentifier = ModuleIds.fromComponent(project, metadataSourceComponent).toString(), - projectStructureMetadata = projectStructureMetadata, - primaryArtifactFile = projectStructureMetadataExtractor.primaryArtifactFile, - hostSpecificArtifactsBySourceSet = if ( - dependencyModule is GradleKpmExternalImportedModule && chosenFragments != null - ) resolveHostSpecificMetadataArtifacts(dependencyModule, chosenFragments) else emptyMap(), - ).asMetadataProvider() + is JarMppDependencyProjectStructureMetadataExtractor -> JarMetadataProvider( + CompositeMetadataArtifactImpl( + moduleDependencyIdentifier = ModuleIds.fromComponent(project, metadataSourceComponent), + moduleDependencyVersion = metadataSourceComponent.moduleVersion?.version ?: "unspecified", + kotlinProjectStructureMetadata = projectStructureMetadata, + primaryArtifactFile = projectStructureMetadataExtractor.primaryArtifactFile, + hostSpecificArtifactFilesBySourceSetName = if ( + dependencyModule is GradleKpmExternalImportedModule && chosenFragments != null + ) resolveHostSpecificMetadataArtifacts(dependencyModule, chosenFragments) else emptyMap(), + ) + ) } MetadataDependencyResolution.ChooseVisibleSourceSets( diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/TransformKotlinGranularMetadataForFragment.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/TransformKotlinGranularMetadataForFragment.kt index 0b1ff91d595..013bd2fbed5 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/TransformKotlinGranularMetadataForFragment.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/TransformKotlinGranularMetadataForFragment.kt @@ -10,8 +10,8 @@ import org.gradle.api.file.FileCollection import org.gradle.api.tasks.* import org.gradle.work.NormalizeLineEndings import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution -import org.jetbrains.kotlin.gradle.plugin.mpp.getAllCompiledSourceSetMetadata import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.util.disambiguateName +import org.jetbrains.kotlin.gradle.plugin.mpp.transformMetadataLibrariesForBuild import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider import org.jetbrains.kotlin.gradle.utils.getValue import java.io.File @@ -72,9 +72,8 @@ internal open class TransformKotlinGranularMetadataForFragment get() = metadataDependencyResolutions .filterIsInstance() .associateWith { chooseVisibleSourceSets -> - chooseVisibleSourceSets.getAllCompiledSourceSetMetadata( - project, outputsDir, materializeFilesIfNecessary = false - ).builtBy(this) + project.files(project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false)) + .builtBy(this) } @TaskAction @@ -87,7 +86,7 @@ internal open class TransformKotlinGranularMetadataForFragment metadataDependencyResolutions .filterIsInstance() .forEach { chooseVisibleSourceSets -> - chooseVisibleSourceSets.getAllCompiledSourceSetMetadata(project, outputsDir, materializeFilesIfNecessary = true) + project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt new file mode 100644 index 00000000000..58b1d1c14ce --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt @@ -0,0 +1,97 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.mpp + +import org.gradle.api.Project +import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider +import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider +import java.io.File + +/** + * Returns a map from 'visibleSourceSetName' to the transformed metadata libraries. + * The map is necessary to support [MetadataDependencyTransformation]'s shape, which + * is used in import and therefore hard to change. + * + * When this function will also support project dependencies and just return the compiled output files. + */ +internal fun Project.transformMetadataLibrariesForIde( + resolution: MetadataDependencyResolution.ChooseVisibleSourceSets +): Map> { + return when (val metadataProvider = resolution.metadataProvider) { + is ProjectMetadataProvider -> resolution.visibleSourceSetNamesExcludingDependsOn.associateWith { visibleSourceSetName -> + metadataProvider.getSourceSetCompiledMetadata(visibleSourceSetName) + } + + is JarMetadataProvider -> transformMetadataLibrariesForIde( + kotlinTransformedMetadataLibraryDirectoryForIde, resolution, metadataProvider + ) + } +} + +/** + * Will transform the [CompositeMetadataArtifact] extracting the visible source sets specified in the [resolution] + * @param materializeFiles: If true, the klib files will actually be created and extracted + * + * In case the [resolution] points to a project dependency, then the output file collections will be returned. + */ +internal fun Project.transformMetadataLibrariesForBuild( + resolution: MetadataDependencyResolution.ChooseVisibleSourceSets, outputDirectory: File, materializeFiles: Boolean +): Iterable { + return when (resolution.metadataProvider) { + is ProjectMetadataProvider -> project.files( + resolution.visibleSourceSetNamesExcludingDependsOn.map { visibleSourceSetName -> + resolution.metadataProvider.getSourceSetCompiledMetadata(visibleSourceSetName) + } + ) + + is JarMetadataProvider -> transformMetadataLibrariesForBuild( + resolution, outputDirectory, materializeFiles, resolution.metadataProvider + ) + } +} + +/* Implementations for transforming the Composite Metadata Artifact */ + +private fun transformMetadataLibrariesForIde( + baseOutputDirectory: File, + resolution: MetadataDependencyResolution.ChooseVisibleSourceSets, + compositeMetadataArtifact: CompositeMetadataArtifact +): Map> { + return compositeMetadataArtifact.read { artifactHandle -> + resolution.visibleSourceSetNamesExcludingDependsOn.mapNotNull { visibleSourceSetName -> + val sourceSet = artifactHandle.findSourceSet(visibleSourceSetName) ?: return@mapNotNull null + val sourceSetMetadataLibrary = sourceSet.metadataLibrary ?: return@mapNotNull null + val metadataLibraryOutputFile = baseOutputDirectory.resolve(sourceSetMetadataLibrary.relativeFile) + metadataLibraryOutputFile.parentFile.mkdirs() + if (!metadataLibraryOutputFile.exists()) { + sourceSetMetadataLibrary.copyTo(metadataLibraryOutputFile) + if (!metadataLibraryOutputFile.exists()) return@mapNotNull null + } + + visibleSourceSetName to listOf(metadataLibraryOutputFile) + }.toMap() + } +} + +private fun transformMetadataLibrariesForBuild( + resolution: MetadataDependencyResolution.ChooseVisibleSourceSets, + outputDirectory: File, + materializeFiles: Boolean, + compositeMetadataArtifact: CompositeMetadataArtifact +): Set { + return compositeMetadataArtifact.read { artifactHandle -> + resolution.visibleSourceSetNamesExcludingDependsOn.mapNotNull { visibleSourceSetName -> + val sourceSet = artifactHandle.findSourceSet(visibleSourceSetName) ?: return@mapNotNull null + val metadataLibrary = sourceSet.metadataLibrary ?: return@mapNotNull null + val metadataLibraryFile = outputDirectory.resolve(metadataLibrary.relativeFile) + if (materializeFiles) { + metadataLibraryFile.parentFile?.mkdirs() + metadataLibrary.copyTo(metadataLibraryFile) + } + metadataLibraryFile + } + }.toSet() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/DefaultKotlinSourceSet.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/DefaultKotlinSourceSet.kt index f5c65b35815..99b82bf4cb0 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/DefaultKotlinSourceSet.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/DefaultKotlinSourceSet.kt @@ -8,7 +8,6 @@ package org.jetbrains.kotlin.gradle.plugin.sources import org.gradle.api.Action -import org.gradle.api.InvalidUserCodeException import org.gradle.api.Project import org.gradle.api.file.SourceDirectorySet import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS @@ -19,7 +18,6 @@ import org.jetbrains.kotlin.gradle.plugin.LanguageSettingsBuilder import org.jetbrains.kotlin.gradle.plugin.mpp.* import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.tooling.core.closure -import org.jetbrains.kotlin.tooling.core.withClosure import java.io.File import java.util.* import javax.inject.Inject @@ -156,15 +154,6 @@ abstract class DefaultKotlinSourceSet @Inject constructor( ?.associateBy { ModuleIds.fromComponent(project, it.dependency) } ?: emptyMap() - val baseDir = SourceSetMetadataStorageForIde.sourceSetStorageWithScope(project, this@DefaultKotlinSourceSet.name, scope) - - if (metadataDependencyResolutionByModule.values.any { it is MetadataDependencyResolution.ChooseVisibleSourceSets }) { - if (baseDir.isDirectory) { - baseDir.deleteRecursively() - } - baseDir.mkdirs() - } - return metadataDependencyResolutionByModule.mapNotNull { (groupAndName, resolution) -> val (group, name) = groupAndName val projectPath = resolution.projectDependency?.path @@ -178,20 +167,11 @@ abstract class DefaultKotlinSourceSet @Inject constructor( MetadataDependencyTransformation(group, name, projectPath, null, emptySet(), emptyMap()) is MetadataDependencyResolution.ChooseVisibleSourceSets -> { - val filesBySourceSet = resolution.visibleSourceSetNamesExcludingDependsOn.associateWith { visibleSourceSetName -> - resolution.metadataProvider.getSourceSetCompiledMetadata( - project, - sourceSetName = visibleSourceSetName, - outputDirectoryWhenMaterialised = baseDir, - materializeFilesIfNecessary = true - ) - }.filter { (_, files) -> files.any(File::exists) } - MetadataDependencyTransformation( group, name, projectPath, resolution.projectStructureMetadata, resolution.allVisibleSourceSetNames, - filesBySourceSet + project.transformMetadataLibrariesForIde(resolution) ) } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/SourceSetMetadataStorageForIde.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/SourceSetMetadataStorageForIde.kt deleted file mode 100644 index 9f32202f5a3..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/SourceSetMetadataStorageForIde.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.plugin.sources - -import org.gradle.api.Project -import org.gradle.api.provider.MapProperty -import org.gradle.api.provider.Property -import org.gradle.api.services.BuildService -import org.gradle.api.services.BuildServiceParameters -import org.gradle.api.services.BuildServiceSpec -import org.gradle.tooling.events.FinishEvent -import org.gradle.tooling.events.OperationCompletionListener -import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull -import java.io.File - -object SourceSetMetadataStorageForIde { - private fun getStorageRoot(project: Project): File = project.rootDir.resolve(".gradle/kotlin/sourceSetMetadata") - - private fun projectStorage(project: Project): File { - val projectPathSegments = generateSequence(project) { it.parent }.map { it.name } - return getStorageRoot(project).resolve( - // Escape dots in project names to avoid ambiguous paths. - projectPathSegments.joinToString(".") { it.replace(".", "_.") } - ) - } - - fun sourceSetStorage(project: Project, sourceSetName: String) = projectStorage(project).resolve(sourceSetName) - - internal fun sourceSetStorageWithScope(project: Project, sourceSetName: String, scope: KotlinDependencyScope) = - sourceSetStorage(project, sourceSetName).resolve(scope.scopeName) -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropMetadataDependencyTransformationTask.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropMetadataDependencyTransformationTask.kt index e03008418cb..91882114372 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropMetadataDependencyTransformationTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropMetadataDependencyTransformationTask.kt @@ -15,14 +15,12 @@ import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider import org.jetbrains.kotlin.gradle.plugin.ide.Idea222Api import org.jetbrains.kotlin.gradle.plugin.ide.ideaImportDependsOn -import org.jetbrains.kotlin.gradle.plugin.mpp.GranularMetadataTransformation -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinProjectStructureMetadata +import org.jetbrains.kotlin.gradle.plugin.mpp.* import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.toKpmModuleIdentifiers import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet -import org.jetbrains.kotlin.gradle.plugin.sources.SourceSetMetadataStorageForIde import org.jetbrains.kotlin.gradle.tasks.dependsOn import org.jetbrains.kotlin.gradle.tasks.locateOrRegisterTask import org.jetbrains.kotlin.gradle.tasks.withType @@ -33,6 +31,7 @@ import org.jetbrains.kotlin.gradle.utils.outputFilesProvider import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION import org.jetbrains.kotlin.project.model.KpmModuleIdentifier import java.io.File +import java.io.Serializable import java.util.concurrent.Callable import javax.inject.Inject @@ -46,7 +45,11 @@ internal fun Project.locateOrRegisterCInteropMetadataDependencyTransformationTas args = listOf( sourceSet, /* outputDirectory = */ - project.buildDir.resolve("kotlinSourceSetMetadata").resolve(sourceSet.name + "-cinterop") + project.kotlinTransformedCInteropMetadataLibraryDirectoryForBuild(sourceSet.name), + /* outputLibraryFilesDiscovery = */ + CInteropMetadataDependencyTransformationTask.OutputLibraryFilesDiscovery.ScanOutputDirectory, + /* cleaning = */ + CInteropMetadataDependencyTransformationTask.Cleaning.DeleteOutputDirectory ), configureTask = { configureTaskOrder(); onlyIfSourceSetIsSharedNative() } ) @@ -70,7 +73,11 @@ internal fun Project.locateOrRegisterCInteropMetadataDependencyTransformationTas args = listOf( sourceSet, /* outputDirectory = */ - SourceSetMetadataStorageForIde.sourceSetStorage(project, sourceSet.name).resolve("cinterop") + project.kotlinTransformedCInteropMetadataLibraryDirectoryForIde, + /* outputLibraryFilesDiscovery = */ + CInteropMetadataDependencyTransformationTask.OutputLibraryFilesDiscovery.Precise, + /* cleaning = */ + CInteropMetadataDependencyTransformationTask.Cleaning.None ), configureTask = { configureTaskOrder(); onlyIfSourceSetIsSharedNative() } ) @@ -99,13 +106,62 @@ private fun CInteropMetadataDependencyTransformationTask.onlyIfSourceSetIsShared internal open class CInteropMetadataDependencyTransformationTask @Inject constructor( @Transient @get:Internal val sourceSet: DefaultKotlinSourceSet, - @get:OutputDirectory val outputDirectory: File + @get:OutputDirectory val outputDirectory: File, + @get:Internal val outputLibraryFilesDiscovery: OutputLibraryFilesDiscovery, + @get:Internal val cleaning: Cleaning ) : DefaultTask() { init { notCompatibleWithConfigurationCache("Task $name does not support Gradle Configuration Cache. Check KT-49933 for more info") } + sealed class OutputLibraryFilesDiscovery : Serializable { + abstract fun resolveOutputLibraryFiles(outputDirectory: File, resolutions: Iterable): Set + + /** + * Can be used when the used output directory is only used by this task. + * In this case, all libraries in the given outputDirectory will be library files produced by this task, so just + * scanning its content will be enough. + */ + object ScanOutputDirectory : OutputLibraryFilesDiscovery() { + override fun resolveOutputLibraryFiles(outputDirectory: File, resolutions: Iterable): Set { + return outputDirectory.walkTopDown().maxDepth(2).filter { it.isFile && it.extension == KLIB_FILE_EXTENSION }.toSet() + } + } + + /** + * Will actually read the [CompositeMetadataArtifact] and infer the exact file locations. + * This can be used if the output directory might be shared with other tasks. + */ + object Precise : OutputLibraryFilesDiscovery() { + override fun resolveOutputLibraryFiles(outputDirectory: File, resolutions: Iterable): Set { + return resolutions.flatMap { chooseVisibleSourceSets -> + if (chooseVisibleSourceSets.metadataProvider !is JarMetadataProvider) return@flatMap emptyList() + chooseVisibleSourceSets.metadataProvider.read { artifactHandle -> + chooseVisibleSourceSets.visibleSourceSetsProvidingCInterops + .mapNotNull { visibleSourceSetName -> artifactHandle.findSourceSet(visibleSourceSetName) } + .flatMap { sourceSet -> sourceSet.cinteropMetadataLibraries } + .map { cInteropMetadataLibrary -> outputDirectory.resolve(cInteropMetadataLibrary.relativeFile) } + } + }.toSet() + } + } + } + + sealed class Cleaning : Serializable { + abstract fun cleanOutputDirectory(outputDirectory: File) + + object DeleteOutputDirectory : Cleaning() { + override fun cleanOutputDirectory(outputDirectory: File) { + if (outputDirectory.isDirectory) outputDirectory.deleteRecursively() + } + } + + object None : Cleaning() { + override fun cleanOutputDirectory(outputDirectory: File) = Unit + } + } + @Suppress("unused") class ChooseVisibleSourceSetProjection( @Input val dependencyModuleIdentifiers: List, @@ -137,33 +193,30 @@ internal open class CInteropMetadataDependencyTransformationTask @Inject constru get() = chooseVisibleSourceSets.map(::ChooseVisibleSourceSetProjection).toSet() @get:Internal - val outputLibraryFiles = outputFilesProvider { - outputDirectory.walkTopDown().maxDepth(2).filter { it.isFile && it.extension == KLIB_FILE_EXTENSION }.toList() - } + val outputLibraryFiles = outputFilesProvider(lazy { + outputLibraryFilesDiscovery.resolveOutputLibraryFiles(outputDirectory, chooseVisibleSourceSets) + }) @TaskAction protected fun transformDependencies() { - if (outputDirectory.isDirectory) { - outputDirectory.deleteRecursively() - } - + cleaning.cleanOutputDirectory(outputDirectory) if (project.getCommonizerTarget(sourceSet) !is SharedCommonizerTarget) return - chooseVisibleSourceSets.flatMap(::materializeMetadata) + chooseVisibleSourceSets.forEach(::materializeMetadata) } private fun materializeMetadata( chooseVisibleSourceSets: ChooseVisibleSourceSets - ): Set = when (chooseVisibleSourceSets.metadataProvider) { + ): Unit = when (chooseVisibleSourceSets.metadataProvider) { /* Nothing to transform: We will use original commonizer output in such cases */ - is ProjectMetadataProvider -> emptySet() + is ProjectMetadataProvider -> Unit /* Extract/Materialize all cinterop files from composite jar file */ - is JarMetadataProvider -> - chooseVisibleSourceSets.visibleSourceSetsProvidingCInterops.flatMap { visibleSourceSetName -> - chooseVisibleSourceSets.metadataProvider.getSourceSetCInteropMetadata( - visibleSourceSetName, outputDirectory, materializeFiles = true - ) - }.toSet() + is JarMetadataProvider -> chooseVisibleSourceSets.metadataProvider.read { artifactHandle -> + chooseVisibleSourceSets.visibleSourceSetsProvidingCInterops + .mapNotNull { visibleSourceSetName -> artifactHandle.findSourceSet(visibleSourceSetName) } + .flatMap { sourceSet -> sourceSet.cinteropMetadataLibraries } + .forEach { cInteropMetadataLibrary -> cInteropMetadataLibrary.copyIntoDirectory(outputDirectory) } + } } private fun Configuration.withoutProjectDependencies(): FileCollection { diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/providerApiUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/providerApiUtils.kt index 14b4d4565a0..8f36e2154c9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/providerApiUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/providerApiUtils.kt @@ -116,6 +116,10 @@ internal fun T.outputFilesProvider(provider: T.() -> Any): Configurab return project.filesProvider(this) { provider() } } +internal fun T.outputFilesProvider(lazy: Lazy): ConfigurableFileCollection { + return project.filesProvider(this) { lazy.value } +} + internal inline fun Project.listProperty(noinline itemsProvider: () -> Iterable): ListProperty = objects.listProperty(T::class.java).apply { set(provider(itemsProvider)) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/CompositeMetadataJarTest.kt b/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/CompositeMetadataJarTest.kt deleted file mode 100644 index a73ded64254..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/CompositeMetadataJarTest.kt +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -@file:Suppress("FunctionName") - -package org.jetbrains.kotlin.gradle - -import org.gradle.kotlin.dsl.support.zipTo -import org.jetbrains.kotlin.gradle.plugin.mpp.CompositeMetadataJar -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinProjectStructureMetadata -import org.jetbrains.kotlin.gradle.plugin.mpp.ModuleDependencyIdentifier -import org.jetbrains.kotlin.gradle.plugin.mpp.SourceSetMetadataLayout -import org.jetbrains.kotlin.gradle.plugin.mpp.SourceSetMetadataLayout.KLIB -import org.jetbrains.kotlin.gradle.plugin.mpp.SourceSetMetadataLayout.METADATA -import org.junit.Rule -import org.junit.rules.TemporaryFolder -import java.io.File -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertTrue -import kotlin.test.fail - -class CompositeMetadataJarTest { - - @get:Rule - val temporaryFolder = TemporaryFolder() - - @Test - fun `empty jar - get metadata - returns file`() { - val primaryArtifactContent = temporaryFolder.newFolder() - val primaryArtifactFile = temporaryFolder.newFile("metadata.jar") - zipTo(primaryArtifactFile, primaryArtifactContent) - assertTrue(primaryArtifactFile.isFile, "Expected primaryArtifactFile.isFile") - - val metadataJar = CompositeMetadataJar( - moduleIdentifier = "test-id", - projectStructureMetadata = createProjectStructureMetadata( - sourceSetBinaryLayout = mapOf("testSourceSetName" to KLIB), - sourceSetCInteropMetadataDirectory = mapOf("testSourceSetName" to "cinterop/testSourceSetName") - ), - primaryArtifactFile = primaryArtifactFile, - hostSpecificArtifactsBySourceSet = emptyMap() - ) - - val metadataOutputDirectory = temporaryFolder.newFolder() - val metadataFile = metadataJar.getSourceSetCompiledMetadata("testSourceSetName", metadataOutputDirectory, true) - - assertEquals( - File("test-id/test-id-testSourceSetName.klib"), - metadataFile.relativeTo(metadataOutputDirectory) - ) - } - - @Test - fun `empty jar - no cinterop metadata directory - get metadata - returns file`() { - val primaryArtifactContent = temporaryFolder.newFolder() - val primaryArtifactFile = temporaryFolder.newFile("metadata.jar") - zipTo(primaryArtifactFile, primaryArtifactContent) - assertTrue(primaryArtifactFile.isFile, "Expected primaryArtifactFile.isFile") - - val metadataJar = CompositeMetadataJar( - moduleIdentifier = "test-id", - projectStructureMetadata = createProjectStructureMetadata( - sourceSetBinaryLayout = mapOf("testSourceSetName" to KLIB), - sourceSetCInteropMetadataDirectory = emptyMap(), - ), - primaryArtifactFile = primaryArtifactFile, - hostSpecificArtifactsBySourceSet = emptyMap() - ) - - val metadataOutputDirectory = temporaryFolder.newFolder() - val metadataFile = metadataJar.getSourceSetCompiledMetadata("testSourceSetName", metadataOutputDirectory, true) - - assertEquals( - File("test-id/test-id-testSourceSetName.klib"), - metadataFile.relativeTo(metadataOutputDirectory) - ) - } - - @Test - fun `empty jar - get cinterop metadata - returns emptySet`() { - val primaryArtifactContent = temporaryFolder.newFolder() - val primaryArtifactFile = temporaryFolder.newFile("metadata.jar") - zipTo(primaryArtifactFile, primaryArtifactContent) - assertTrue(primaryArtifactFile.isFile, "Expected primaryArtifactFile.isFile") - - val metadataJar = CompositeMetadataJar( - moduleIdentifier = "test-id", - projectStructureMetadata = createProjectStructureMetadata(), - primaryArtifactFile = primaryArtifactFile, - hostSpecificArtifactsBySourceSet = emptyMap() - ) - - val metadataOutputDirectory = temporaryFolder.newFolder() - val metadataFiles = metadataJar.getSourceSetCInteropMetadata("testSourceSetName", metadataOutputDirectory, true) - - assertEquals( - emptySet(), metadataFiles, - "Expected no cinterop metadata files discovered in jar" - ) - } - - @Test - fun `get metadata`() { - /* Setup Artifact content */ - val primaryArtifactContent = temporaryFolder.newFolder() - - primaryArtifactContent.resolve("sourceSetA/sourceSetAStub1.txt") - .withParentDirectoriesCreated() - .writeText("Content of sourceSetA stub1") - - primaryArtifactContent.resolve("sourceSetA/nested/sourceSetAStub2.txt") - .withParentDirectoriesCreated() - .writeText("Content of sourceSetB stub2") - - primaryArtifactContent.resolve("sourceSetB/sourceSetBStub1.txt") - .withParentDirectoriesCreated() - .writeText("Content of sourceSetB stub1") - - primaryArtifactContent.resolve("sourceSetB/nested/sourceSetBStub2.txt") - .withParentDirectoriesCreated() - .writeText("Content of sourceSetB stub2") - - /* Create metadata jar */ - val primaryArtifactFile = temporaryFolder.newFile("metadata.jar") - zipTo(primaryArtifactFile, primaryArtifactContent) - - val metadataJar = CompositeMetadataJar( - moduleIdentifier = "test-id", - projectStructureMetadata = createProjectStructureMetadata( - sourceSetBinaryLayout = mapOf("sourceSetA" to KLIB, "sourceSetB" to METADATA) - ), - primaryArtifactFile = primaryArtifactFile, - hostSpecificArtifactsBySourceSet = emptyMap() - ) - - /* Extract and assert sourceSetA */ - val metadataOutputDirectory = temporaryFolder.newFolder() - - val sourceSetAMetadataFile = metadataJar.getSourceSetCompiledMetadata("sourceSetA", metadataOutputDirectory, true) - assertTrue(sourceSetAMetadataFile.isFile, "Expected sourceSetAMetadataFile.isFile") - assertEquals( - KLIB.archiveExtension, sourceSetAMetadataFile.extension, - "Expected correct archiveExtension for extracted sourceSetA" - ) - assertZipContentEquals( - temporaryFolder, - primaryArtifactContent.resolve("sourceSetA"), sourceSetAMetadataFile, - "Expected correct content of extracted 'sourceSetA'" - ) - - /* Extract and assert sourceSetB */ - val sourceSetBMetadataFile = metadataJar.getSourceSetCompiledMetadata("sourceSetB", metadataOutputDirectory, true) - assertTrue(sourceSetAMetadataFile.isFile, "Expected sourceSetBMetadataFile.isFile") - assertEquals( - METADATA.archiveExtension, sourceSetBMetadataFile.extension, - "Expected correct archiveExtension for extracted sourceSetB" - ) - assertZipContentEquals( - temporaryFolder, - primaryArtifactContent.resolve("sourceSetB"), sourceSetBMetadataFile, - "Expected correct content of extracted 'sourceSetA'" - ) - } - - @Test - fun `get cinterop metadata`() { - /* Setup Artifact content */ - val primaryArtifactContent = temporaryFolder.newFolder() - - primaryArtifactContent.resolve("sourceSetA-cinterop/interopA0/stub0") - .withParentDirectoriesCreated() - .writeText("stub0 content") - - primaryArtifactContent.resolve("sourceSetA-cinterop/interopA0/nested/stub1") - .withParentDirectoriesCreated() - .writeText("stub1 content") - - primaryArtifactContent.resolve("sourceSetA-cinterop/interopA1/stub2") - .withParentDirectoriesCreated() - .writeText("stub2 content") - - primaryArtifactContent.resolve("nested/sourceSetB/interops/interopB0/stub3") - .withParentDirectoriesCreated() - .writeText("stub3 content") - - /* Create metadata jar */ - val primaryArtifactFile = temporaryFolder.newFile("metadata.jar") - zipTo(primaryArtifactFile, primaryArtifactContent) - - val metadataJar = CompositeMetadataJar( - moduleIdentifier = "test-id", - projectStructureMetadata = createProjectStructureMetadata( - sourceSetCInteropMetadataDirectory = mapOf( - "sourceSetA" to "sourceSetA-cinterop", - "sourceSetB" to "nested/sourceSetB/interops/" - ) - ), - primaryArtifactFile = primaryArtifactFile, - hostSpecificArtifactsBySourceSet = emptyMap() - ) - - /* Assertions on sourceSetA */ - run { - val sourceSetAOutputDirectory = temporaryFolder.newFolder() - val sourceSetAInteropMetadataFiles = metadataJar.getSourceSetCInteropMetadata("sourceSetA", sourceSetAOutputDirectory, true) - assertEquals(2, sourceSetAInteropMetadataFiles.size, "Expected 2 cinterops in sourceSetA") - - /* Assertions on interopA0 */ - run { - val interopA0MetadataFile = sourceSetAInteropMetadataFiles.firstOrNull { it.name == "interopA0.klib" } - ?: fail("Failed to find 'interopA0.klib'") - assertZipContentEquals( - temporaryFolder, - primaryArtifactContent.resolve("sourceSetA-cinterop/interopA0"), interopA0MetadataFile, - "Expected correct content for extracted 'interopA0'" - ) - } - - /* Assertions on interopA1 */ - run { - val interopA1MetadataFile = sourceSetAInteropMetadataFiles.firstOrNull { it.name == "interopA1.klib" } - ?: fail("Failed to find 'interopA1.klib") - assertZipContentEquals( - temporaryFolder, - primaryArtifactContent.resolve("sourceSetA-cinterop/interopA1"), interopA1MetadataFile, - "Expected correct content for extracted 'interopA1'" - ) - } - } - - /* Assertions on sourceSetB */ - run { - val sourceSetBOutputDirectory = temporaryFolder.newFolder() - val sourceSetBInteropMetadataFiles = metadataJar.getSourceSetCInteropMetadata("sourceSetB", sourceSetBOutputDirectory, true) - assertEquals(1, sourceSetBInteropMetadataFiles.size, "Expected only one cinterop in sourceSetB") - - val interopB0MetadataFile = sourceSetBInteropMetadataFiles.firstOrNull { it.name == "interopB0.klib" } - ?: fail("Failed to find 'interopB0.klib'") - assertZipContentEquals( - temporaryFolder, - primaryArtifactContent.resolve("nested/sourceSetB/interops/interopB0"), interopB0MetadataFile, - "Expected correct content for extracted 'interopB0'" - ) - } - } -} - -private fun createProjectStructureMetadata( - sourceSetNamesByVariantName: Map> = emptyMap(), - sourceSetsDependsOnRelation: Map> = emptyMap(), - sourceSetCInteropMetadataDirectory: Map = emptyMap(), - sourceSetBinaryLayout: Map = emptyMap(), - sourceSetModuleDependencies: Map> = emptyMap(), - hostSpecificSourceSets: Set = emptySet(), - isPublishedAsRoot: Boolean = true, -): KotlinProjectStructureMetadata { - return KotlinProjectStructureMetadata( - sourceSetNamesByVariantName = sourceSetNamesByVariantName, - sourceSetsDependsOnRelation = sourceSetsDependsOnRelation, - sourceSetBinaryLayout = sourceSetBinaryLayout, - sourceSetCInteropMetadataDirectory = sourceSetCInteropMetadataDirectory, - sourceSetModuleDependencies = sourceSetModuleDependencies, - hostSpecificSourceSets = hostSpecificSourceSets, - isPublishedAsRoot = isPublishedAsRoot - ) -} - -private fun File.withParentDirectoriesCreated(): File = apply { parentFile.mkdirs() }