[Gradle] Add support Configuration Cache for MetadataDependencyTransformationTask

* Write transformedLibraries to a file after Task execution * Write visible source sets to a file after Task execution * Rearrange task properties, inputs outputs in the way that task state is serializable to configuration cache. * Introduce transformDependenciesMetadata task to be an umbrella for other transform tasks. This will be convinient for tests and other development purposes ^KT-49933
2023-01-12 16:37:41 +01:00
parent 4941e8b7de
commit 357e14dc04
7 changed files with 116 additions and 104 deletions
@@ -136,9 +136,12 @@ internal class GranularMetadataTransformation(

    val metadataDependencyResolutions: Iterable<MetadataDependencyResolution> by lazy { doTransform() }

-    val visibleSourceSetsByComponentId: Map<ComponentIdentifierKey, Set<String>> = metadataDependencyResolutions
-        .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
-        .associate { it.dependency.id.uniqueKey to it.allVisibleSourceSetNames }
+    val visibleSourceSetsByComponentId: Map<ComponentIdentifierKey, Set<String>> by lazy {
+        metadataDependencyResolutions
+            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+            .groupBy { it.dependency.id.uniqueKey }
+            .mapValues { (_, visibleSourceSets) -> visibleSourceSets.flatMapTo(mutableSetOf()) { it.allVisibleSourceSetNames } }
+    }

    private fun doTransform(): Iterable<MetadataDependencyResolution> {
        val result = mutableListOf<MetadataDependencyResolution>()
@@ -6,13 +6,14 @@
 package org.jetbrains.kotlin.gradle.plugin.mpp

 import org.gradle.api.Project
+import org.gradle.api.file.ProjectLayout
 import java.io.File

 private const val kotlinTransformedMetadataLibraries = "kotlinTransformedMetadataLibraries"
 private const val kotlinTransformedCInteropMetadataLibraries = "kotlinTransformedCInteropMetadataLibraries"

-internal fun Project.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
-    buildDir.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName)
+internal fun ProjectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
+    buildDirectory.get().asFile.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName)

 internal val Project.kotlinTransformedMetadataLibraryDirectoryForIde: File
    get() = rootDir.resolve(".gradle").resolve("kotlin").resolve(kotlinTransformedMetadataLibraries)
@@ -6,24 +6,22 @@
 package org.jetbrains.kotlin.gradle.plugin.mpp

 import org.gradle.api.DefaultTask
+import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.FileCollection
+import org.gradle.api.file.ProjectLayout
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.*
 import org.gradle.work.NormalizeLineEndings
 import org.jetbrains.kotlin.gradle.dsl.kotlinExtension
-import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
-import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope
 import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope.*
 import org.jetbrains.kotlin.gradle.plugin.sources.internal
-import org.jetbrains.kotlin.gradle.targets.metadata.ALL_COMPILE_METADATA_CONFIGURATION_NAME
 import org.jetbrains.kotlin.gradle.targets.metadata.KotlinMetadataTargetConfigurator
-import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider
 import org.jetbrains.kotlin.gradle.targets.metadata.dependsOnClosureWithInterCompilationDependencies
-import org.jetbrains.kotlin.gradle.utils.filesProvider
-import org.jetbrains.kotlin.gradle.utils.getValue
-import org.jetbrains.kotlin.gradle.utils.notCompatibleWithConfigurationCacheCompat
-import org.jetbrains.kotlin.gradle.utils.outputFilesProvider
+import org.jetbrains.kotlin.gradle.tasks.locateTask
+import org.jetbrains.kotlin.gradle.utils.*
 import java.io.File
 import javax.inject.Inject

@@ -34,78 +32,103 @@ typealias TransformKotlinGranularMetadata = MetadataDependencyTransformationTask

 open class MetadataDependencyTransformationTask
@Inject constructor(
-    @get:Internal
-    @field:Transient
-    val kotlinSourceSet: KotlinSourceSet
+    kotlinSourceSet: KotlinSourceSet,
+    private val objectFactory: ObjectFactory,
+    private val projectLayout: ProjectLayout
 ) : DefaultTask() {

-    init {
-        notCompatibleWithConfigurationCacheCompat(
-            "Task $name does not support Gradle Configuration Cache. Check KT-49933 for more info"
-        )
-    }
+    //region Task Configuration State & Inputs
+    private val transformationParameters = GranularMetadataTransformation.Params(project, kotlinSourceSet)

    @get:OutputDirectory
-    val outputsDir: File by project.provider {
-        project.kotlinTransformedMetadataLibraryDirectoryForBuild(kotlinSourceSet.name)
-    }
+    val outputsDir: File get() = projectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(transformationParameters.sourceSetName)

    @Suppress("unused") // Gradle input
    @get:InputFiles
    @get:PathSensitive(PathSensitivity.RELATIVE)
    @get:IgnoreEmptyDirectories
    @get:NormalizeLineEndings
-    internal val configurationToResolve: FileCollection get() = kotlinSourceSet.internal.resolvableMetadataConfiguration
+    internal val configurationToResolve: FileCollection = kotlinSourceSet.internal.resolvableMetadataConfiguration

-    private val participatingSourceSets: Set<KotlinSourceSet>
-        get() = kotlinSourceSet.internal.withDependsOnClosure.toMutableSet().apply {
+    @delegate:Transient // Only needed for configuring task inputs
+    private val participatingSourceSets: Set<KotlinSourceSet> by lazy {
+        kotlinSourceSet.internal.withDependsOnClosure.toMutableSet().apply {
            if (any { it.name == KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME })
                add(project.kotlinExtension.sourceSets.getByName(KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME))
        }
+    }

    @Suppress("unused") // Gradle input
    @get:Input
-    internal val inputSourceSetsAndCompilations: Map<String, Iterable<String>> by project.provider {
+    internal val inputSourceSetsAndCompilations: Map<String, Iterable<String>> by lazy {
        participatingSourceSets.associate { sourceSet ->
            sourceSet.name to sourceSet.internal.compilations.map { it.name }.sorted()
        }
    }

-    private val participatingCompilations: Iterable<KotlinCompilation<*>>
-        get() = participatingSourceSets.flatMap { it.internal.compilations }.toSet()
-
    @Suppress("unused") // Gradle input
    @get:Input
-    internal val inputCompilationDependencies: Map<String, Set<List<String?>>> by project.provider {
-        participatingCompilations.associate {
+    internal val inputCompilationDependencies: Map<String, Set<List<String?>>> by lazy {
+        participatingSourceSets.flatMap { it.internal.compilations }.associate {
            it.name to project.configurations.getByName(it.compileDependencyConfigurationName)
                .allDependencies.map { listOf(it.group, it.name, it.version) }.toSet()
        }
    }

-    private val parentVisibleSourceSets: List<Map<String, Set<String>>> by lazy {
-        dependsOnClosureWithInterCompilationDependencies(kotlinSourceSet).map {
-            project.tasks.withType(MetadataDependencyTransformationTask::class.java)
-                .getByName(KotlinMetadataTargetConfigurator.transformGranularMetadataTaskName(it.name))
-                .transformation
-                .visibleSourceSetsByComponentId
+    @get:OutputFile
+    val transformedLibrariesFileIndex: RegularFileProperty = objectFactory
+        .fileProperty()
+        .apply { set(outputsDir.resolve("${kotlinSourceSet.name}.transformedLibraries")) }
+
+    @get:OutputFile
+    val visibleSourceSetsFile: RegularFileProperty = objectFactory
+        .fileProperty()
+        .apply { set(outputsDir.resolve("${kotlinSourceSet.name}.visibleSourceSets")) }
+
+    @get:InputFiles
+    val parentVisibleSourceSetFiles: ConfigurableFileCollection = objectFactory
+        .fileCollection()
+        .from(
+            {
+                val parentSourceSets: List<Provider<File>> = dependsOnClosureWithInterCompilationDependencies(kotlinSourceSet).mapNotNull {
+                    project
+                        .tasks
+                        .locateTask<MetadataDependencyTransformationTask>(KotlinMetadataTargetConfigurator.transformGranularMetadataTaskName(it.name))
+                        ?.flatMap { it.visibleSourceSetsFile.map { it.asFile } }
+                }
+
+                parentSourceSets
+            }
+        )
+
+    //endregion Task Configuration State & Inputs
+
+    @TaskAction
+    fun transformMetadata() {
+        val transformation = GranularMetadataTransformation(
+            params = transformationParameters,
+            parentVisibleSourceSetsProvider = { parentVisibleSourceSetFiles.map(::readVisibleSourceSetsFile) }
+        )
+
+        if (outputsDir.isDirectory) {
+            outputsDir.deleteRecursively()
        }
+        outputsDir.mkdirs()
+
+        val metadataDependencyResolutions = transformation.metadataDependencyResolutions
+
+        metadataDependencyResolutions
+            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+            .forEach { chooseVisibleSourceSets ->
+                objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true)
+            }
+
+        writeTransformedLibraries(metadataDependencyResolutions)
+        writeVisibleSourceSets(transformation.visibleSourceSetsByComponentId)
    }

-    private val transformation = GranularMetadataTransformation(
-        params = GranularMetadataTransformation.Params(project, kotlinSourceSet),
-        parentVisibleSourceSetsProvider = { parentVisibleSourceSets }
-    )
-
-    @get:Internal
-    @delegate:Transient // exclude from Gradle instant execution state
-    internal val metadataDependencyResolutions: Iterable<MetadataDependencyResolution> by project.provider {
-        transformation.metadataDependencyResolutions
-    }
-
-    @get:Internal
-    internal val transformedLibraries: Provider<Iterable<File>> = project.provider {
-        metadataDependencyResolutions.flatMap { resolution ->
+    private fun writeTransformedLibraries(resolutions: Iterable<MetadataDependencyResolution>) {
+        val fileList = resolutions.flatMap { resolution ->
            val files: Iterable<File> = when (resolution) {
                is MetadataDependencyResolution.ChooseVisibleSourceSets -> when (val metadataProvider = resolution.metadataProvider) {
                    is MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ArtifactMetadataProvider -> metadataProvider.read { content ->
@@ -123,8 +146,7 @@ open class MetadataDependencyTransformationTask
                    }
                }
                is MetadataDependencyResolution.Exclude -> emptySet()
-                is MetadataDependencyResolution.KeepOriginalDependency -> transformation
-                    .params
+                is MetadataDependencyResolution.KeepOriginalDependency -> transformationParameters
                    .resolvedMetadataConfiguration
                    .componentArtifacts(resolution.dependency)
                    .map { it.file }
@@ -132,47 +154,28 @@ open class MetadataDependencyTransformationTask

            files
        }
+
+        val content = fileList.joinToString("\n")
+
+        transformedLibrariesFileIndex.get().asFile.writeText(content)
    }

-    /**
-     * Contains "actually transformed" metadata libraries (extracted from any [CompositeMetadataArtifact]) as well as
-     * [FileCollection]s pointing to project dependency klibs.
-     *
-     * Project dependencies are still required to be listed here (despite not being produced by this task), since
-     * this [filesByResolution] will be used to build the compile-classpath for metadata compilations.
-     */
-    @get:Internal
-    internal val filesByResolution: Map<out MetadataDependencyResolution, FileCollection>
-        get() = metadataDependencyResolutions
-            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
-            .associateWith { chooseVisibleSourceSets ->
-                outputFilesProvider {
-                    project.transformMetadataLibrariesForBuild(
-                        chooseVisibleSourceSets, outputsDir, materializeFiles = false
-                    )
-                }
-            }
-
-    @TaskAction
-    fun transformMetadata() {
-        if (outputsDir.isDirectory) {
-            outputsDir.deleteRecursively()
+    private fun writeVisibleSourceSets(visibleSourceSetsByComponentId: Map<String, Set<String>>) {
+        val content = visibleSourceSetsByComponentId.entries.joinToString("\n") { (id, visibleSourceSets) ->
+            "$id => ${visibleSourceSets.joinToString(",")}"
        }
-        outputsDir.mkdirs()
-
-        metadataDependencyResolutions
-            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
-            .forEach { chooseVisibleSourceSets ->
-                project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true)
-            }
+        visibleSourceSetsFile.get().asFile.writeText(content)
    }
-}

-internal class SourceSetResolvedMetadataProvider(
-    taskProvider: TaskProvider<out MetadataDependencyTransformationTask>
-) : ResolvedMetadataFilesProvider {
-    override val buildDependencies: Iterable<TaskProvider<*>> = listOf(taskProvider)
-    override val metadataResolutions: Iterable<MetadataDependencyResolution> by taskProvider.map { it.metadataDependencyResolutions }
-    override val metadataFilesByResolution: Map<out MetadataDependencyResolution, FileCollection>
-            by taskProvider.map { it.filesByResolution }
-}
+    private fun readVisibleSourceSetsFile(file: File): Map<String, Set<String>> = file
+        .readLines()
+        .associate { string ->
+            val (id, visibleSourceSetsString) = string.split(" => ")
+            id to visibleSourceSetsString.split(",").toSet()
+        }
+
+    @get:Internal // Warning! transformedLibraries is available only after Task Execution
+    val transformedLibraries: Provider<List<File>> get() = transformedLibrariesFileIndex.map { regularFile ->
+        regularFile.asFile.readLines().map { File(it) }
+    }
+}
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20

 import org.gradle.api.DefaultTask
 import org.gradle.api.file.FileCollection
+import org.gradle.api.model.ObjectFactory
 import org.gradle.api.tasks.*
 import org.gradle.work.NormalizeLineEndings
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution
@@ -22,6 +23,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask
    @get:Internal
    @field:Transient
    val fragment: GradleKpmFragment,
+    private val objectFactory: ObjectFactory,
    //FIXME annotations
    private val transformation: GradleKpmFragmentGranularMetadataResolver
 ) : DefaultTask() {
@@ -72,7 +74,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask
        get() = metadataDependencyResolutions
            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
            .associateWith { chooseVisibleSourceSets ->
-                project.files(project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false))
+                project.files(objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false))
                    .builtBy(this)
            }

@@ -86,7 +88,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask
        metadataDependencyResolutions
            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
            .forEach { chooseVisibleSourceSets ->
-                project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true)
+                objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true)
            }
    }
 }
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.gradle.plugin.mpp

 import org.gradle.api.Project
+import org.gradle.api.model.ObjectFactory
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ArtifactMetadataProvider
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider
 import java.io.File
@@ -37,11 +38,11 @@ internal fun Project.transformMetadataLibrariesForIde(
 *
 * In case the [resolution] points to a project dependency, then the output file collections will be returned.
 */
-internal fun Project.transformMetadataLibrariesForBuild(
+internal fun ObjectFactory.transformMetadataLibrariesForBuild(
    resolution: MetadataDependencyResolution.ChooseVisibleSourceSets, outputDirectory: File, materializeFiles: Boolean
 ): Iterable<File> {
    return when (resolution.metadataProvider) {
-        is ProjectMetadataProvider -> project.files(
+        is ProjectMetadataProvider -> fileCollection().from(
            resolution.visibleSourceSetNamesExcludingDependsOn.map { visibleSourceSetName ->
                resolution.metadataProvider.getSourceSetCompiledMetadata(visibleSourceSetName)
            }
@@ -6,8 +6,7 @@
 package org.jetbrains.kotlin.gradle.targets.metadata

 import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.artifacts.component.ComponentIdentifier
+import org.gradle.api.Task
 import org.gradle.api.artifacts.result.ResolvedArtifactResult
 import org.gradle.api.attributes.AttributeContainer
 import org.gradle.api.attributes.Category
@@ -57,6 +56,7 @@ class KotlinMetadataTargetConfigurator :
    KotlinOnlyTargetConfigurator<KotlinCompilation<*>, KotlinMetadataTarget>(createTestCompilation = false) {
    companion object {
        internal const val ALL_METADATA_JAR_NAME = "allMetadataJar"
+        internal const val TRANSFORM_ALL_SOURCESETS_DEPENDENCIES_METADATA = "transformDependenciesMetadata"

        internal fun transformGranularMetadataTaskName(sourceSetName: String) =
            lowerCamelCaseName("transform", sourceSetName, "DependenciesMetadata")
@@ -344,6 +344,8 @@ class KotlinMetadataTargetConfigurator :
                "Generates serialized dependencies metadata for compilation '${compilation.name}' of target '${compilation.target.name}' (for tooling)"
        }

+        project.locateOrRegisterTask<Task>(TRANSFORM_ALL_SOURCESETS_DEPENDENCIES_METADATA).dependsOn(transformationTask)
+
        val artifacts = sourceSet.internal.resolvableMetadataConfiguration.incoming.artifacts

        // Metadata from visible source sets within dependsOn closure
@@ -171,9 +171,7 @@ abstract class AbstractKotlinNativeCompile<
        {
            // Avoid resolving these dependencies during task graph construction when we can't build the target:
            if (konanTarget.enabledOnCurrentHost)
-                objectFactory.fileCollection().from(
-                    compilation.compileDependencyFiles.filterOutPublishableInteropLibs(project)
-                )
+                objectFactory.fileCollection().from({ compilation.compileDependencyFiles })
            else objectFactory.fileCollection()
        }
    )
@@ -467,6 +465,8 @@ internal constructor(
        return SharedCompilationData(manifestFile, isAllowCommonizer, refinesModule)
    }

+    private val libDirectories = project.buildLibDirectories()
+
    @TaskAction
    fun compile() {
        val output = outputFile.get()
@@ -482,7 +482,7 @@ internal constructor(
            optimized,
            debuggable,
            konanTarget,
-            libraries.files.filterKlibsPassedToCompiler(),
+            libraries.filterOutPublishableInteropLibs(libDirectories).files.filterKlibsPassedToCompiler(),
            languageSettings,
            enableEndorsedLibs,
            compilerOptions,