Rework adding sources to FilteringSourceRootsContainer in JVM tasks

Support adding lazily-evaluated file collections (i.e. tracking their source roots lazily, as opposed to eagerly evaluating the source roots.
2021-03-31 13:21:33 +03:00
parent e59a57d11c
commit 6be33448d6
11 changed files with 113 additions and 61 deletions
@@ -38,7 +38,9 @@ import javax.inject.Inject
 open class KaptGenerateStubsTask @Inject constructor(
    objectFactory: ObjectFactory
 ) : KotlinCompile() {
-    override val sourceRootsContainer = FilteringSourceRootsContainer(emptyList(), { isSourceRootAllowed(it) })
+
+    @field:Transient
+    override val sourceRootsContainer = FilteringSourceRootsContainer(objectFactory, { isSourceRootAllowed(it) })

    override val kotlinOptions: KotlinJvmOptions = KotlinJvmOptionsImpl()

@@ -109,10 +111,11 @@ open class KaptGenerateStubsTask @Inject constructor(
    }

    private val sourceRoots by project.provider {
-        kotlinCompileTask.getSourceRoots().let {
-            val javaSourceRoots = it.javaSourceRoots.filterTo(HashSet()) { isSourceRootAllowed(it) }
-            val kotlinSourceFiles = it.kotlinSourceFiles.filterTo(ArrayList()) { isSourceRootAllowed(it) }
-            SourceRoots.ForJvm(kotlinSourceFiles, javaSourceRoots)
+        kotlinCompileTask.getSourceRoots().let { compileTaskSourceRoots ->
+            SourceRoots.ForJvm(
+                compileTaskSourceRoots.kotlinSourceFiles.filterTo(mutableListOf()) { isSourceRootAllowed(it) },
+                javaSourceRootsProvider = { compileTaskSourceRoots.javaSourceRoots.filterTo(mutableSetOf()) { isSourceRootAllowed(it) } }
+            )
        }
    }

@@ -132,12 +132,16 @@ abstract class KaptTask @Inject constructor(
    @get:Internal
    var useBuildCache: Boolean = false

+    private val sourceRootsFromKotlinTask by project.provider {
+        kotlinCompileTask.sourceRootsContainer.sourceRoots
+    }
+
    @get:InputFiles
    @get:PathSensitive(PathSensitivity.RELATIVE)
    val source: FileCollection = objectFactory
        .fileCollection()
        .from(
-            { kotlinCompileTask.sourceRootsContainer.sourceRoots },
+            { sourceRootsFromKotlinTask },
            stubsDir
        )
        .asFileTree
@@ -15,6 +15,7 @@ import org.gradle.api.artifacts.maven.MavenResolver
 import org.gradle.api.attributes.Usage
 import org.gradle.api.file.ConfigurableFileTree
 import org.gradle.api.file.FileCollection
+import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.logging.Logging
 import org.gradle.api.plugins.InvalidPluginException
 import org.gradle.api.plugins.JavaPlugin
@@ -146,11 +147,11 @@ internal abstract class KotlinSourceSetProcessor<T : AbstractKotlinCompile<*>>(
        java.allSource.srcDirs(kotlinSrcDirsToAdd)
    }

-    private fun filterOutJavaSrcDirsIfPossible(sourceDirectories: FileCollection): FileCollection {
+    private fun filterOutJavaSrcDirsIfPossible(sourceDirectories: SourceDirectorySet): FileCollection {
        val java = javaSourceSet ?: return sourceDirectories

        // Build a lazily-resolved file collection that filters out Java sources from sources of this sourceDirectorySet
-        return sourceDirectories.minus(java.java.sourceDirectories)
+        return sourceDirectories.sourceDirectories.minus(java.java.sourceDirectories)
    }

    private fun createAdditionalClassesTaskForIdeRunner() {
@@ -43,6 +43,7 @@ import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.KonanTarget.*
 import org.jetbrains.kotlin.konan.target.presetName
 import org.jetbrains.kotlin.statistics.metrics.StringMetrics
+import java.io.File

 class KotlinMultiplatformPlugin : Plugin<Project> {

@@ -294,7 +295,7 @@ internal fun sourcesJarTask(compilation: KotlinCompilation<*>, componentName: St

 internal fun sourcesJarTask(
    project: Project,
-    sourceSets: Lazy<Map<String, FileCollection>>,
+    sourceSets: Lazy<Map<String, Iterable<File>>>,
    componentName: String?,
    artifactNameAppendix: String
 ): TaskProvider<Jar> = sourcesJarTaskNamed(lowerCamelCaseName(componentName, "sourcesJar"), project, sourceSets, artifactNameAppendix)
@@ -302,7 +303,7 @@ internal fun sourcesJarTask(
 internal fun sourcesJarTaskNamed(
    taskName: String,
    project: Project,
-    sourceSets: Lazy<Map<String, FileCollection>>,
+    sourceSets: Lazy<Map<String, Iterable<File>>>,
    artifactNameAppendix: String
 ): TaskProvider<Jar> {
    project.locateTask<Jar>(taskName)?.let {
@@ -307,7 +307,7 @@ internal fun addCommonSourcesToKotlinCompileTask(
    project: Project,
    taskName: String,
    sourceFileExtensions: Iterable<String>,
-    sources: () -> Iterable<File>
+    sources: () -> Any
 ) = addSourcesToKotlinCompileTask(project, taskName, sourceFileExtensions, lazyOf(true), sources)

 // FIXME this function dangerously ignores an incorrect type of the task (e.g. if the actual task is a K/N one); consider reporting a failure
@@ -316,13 +316,18 @@ internal fun addSourcesToKotlinCompileTask(
    taskName: String,
    sourceFileExtensions: Iterable<String>,
    addAsCommonSources: Lazy<Boolean> = lazyOf(false),
-    sources: () -> Iterable<File>
+    /** Evaluated as project.files(...) */
+    sources: () -> Any
 ) {
    fun AbstractKotlinCompile<*>.configureAction() {
-        source(project.files(Callable { sources() }))
+        // In this call, the super-implementation of `source` adds the directories files to the roots of the union file tree,
+        // so it's OK to pass just the source roots.
+        source(Callable(sources))
        sourceFilesExtensions(sourceFileExtensions)
-        commonSourceSet += project.files(Callable {
-            if (addAsCommonSources.value) sources() else emptyList<Any>()
+
+        // The `commonSourceSet` is passed to the compiler as-is, converted with toList
+        commonSourceSet += project.files(Callable<Any> {
+            if (addAsCommonSources.value) sources else emptyList<Any>()
        })
    }

@@ -7,37 +7,47 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20

 import org.gradle.api.Project
 import org.gradle.api.file.FileCollection
+import org.gradle.api.provider.Provider
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.jetbrains.kotlin.project.model.utils.variantsContainingFragment
+import java.io.File

+typealias SourceRoots = Iterable<File>
+typealias SourceRootsProvider = Provider<out SourceRoots>
+typealias MultipleSourceRootsProvider = Provider<out Iterable<SourceRootsProvider>>
+typealias SourceRootsProvidersByFragment = Map<KotlinGradleFragment, SourceRootsProvider>
+
+/** Note: the API is [Provider]-based rather than FileCollection-based because FileCollection API erases the internal structure of the
+ * file sets, and this internal structure is currently needed for correctly inferring Java source roots from the sources added to the
+ * JVM compilations (it is important to pass sources in SourceDirectorySets) */
 open class FragmentSourcesProvider {
    protected open fun getSourcesFromFragmentsAsMap(
        fragments: Iterable<KotlinGradleFragment>
-    ): Map<KotlinGradleFragment, FileCollection> =
-        fragments.associateWith { it.project.files(it.project.provider { it.kotlinSourceRoots }) }
+    ): SourceRootsProvidersByFragment =
+        fragments.associateWith { it.project.provider { it.kotlinSourceRoots } }

-    open fun getFragmentOwnSources(fragment: KotlinGradleFragment): FileCollection =
+    open fun getFragmentOwnSources(fragment: KotlinGradleFragment): SourceRootsProvider =
        getSourcesFromFragmentsAsMap(listOf(fragment)).values.single()

-    open fun getAllFragmentSourcesAsMap(module: KotlinGradleModule): Map<KotlinGradleFragment, FileCollection> =
+    open fun getAllFragmentSourcesAsMap(module: KotlinGradleModule): SourceRootsProvidersByFragment =
        getSourcesFromFragmentsAsMap(module.fragments)

-    open fun getSourcesFromRefinesClosureAsMap(fragment: KotlinGradleFragment): Map<KotlinGradleFragment, FileCollection> =
+    open fun getSourcesFromRefinesClosureAsMap(fragment: KotlinGradleFragment): SourceRootsProvidersByFragment =
        getSourcesFromFragmentsAsMap(fragment.refinesClosure)

-    open fun getSourcesFromRefinesClosure(fragment: KotlinGradleFragment): FileCollection =
-        fragment.project.files(fragment.project.provider { getSourcesFromRefinesClosureAsMap(fragment).values })
+    open fun getSourcesFromRefinesClosure(fragment: KotlinGradleFragment): MultipleSourceRootsProvider =
+        fragment.project.provider { getSourcesFromRefinesClosureAsMap(fragment).values }

-    open fun getCommonSourcesFromRefinesClosure(fragment: KotlinGradleFragment): FileCollection {
+    open fun getCommonSourcesFromRefinesClosure(fragment: KotlinGradleFragment): MultipleSourceRootsProvider {
        val containingModule = fragment.containingModule
        val project = containingModule.project
        getSourcesFromRefinesClosureAsMap(fragment)
-        return project.files(project.provider {
+        return project.provider {
            fragment.refinesClosure.filter {
                // Every fragment refined by some other fragment should be considered common, even if it is included in just one variant
                containingModule.variantsContainingFragment(it).toSet() != setOf(it)
-            }.map { it.kotlinSourceRoots }
-        })
+            }.map { project.provider { it.kotlinSourceRoots } }
+        }
    }
 }

@@ -22,10 +22,10 @@ open class KotlinCompilationTaskConfigurator(
 ) {
    open val fragmentSourcesProvider: FragmentSourcesProvider = FragmentSourcesProvider()

-    open fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment) =
+    open fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider =
        fragmentSourcesProvider.getSourcesFromRefinesClosure(fragment)

-    open fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment) =
+    open fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider =
        fragmentSourcesProvider.getCommonSourcesFromRefinesClosure(fragment)

    fun createKotlinJvmCompilationTask(
@@ -51,7 +51,7 @@ abstract class AbstractKotlinGradleVariantFactory<T : KotlinGradleVariant>(
        sourcesJarTaskNamed(
            fragment.sourceArchiveTaskName,
            project,
-            lazy { FragmentSourcesProvider().getSourcesFromRefinesClosureAsMap(fragment).mapKeys { it.key.fragmentName } },
+            lazy { FragmentSourcesProvider().getSourcesFromRefinesClosureAsMap(fragment).entries.associate { it.key.fragmentName to it.value.get() } },
            fragment.name
        )
    }
@@ -9,6 +9,7 @@ import org.gradle.api.DefaultTask
 import org.gradle.api.Project
 import org.gradle.api.attributes.Usage
 import org.gradle.api.file.FileCollection
+import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.TaskProvider
 import org.gradle.jvm.tasks.Jar
 import org.jetbrains.kotlin.gradle.dsl.pm20Extension
@@ -27,6 +28,7 @@ import org.jetbrains.kotlin.gradle.utils.dashSeparatedName
 import org.jetbrains.kotlin.gradle.utils.lowerCamelCaseName
 import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
 import org.jetbrains.kotlin.project.model.KotlinModuleFragment
+import java.io.File
 import java.util.concurrent.Callable

 internal fun configureMetadataResolutionAndBuild(module: KotlinGradleModule) {
@@ -65,7 +67,7 @@ internal fun configureMetadataExposure(module: KotlinGradleModule) {
    val sourcesArtifact = sourcesJarTaskNamed(
        module.disambiguateName("allSourcesJar"),
        project,
-        lazy { FragmentSourcesProvider().getAllFragmentSourcesAsMap(module).mapKeys { it.key.fragmentName } },
+        lazy { FragmentSourcesProvider().getAllFragmentSourcesAsMap(module).entries.associate { it.key.fragmentName to it.value.get() } },
        sourcesArtifactAppendix
    )
    DocumentationVariantConfigurator().createSourcesElementsConfiguration(
@@ -222,12 +224,12 @@ private class MetadataCompilationTasksConfigurator(project: Project) : KotlinCom
        addCommonSourcesToKotlinCompileTask(project, compilationData.compileKotlinTaskName, emptyList()) { commonSources }
    }

-    override fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment): FileCollection {
-        return fragmentSourcesProvider.getFragmentOwnSources(fragment)
+    override fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider {
+        return project.provider { listOf(fragmentSourcesProvider.getFragmentOwnSources(fragment)) }
    }

-    override fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment): FileCollection {
-        return fragmentSourcesProvider.getFragmentOwnSources(fragment)
+    override fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider {
+        return project.provider { listOf(fragmentSourcesProvider.getFragmentOwnSources(fragment)) }
    }
 }

@@ -1,14 +1,18 @@
 package org.jetbrains.kotlin.gradle.tasks

+import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTree
 import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.logging.Logger
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.provider.Provider
 import org.jetbrains.kotlin.gradle.logging.kotlinDebug
 import org.jetbrains.kotlin.gradle.utils.isJavaFile
 import org.jetbrains.kotlin.gradle.utils.isKotlinFile
 import org.jetbrains.kotlin.gradle.utils.isParentOf
 import java.io.File
 import java.util.*
+import java.util.concurrent.Callable

 internal sealed class SourceRoots(val kotlinSourceFiles: List<File>) {
    private companion object {
@@ -20,14 +24,21 @@ internal sealed class SourceRoots(val kotlinSourceFiles: List<File>) {
        logger.kotlinDebug { "$taskName source roots: ${dumpPaths(kotlinSourceFiles)}" }
    }

-    class ForJvm(kotlinSourceFiles: List<File>, val javaSourceRoots: Set<File>) : SourceRoots(kotlinSourceFiles) {
+    class ForJvm constructor(
+        kotlinSourceFiles: List<File>,
+        private val javaSourceRootsProvider: () -> Set<File>
+    ) : SourceRoots(kotlinSourceFiles) {
+        val javaSourceRoots: Set<File>
+            get() = javaSourceRootsProvider()
+
+        constructor(kotlinSourceFiles: List<File>, allSourceRoots: FileCollection, taskSource: FileCollection) : this(kotlinSourceFiles, {
+            findRootsForSources(allSourceRoots, taskSource.filter(File::isJavaFile)).toSet()
+        })
+
        companion object {
            fun create(taskSource: FileTree, sourceRoots: FilteringSourceRootsContainer, sourceFilesExtensions: List<String>): ForJvm {
                val kotlinSourceFiles = (taskSource as Iterable<File>).filter { it.isKotlinFile(sourceFilesExtensions) }
-                val javaSourceRoots = findRootsForSources(
-                    sourceRoots.sourceRoots, taskSource.filter(File::isJavaFile)
-                )
-                return ForJvm(kotlinSourceFiles, javaSourceRoots)
+                return ForJvm(kotlinSourceFiles, sourceRoots.sourceRoots, taskSource)
            }

            private fun findRootsForSources(allSourceRoots: Iterable<File>, sources: Iterable<File>): Set<File> {
@@ -60,33 +71,43 @@ internal sealed class SourceRoots(val kotlinSourceFiles: List<File>) {
    }
 }

-internal class FilteringSourceRootsContainer(roots: List<File> = emptyList(), val filter: (File) -> Boolean = { true }) {
-    private val mutableSourceRoots = roots.filterTo(mutableListOf(), filter)
+internal class FilteringSourceRootsContainer(
+    private val objectFactory: ObjectFactory,
+    val filter: (File) -> Boolean = { true }
+) {
+    private val sourceContainers: MutableList<Any> = mutableListOf()

-    val sourceRoots: List<File>
-        get() = mutableSourceRoots
+    private fun getFilteredSourceRootsFrom(any: Any) = objectFactory.fileCollection().from(Callable {
+        val resultItems = mutableListOf<Any>()
+        fun getRootsFrom(item: Any?) {
+            when (item) {
+                is SourceDirectorySet -> resultItems.add(item.sourceDirectories)
+                is Callable<*> -> getRootsFrom(item.call())
+                is Provider<*> -> if (item.isPresent) getRootsFrom(item.get())
+                is FileCollection -> resultItems.add(item)
+                is Iterable<*> -> item.forEach { getRootsFrom(it) }
+                is Array<*> -> item.forEach { getRootsFrom(it) }
+                is Any /* not null */ -> resultItems.add(item)
+            }
+        }
+        getRootsFrom(any)
+        resultItems
+    }).filter(filter)
+
+    val sourceRoots: FileCollection
+        get() = getFilteredSourceRootsFrom(sourceContainers)

    fun clear() {
-        mutableSourceRoots.clear()
+        sourceContainers.clear()
    }

-    fun set(source: Any?): List<File> {
+    fun set(source: Any): FileCollection {
        clear()
        return add(source)
    }

-    fun add(vararg sources: Any?): List<File> {
-        val filteredDirs = mutableListOf<File>()
-        for (source in sources) {
-            when (source) {
-                is SourceDirectorySet -> filteredDirs += source.srcDirs.filter { filter(it) }
-                is File -> if (filter(source)) filteredDirs.add(source)
-                is Collection<*> -> source.forEach { filteredDirs += add(it) }
-                is Array<*> -> source.forEach { filteredDirs += add(it) }
-            }
-        }
-
-        mutableSourceRoots += filteredDirs
-        return filteredDirs
+    fun add(vararg sources: Any): FileCollection {
+        sourceContainers.addAll(sources.toList())
+        return getFilteredSourceRootsFrom(sources)
    }
 }
@@ -471,7 +471,13 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
        get() = taskData.compilation.kotlinOptions as KotlinJvmOptions

    @get:Internal
-    internal open val sourceRootsContainer = FilteringSourceRootsContainer()
+    @field:Transient
+    internal open val sourceRootsContainer = FilteringSourceRootsContainer(project.objects)
+
+    private val jvmSourceRoots by project.provider {
+        // serialize in the task state for configuration caching; avoid building anew in task execution, as it may access the project model
+        SourceRoots.ForJvm.create(source, sourceRootsContainer, sourceFilesExtensions)
+    }

    /** A package prefix that is used for locating Java sources in a directory structure with non-full-depth packages.
     *
@@ -512,8 +518,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
        KotlinJvmCompilerArgumentsContributor(KotlinJvmCompilerArgumentsProvider(this))
    }

-    @Internal
-    override fun getSourceRoots() = SourceRoots.ForJvm.create(getSource(), sourceRootsContainer, sourceFilesExtensions)
+    override fun getSourceRoots(): SourceRoots.ForJvm = jvmSourceRoots

    override fun callCompilerAsync(args: K2JVMCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) {
        sourceRoots as SourceRoots.ForJvm