[Gradle][MPP] Implement cinterop metadata dependency transformation

^KT-46198 Verification Pending

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/CompositeMetadataJar.kt

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.mpp
+
+import org.jetbrains.kotlin.gradle.utils.copyZipFilePartially
+import org.jetbrains.kotlin.gradle.utils.listChildren
+import java.io.File
+import java.util.zip.ZipFile
+
+internal fun CompositeMetadataJar(
+    moduleIdentifier: String,
+    projectStructureMetadata: KotlinProjectStructureMetadata,
+    primaryArtifactFile: File,
+    hostSpecificArtifactsBySourceSet: Map<String, File>,
+): CompositeMetadataJar = CompositeMetadataJarImpl(
+    moduleIdentifier, projectStructureMetadata, primaryArtifactFile, hostSpecificArtifactsBySourceSet
+)
+
+internal interface CompositeMetadataJar {
+    fun getSourceSetCompiledMetadata(
+        sourceSetName: String, outputDirectory: File, materializeFile: Boolean
+    ): File
+
+    fun getSourceSetCInteropMetadata(
+        sourceSetName: String, outputDirectory: File, materializeFiles: Boolean
+    ): Set<File>
+}
+
+private class CompositeMetadataJarImpl(
+    private val moduleIdentifier: String,
+    private val projectStructureMetadata: KotlinProjectStructureMetadata,
+    private val primaryArtifactFile: File,
+    private val hostSpecificArtifactsBySourceSet: Map<String, File>,
+) : CompositeMetadataJar {
+
+    override fun getSourceSetCompiledMetadata(
+        sourceSetName: String, outputDirectory: File, materializeFile: Boolean
+    ): File {
+        val artifactFile = getArtifactFile(sourceSetName)
+        val moduleOutputDirectory = outputDirectory.resolve(moduleIdentifier).also { it.mkdirs() }
+
+        val extension = projectStructureMetadata.sourceSetBinaryLayout[sourceSetName]?.archiveExtension
+            ?: SourceSetMetadataLayout.METADATA.archiveExtension
+
+        val metadataOutputFile = moduleOutputDirectory.resolve("$moduleIdentifier-$sourceSetName.$extension")
+
+        /** In composite builds, we don't really need tro process the file in IDE import, so ignore it if it's missing */
+        // refactor: allow only included builds to provide no artifacts, and allow this only in IDE import
+        if (!artifactFile.isFile) {
+            return metadataOutputFile
+        }
+
+        if (!materializeFile) {
+            return metadataOutputFile
+        }
+
+        if (metadataOutputFile.exists()) {
+            metadataOutputFile.delete()
+        }
+
+        copyZipFilePartially(artifactFile, metadataOutputFile, "$sourceSetName/")
+        return metadataOutputFile
+    }
+
+    override fun getSourceSetCInteropMetadata(
+        sourceSetName: String, outputDirectory: File, materializeFiles: Boolean
+    ): Set<File> {
+        val artifactFile = getArtifactFile(sourceSetName)
+        val moduleOutputDirectory = outputDirectory.resolve(moduleIdentifier).also(File::mkdirs)
+
+        ZipFile(getArtifactFile(sourceSetName)).use { compoundMetadataArtifactZipFile ->
+            val cinteropRootDirectory = compoundMetadataArtifactZipFile.entries().asSequence()
+                .firstOrNull { zipEntry -> zipEntry.name == "$sourceSetName-cinterop/" && zipEntry.isDirectory }
+                ?: return emptySet()
+
+            val cinterops = compoundMetadataArtifactZipFile.listChildren(cinteropRootDirectory)
+
+            val cinteropsByOutputFile = cinterops.associateBy { cinteropZipEntry ->
+                moduleOutputDirectory.resolve("${cinteropZipEntry.name.removePrefix(cinteropRootDirectory.name).removeSuffix("/")}.klib")
+            }
+
+            if (materializeFiles) {
+                cinteropsByOutputFile.forEach { (cinteropOutputFile, cinteropZipEntry) ->
+                    copyZipFilePartially(artifactFile, cinteropOutputFile, cinteropZipEntry.name)
+                }
+            }
+
+            return cinteropsByOutputFile.keys
+        }
+    }
+
+    private fun getArtifactFile(sourceSetName: String): File =
+        hostSpecificArtifactsBySourceSet[sourceSetName] ?: primaryArtifactFile
+}
+

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GetAllCompiledSourceSetMetadata.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.mpp
+
+import org.gradle.api.Project
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.file.FileCollection
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider
+import java.io.File
+
+/**
+ * @param outputDirectoryWhenMaterialised: Root output directory to put files if [materializeFilesIfNecessary] is set
+ * and the artifact is indeed a [CompositeMetadataJar]
+ *
+ * @param materializeFilesIfNecessary: If the artifact is a [CompositeMetadataJar] then this flag will tell if
+ * compiled source set metadata should be extracted from this jar file into the given [outputDirectoryWhenMaterialised]
+ */
+internal fun ChooseVisibleSourceSets.getAllCompiledSourceSetMetadata(
+    project: Project,
+    outputDirectoryWhenMaterialised: File,
+    materializeFilesIfNecessary: Boolean
+): ConfigurableFileCollection = project.files(
+    visibleSourceSetNamesExcludingDependsOn.map { visibleSourceSetName ->
+        metadataProvider.getSourceSetCompiledMetadata(
+            project, visibleSourceSetName, outputDirectoryWhenMaterialised, materializeFilesIfNecessary
+        )
+    }
+)
+
+internal fun ChooseVisibleSourceSets.MetadataProvider.getSourceSetCompiledMetadata(
+    project: Project,
+    sourceSetName: String,
+    outputDirectoryWhenMaterialised: File,
+    materializeFilesIfNecessary: Boolean
+): FileCollection = when (this) {
+    is ProjectMetadataProvider -> getSourceSetCompiledMetadata(sourceSetName)
+    is JarMetadataProvider -> project.files(
+        getSourceSetCompiledMetadata(
+            sourceSetName, outputDirectoryWhenMaterialised, materializeFilesIfNecessary
+        )
+    )
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt

@@ -13,24 +13,14 @@ import org.gradle.api.artifacts.result.ResolvedComponentResult
 import org.gradle.api.artifacts.result.ResolvedDependencyResult
 import org.gradle.api.attributes.Attribute
 import org.gradle.api.file.FileCollection
-import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
-import org.jetbrains.kotlin.gradle.dsl.topLevelExtension
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
-import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinPm20ProjectExtension
-import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.toSingleModuleIdentifier
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.Companion.asMetadataProvider
 import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope
 import org.jetbrains.kotlin.gradle.plugin.sources.sourceSetDependencyConfigurationByScope
 import org.jetbrains.kotlin.gradle.targets.metadata.ALL_COMPILE_METADATA_CONFIGURATION_NAME
 import org.jetbrains.kotlin.gradle.targets.metadata.ALL_RUNTIME_METADATA_CONFIGURATION_NAME
 import org.jetbrains.kotlin.gradle.targets.metadata.dependsOnClosureWithInterCompilationDependencies
-import org.jetbrains.kotlin.project.model.KotlinModuleIdentifier
-import java.io.File
-import java.io.InputStream
 import java.util.*
-import java.util.zip.ZipEntry
-import java.util.zip.ZipFile
-import java.util.zip.ZipOutputStream
-import javax.xml.parsers.DocumentBuilderFactory
 
 internal sealed class MetadataDependencyResolution(
     @field:Transient // can't be used with Gradle Instant Execution, but fortunately not needed when deserialized
@@ -60,23 +50,31 @@ internal sealed class MetadataDependencyResolution(
         projectDependency: Project?
     ) : MetadataDependencyResolution(dependency, projectDependency)
 
-    abstract class ChooseVisibleSourceSets(
+    class ChooseVisibleSourceSets internal constructor(
         dependency: ResolvedComponentResult,
         projectDependency: Project?,
         val projectStructureMetadata: KotlinProjectStructureMetadata,
         val allVisibleSourceSetNames: Set<String>,
         val visibleSourceSetNamesExcludingDependsOn: Set<String>,
-        val visibleTransitiveDependencies: Set<ResolvedDependencyResult>
+        val visibleTransitiveDependencies: Set<ResolvedDependencyResult>,
+        internal val metadataProvider: MetadataProvider
     ) : MetadataDependencyResolution(dependency, projectDependency) {
-        /** Returns the mapping of source set names to files which should be used as the [dependency] parts representing the source sets.
-         * If any temporary files need to be created, their paths are built from the [baseDir].
-         * If [createFiles] is true, these temporary files are actually re-created during the call,
-         * otherwise only their paths are returned, while the files might be missing.
-         */
-        fun getMetadataFilesBySourceSet(baseDir: File, createFiles: Boolean): Map<String, FileCollection> =
-            getExtractableMetadataFiles(baseDir).getMetadataFilesPerSourceSet(createFiles)
 
-        abstract fun getExtractableMetadataFiles(baseDir: File): ExtractableMetadataFiles
+        internal sealed class MetadataProvider {
+            class JarMetadataProvider(private val compositeMetadataArtifact: CompositeMetadataJar) :
+                MetadataProvider(), CompositeMetadataJar by compositeMetadataArtifact
+
+            abstract class ProjectMetadataProvider : MetadataProvider() {
+                enum class MetadataConsumer { Ide, Cli }
+
+                abstract fun getSourceSetCompiledMetadata(sourceSetName: String): FileCollection
+                abstract fun getSourceSetCInteropMetadata(sourceSetName: String, consumer: MetadataConsumer): FileCollection
+            }
+
+            companion object {
+                fun CompositeMetadataJar.asMetadataProvider() = JarMetadataProvider(this)
+            }
+        }
 
         override fun toString(): String =
             super.toString() + ", sourceSets = " + allVisibleSourceSetNames.joinToString(", ", "[", "]") {
@@ -109,7 +107,7 @@ internal class GranularMetadataTransformation(
     private val allSourceSetsConfiguration: Configuration =
         commonMetadataDependenciesConfigurationForScopes(project, sourceSetRequestedScopes)
 
-    private val configurationToResolve: Configuration by lazy {
+    internal val configurationToResolve: Configuration by lazy {
         resolvableMetadataConfiguration(project, allSourceSetsConfiguration, requestedDependencies)
     }
 
@@ -204,10 +202,8 @@ internal class GranularMetadataTransformation(
         parentResolutionsForModule: Iterable<MetadataDependencyResolution>,
         parent: ResolvedComponentResult?
     ): MetadataDependencyResolution {
-        val mppDependencyMetadataExtractor = getMetadataExtractor(
-            project,
-            module,
-            configurationToResolve,
+        val mppDependencyMetadataExtractor = MppDependencyProjectStructureMetadataExtractor.create(
+            project, module, configurationToResolve,
             resolveViaAvailableAt = false // we will process the available-at module as a dependency later in the queue
         )
 
@@ -225,10 +221,6 @@ internal class GranularMetadataTransformation(
                 resolvedToProject
             )
 
-        if (mppDependencyMetadataExtractor is JarArtifactMppDependencyMetadataExtractor) {
-            mppDependencyMetadataExtractor.metadataArtifactBySourceSet.putAll(sourceSetVisibility.hostSpecificMetadataArtifactBySourceSet)
-        }
-
         val allVisibleSourceSets = sourceSetVisibility.visibleSourceSetNames
 
         val sourceSetsVisibleInParents = parentResolutionsForModule
@@ -253,33 +245,32 @@ internal class GranularMetadataTransformation(
 
         val visibleSourceSetsExcludingDependsOn = allVisibleSourceSets.filterTo(mutableSetOf()) { it !in sourceSetsVisibleInParents }
 
-        return ChooseVisibleSourceSetsImpl(
-            module, resolvedToProject, projectStructureMetadata, allVisibleSourceSets, visibleSourceSetsExcludingDependsOn,
-            transitiveDependenciesToVisit, mppDependencyMetadataExtractor
+        val metadataProvider = when (mppDependencyMetadataExtractor) {
+            is ProjectMppDependencyProjectStructureMetadataExtractor -> ProjectMetadataProvider(
+                dependencyProject = mppDependencyMetadataExtractor.dependencyProject,
+                moduleIdentifier = mppDependencyMetadataExtractor.moduleIdentifier
+            )
+
+            is JarMppDependencyProjectStructureMetadataExtractor -> CompositeMetadataJar(
+                moduleIdentifier = ModuleIds.fromComponent(project, module).toString(),
+                projectStructureMetadata = projectStructureMetadata,
+                primaryArtifactFile = mppDependencyMetadataExtractor.primaryArtifactFile,
+                hostSpecificArtifactsBySourceSet = sourceSetVisibility.hostSpecificMetadataArtifactBySourceSet,
+            ).asMetadataProvider()
+        }
+
+        return MetadataDependencyResolution.ChooseVisibleSourceSets(
+            dependency = module,
+            projectDependency = resolvedToProject,
+            projectStructureMetadata = projectStructureMetadata,
+            allVisibleSourceSetNames = allVisibleSourceSets,
+            visibleSourceSetNamesExcludingDependsOn = visibleSourceSetsExcludingDependsOn,
+            visibleTransitiveDependencies = transitiveDependenciesToVisit,
+            metadataProvider = metadataProvider
         )
     }
 }
 
-internal class ChooseVisibleSourceSetsImpl(
-    dependency: ResolvedComponentResult,
-    projectDependency: Project?,
-    projectStructureMetadata: KotlinProjectStructureMetadata,
-    allVisibleSourceSetNames: Set<String>,
-    visibleSourceSetNamesExcludingDependsOn: Set<String>,
-    visibleTransitiveDependencies: Set<ResolvedDependencyResult>,
-    private val metadataExtractor: MppDependencyMetadataExtractor
-) : MetadataDependencyResolution.ChooseVisibleSourceSets(
-    dependency,
-    projectDependency,
-    projectStructureMetadata,
-    allVisibleSourceSetNames,
-    visibleSourceSetNamesExcludingDependsOn,
-    visibleTransitiveDependencies
-) {
-    override fun getExtractableMetadataFiles(baseDir: File): ExtractableMetadataFiles =
-        metadataExtractor.getExtractableMetadataFiles(visibleSourceSetNamesExcludingDependsOn, baseDir)
-}
-
 internal fun ResolvedComponentResult.toProjectOrNull(currentProject: Project): Project? {
     val identifier = id
     return when {
@@ -354,283 +345,3 @@ internal fun requestedDependencies(
     val otherContributingSourceSets = dependsOnClosureWithInterCompilationDependencies(project, sourceSet)
     return listOf(sourceSet, *otherContributingSourceSets.toTypedArray()).flatMap(::collectScopedDependenciesFromSourceSet)
 }
-
-
-internal abstract class MppDependencyMetadataExtractor(val project: Project, val component: ResolvedComponentResult) {
-    abstract fun getProjectStructureMetadata(): KotlinProjectStructureMetadata?
-
-    abstract fun getExtractableMetadataFiles(
-        visibleSourceSetNames: Set<String>,
-        baseDir: File
-    ): ExtractableMetadataFiles
-}
-
-private class ProjectMppDependencyMetadataExtractor(
-    project: Project,
-    dependency: ResolvedComponentResult,
-    val moduleIdentifier: KotlinModuleIdentifier,
-    val dependencyProject: Project
-) : MppDependencyMetadataExtractor(project, dependency) {
-    override fun getProjectStructureMetadata(): KotlinProjectStructureMetadata? {
-        return when (val topLevelExtension = dependencyProject.topLevelExtension) {
-            is KotlinPm20ProjectExtension -> buildProjectStructureMetadata(
-                topLevelExtension.modules.single { it.moduleIdentifier == moduleIdentifier }
-            )
-            else -> buildKotlinProjectStructureMetadata(dependencyProject)
-        }
-    }
-
-    override fun getExtractableMetadataFiles(
-        visibleSourceSetNames: Set<String>,
-        baseDir: File
-    ): ExtractableMetadataFiles {
-        val result = when (val projectExtension = dependencyProject.topLevelExtension) {
-            is KotlinMultiplatformExtension -> projectExtension.targets.getByName(KotlinMultiplatformPlugin.METADATA_TARGET_NAME).compilations
-                .filter { it.name in visibleSourceSetNames }.associate { it.defaultSourceSet.name to it.output.classesDirs }
-            is KotlinPm20ProjectExtension -> {
-                val moduleId = moduleIdentifier
-                val module = projectExtension.modules.single { it.moduleIdentifier == moduleId }
-                val metadataCompilationRegistry = projectExtension.metadataCompilationRegistryByModuleId.getValue(moduleId)
-                visibleSourceSetNames.associateWith {
-                    metadataCompilationRegistry.byFragment(module.fragments.getByName(it)).output.classesDirs
-                }
-            }
-            else -> error("unexpected top-level Kotlin extension $projectExtension")
-        }
-
-        return object : ExtractableMetadataFiles() {
-            override fun getMetadataFilesPerSourceSet(createFiles: Boolean): Map<String, FileCollection> = result
-        }
-    }
-}
-
-private class IncludedBuildMetadataExtractor(
-    project: Project,
-    dependency: ResolvedComponentResult,
-    primaryArtifact: File
-) : JarArtifactMppDependencyMetadataExtractor(project, dependency, primaryArtifact) {
-
-    private val id: ProjectComponentIdentifier
-
-    init {
-        val id = dependency.id
-        require(id is ProjectComponentIdentifier) { "dependency should resolve to a project" }
-        require(!id.build.isCurrentBuild) { "should be a project from an included build" }
-        this.id = id
-    }
-
-    override fun getProjectStructureMetadata(): KotlinProjectStructureMetadata? =
-        GlobalProjectStructureMetadataStorage.getProjectStructureMetadata(project, id.build.name, id.projectPath)
-}
-
-internal open class JarArtifactMppDependencyMetadataExtractor(
-    project: Project,
-    dependency: ResolvedComponentResult,
-    val primaryArtifact: File
-) : MppDependencyMetadataExtractor(project, dependency) {
-
-    // TODO: add proper API to make an artifact extractor with just the composite artifact "evolve" into one with host-specific artifacts
-    val metadataArtifactBySourceSet: MutableMap<String, File> = mutableMapOf()
-
-    private fun parseJsonProjectStructureMetadata(input: InputStream) =
-        parseKotlinSourceSetMetadataFromJson(input.reader().readText())
-
-    private fun parseXmlProjectStructureMetadata(input: InputStream) =
-        parseKotlinSourceSetMetadataFromXml(DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input))
-
-    override fun getProjectStructureMetadata(): KotlinProjectStructureMetadata? {
-        return ZipFile(primaryArtifact).use { zip ->
-            val (metadata, parseFunction) =
-                zip.getEntry("META-INF/$MULTIPLATFORM_PROJECT_METADATA_JSON_FILE_NAME")?.to(::parseJsonProjectStructureMetadata)
-                    ?: zip.getEntry("META-INF/$MULTIPLATFORM_PROJECT_METADATA_FILE_NAME")?.to(::parseXmlProjectStructureMetadata)
-                    ?: return null
-
-            zip.getInputStream(metadata).use(parseFunction)
-        }
-    }
-
-    override fun getExtractableMetadataFiles(
-        visibleSourceSetNames: Set<String>,
-        baseDir: File
-    ): ExtractableMetadataFiles {
-        val primaryArtifact = primaryArtifact
-        val moduleId = ModuleIds.fromComponent(project, component)
-
-        return JarExtractableMetadataFiles(
-            module = moduleId,
-            project = project,
-            baseDir = baseDir,
-            artifactBySourceSet = visibleSourceSetNames.associateWith { (metadataArtifactBySourceSet[it] ?: primaryArtifact) },
-            projectStructureMetadata = checkNotNull(getProjectStructureMetadata()) {
-                "project structure metadata is needed to extract files"
-            }
-        )
-    }
-
-    private class JarExtractableMetadataFiles(
-        private val module: ModuleDependencyIdentifier,
-        private val project: Project,
-        private val baseDir: File,
-        private val artifactBySourceSet: Map<String, File>,
-        private val projectStructureMetadata: KotlinProjectStructureMetadata
-    ) : ExtractableMetadataFiles() {
-
-        override fun getMetadataFilesPerSourceSet(createFiles: Boolean): Map<String, FileCollection> {
-            return artifactBySourceSet.mapValues { (sourceSetName, artifact) ->
-                project.files(
-                    getCompiledSourceSetMetadata(sourceSetName, artifact, createFiles)
-                )
-            }
-        }
-
-        private fun getCompiledSourceSetMetadata(sourceSetName: String, artifact: File, createFiles: Boolean): File {
-            val moduleString = "${module.groupId}-${module.moduleId}"
-            val transformedModuleRoot = run { baseDir.resolve(moduleString).also { it.mkdirs() } }
-
-            val extension = projectStructureMetadata.sourceSetBinaryLayout[sourceSetName]?.archiveExtension
-                ?: SourceSetMetadataLayout.METADATA.archiveExtension
-
-            val metadataOutputFile = transformedModuleRoot.resolve("$moduleString-$sourceSetName.$extension")
-
-            /** In composite builds, we don't really need tro process the file in IDE import, so ignore it if it's missing */
-            // refactor: allow only included builds to provide no artifacts, and allow this only in IDE import
-            if (!artifact.isFile) {
-                return metadataOutputFile
-            }
-
-            if (!createFiles) {
-                return metadataOutputFile
-            }
-
-            if (metadataOutputFile.exists()) {
-                metadataOutputFile.delete()
-            }
-
-            ZipFile(artifact).use { zip ->
-                val entries = zip.entries().asSequence().filter { it.name.startsWith("$sourceSetName/") }.toList()
-
-                // TODO: once IJ supports non-JAR metadata dependencies, extract to a directory, not a JAR
-                // Also, if both IJ and the CLI compiler can read metadata from a path inside a JAR, then no operations will be needed
-
-                if (entries.any()) {
-                    ZipOutputStream(metadataOutputFile.outputStream()).use { resultZipOutput ->
-                        for (entry in entries) {
-                            if (entry.isDirectory)
-                                continue
-
-                            // Drop the source set name from the entry path
-                            val resultEntry = ZipEntry(entry.name.substringAfter("/"))
-
-                            zip.getInputStream(entry).use { inputStream ->
-                                resultZipOutput.putNextEntry(resultEntry)
-                                inputStream.copyTo(resultZipOutput)
-                                resultZipOutput.closeEntry()
-                            }
-                        }
-                    }
-                }
-            }
-
-            return metadataOutputFile
-        }
-    }
-}
-
-internal fun getMetadataExtractor(
-    project: Project,
-    resolvedComponentResult: ResolvedComponentResult,
-    configuration: Configuration,
-    resolveViaAvailableAt: Boolean
-): MppDependencyMetadataExtractor? {
-    val resolvedMppVariantsProvider = ResolvedMppVariantsProvider.get(project)
-    val moduleIdentifier =
-        resolvedComponentResult.toSingleModuleIdentifier() // FIXME this loses information about auxiliary module deps
-    // TODO check how this code works with multi-capability resolutions
-
-    return mppDependencyMetadataExtractor(
-        resolvedMppVariantsProvider,
-        moduleIdentifier,
-        configuration,
-        resolveViaAvailableAt,
-        resolvedComponentResult,
-        project
-    )
-}
-
-internal fun getMetadataExtractor(
-    project: Project,
-    resolvedComponentResult: ResolvedComponentResult,
-    moduleIdentifier: KotlinModuleIdentifier,
-    configuration: Configuration
-): MppDependencyMetadataExtractor? {
-    val resolvedMppVariantsProvider = ResolvedMppVariantsProvider.get(project)
-    return mppDependencyMetadataExtractor(
-        resolvedMppVariantsProvider,
-        moduleIdentifier,
-        configuration,
-        true,
-        resolvedComponentResult,
-        project
-    )
-}
-
-private fun mppDependencyMetadataExtractor(
-    resolvedMppVariantsProvider: ResolvedMppVariantsProvider,
-    moduleIdentifier: KotlinModuleIdentifier,
-    configuration: Configuration,
-    resolveViaAvailableAt: Boolean,
-    resolvedComponentResult: ResolvedComponentResult,
-    project: Project
-): MppDependencyMetadataExtractor? {
-    var resolvedViaAvailableAt = false
-
-    val metadataArtifact = resolvedMppVariantsProvider.getResolvedArtifactByPlatformModule(
-        moduleIdentifier,
-        configuration
-    ) ?: if (resolveViaAvailableAt) {
-        resolvedMppVariantsProvider.getHostSpecificMetadataArtifactByRootModule(
-            moduleIdentifier,
-            configuration
-        )?.also {
-            resolvedViaAvailableAt = true
-        }
-    } else null
-
-    val actualComponent = if (resolvedViaAvailableAt) {
-        resolvedComponentResult.dependencies.filterIsInstance<ResolvedDependencyResult>().singleOrNull()?.selected
-            ?: resolvedComponentResult
-    } else resolvedComponentResult
-
-    val moduleId = actualComponent.id
-    return when {
-        moduleId is ProjectComponentIdentifier -> when {
-            moduleId.build.isCurrentBuild ->
-                ProjectMppDependencyMetadataExtractor(project, actualComponent, moduleIdentifier, project.project(moduleId.projectPath))
-            metadataArtifact != null ->
-                IncludedBuildMetadataExtractor(project, actualComponent, metadataArtifact)
-            else -> null
-        }
-        metadataArtifact != null -> JarArtifactMppDependencyMetadataExtractor(project, actualComponent, metadataArtifact)
-        else -> null
-    }
-}
-
-internal fun getProjectStructureMetadata(
-    project: Project,
-    module: ResolvedComponentResult,
-    configuration: Configuration,
-    moduleIdentifier: KotlinModuleIdentifier? = null
-): KotlinProjectStructureMetadata? {
-    val extractor = if (moduleIdentifier != null)
-        getMetadataExtractor(project, module, moduleIdentifier, configuration)
-    else
-        getMetadataExtractor(project, module, configuration, resolveViaAvailableAt = true)
-
-    return extractor?.getProjectStructureMetadata()
-}
-
-// This class is needed to encapsulate how we extract the files and point to them in a way that doesn't capture the Gradle project state
-internal abstract class ExtractableMetadataFiles {
-    abstract fun getMetadataFilesPerSourceSet(createFiles: Boolean): Map<String, FileCollection>
-}
-

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinProjectStructureMetadata.kt

@@ -13,7 +13,6 @@ import org.gradle.api.Project
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.Nested
-import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
 import org.jetbrains.kotlin.gradle.dsl.topLevelExtensionOrNull
 import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinGradleModule
@@ -28,6 +27,7 @@ import org.w3c.dom.Document
 import org.w3c.dom.Element
 import org.w3c.dom.Node
 import org.w3c.dom.NodeList
+import java.io.Serializable
 import java.io.StringWriter
 import javax.xml.parsers.DocumentBuilderFactory
 
@@ -35,7 +35,7 @@ import javax.xml.parsers.DocumentBuilderFactory
 open class ModuleDependencyIdentifier(
     open val groupId: String?,
     open val moduleId: String
-) {
+) : Serializable {
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other !is ModuleDependencyIdentifier) return false
@@ -54,6 +54,10 @@ open class ModuleDependencyIdentifier(
 
     operator fun component1(): String? = groupId
     operator fun component2(): String = moduleId
+
+    override fun toString(): String {
+        return "${groupId}-${moduleId}"
+    }
 }
 
 class ChangingModuleDependencyIdentifier(
@@ -71,7 +75,7 @@ sealed class SourceSetMetadataLayout(
     val name: String,
     @get:Internal
     val archiveExtension: String
-) {
+) : Serializable {
     object METADATA : SourceSetMetadataLayout("metadata", "jar")
     object KLIB : SourceSetMetadataLayout("klib", "klib")
 
@@ -109,7 +113,7 @@ data class KotlinProjectStructureMetadata(
 
     @Input
     val formatVersion: String = FORMAT_VERSION_0_3_1
-) {
+) : Serializable {
     @Suppress("UNUSED") // Gradle input
     @get:Input
     internal val sourceSetModuleDependenciesInput: Map<String, Set<Pair<String, String>>>
@@ -296,7 +300,7 @@ internal fun parseKotlinSourceSetMetadataFromXml(document: Document): KotlinProj
     val nodeNamed: Element.(String) -> Element? = { name -> getElementsByTagName(name).elements.singleOrNull() }
     val valueNamed: Element.(String) -> String? =
         { name -> getElementsByTagName(name).run { if (length > 0) item(0).textContent else null } }
-    val multiObjects: Element.(String) -> Iterable<Element> = { name -> nodeNamed(name)?.childNodes?.elements ?: emptyList()}
+    val multiObjects: Element.(String) -> Iterable<Element> = { name -> nodeNamed(name)?.childNodes?.elements ?: emptyList() }
     val multiValues: Element.(String) -> Iterable<String> = { name -> getElementsByTagName(name).elements.map { it.textContent } }
 
     return parseKotlinSourceSetMetadata(

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MppDependencyProjectStructureMetadataExtractor.kt

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.mpp
+
+import org.gradle.api.Project
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier
+import org.gradle.api.artifacts.result.ResolvedComponentResult
+import org.jetbrains.kotlin.gradle.dsl.topLevelExtension
+import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinPm20ProjectExtension
+import org.jetbrains.kotlin.project.model.KotlinModuleIdentifier
+import java.io.File
+import java.io.InputStream
+import java.util.zip.ZipFile
+import javax.xml.parsers.DocumentBuilderFactory
+
+sealed class MppDependencyProjectStructureMetadataExtractor {
+    abstract fun getProjectStructureMetadata(): KotlinProjectStructureMetadata?
+
+    companion object Factory
+}
+
+internal class ProjectMppDependencyProjectStructureMetadataExtractor(
+    val moduleIdentifier: KotlinModuleIdentifier,
+    val dependencyProject: Project
+) : MppDependencyProjectStructureMetadataExtractor() {
+
+    override fun getProjectStructureMetadata(): KotlinProjectStructureMetadata? {
+        return when (val topLevelExtension = dependencyProject.topLevelExtension) {
+            is KotlinPm20ProjectExtension -> buildProjectStructureMetadata(
+                topLevelExtension.modules.single { it.moduleIdentifier == moduleIdentifier }
+            )
+            else -> buildKotlinProjectStructureMetadata(dependencyProject)
+        }
+    }
+}
+
+internal open class JarMppDependencyProjectStructureMetadataExtractor(
+    val primaryArtifactFile: File
+) : MppDependencyProjectStructureMetadataExtractor() {
+
+    private fun parseJsonProjectStructureMetadata(input: InputStream) =
+        parseKotlinSourceSetMetadataFromJson(input.reader().readText())
+
+    private fun parseXmlProjectStructureMetadata(input: InputStream) =
+        parseKotlinSourceSetMetadataFromXml(DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input))
+
+    override fun getProjectStructureMetadata(): KotlinProjectStructureMetadata? {
+        return ZipFile(primaryArtifactFile).use { zip ->
+            val (metadata, parseFunction) =
+                zip.getEntry("META-INF/$MULTIPLATFORM_PROJECT_METADATA_JSON_FILE_NAME")?.to(::parseJsonProjectStructureMetadata)
+                    ?: zip.getEntry("META-INF/$MULTIPLATFORM_PROJECT_METADATA_FILE_NAME")?.to(::parseXmlProjectStructureMetadata)
+                    ?: return null
+
+            zip.getInputStream(metadata).use(parseFunction)
+        }
+    }
+}
+
+internal class IncludedBuildMppDependencyProjectStructureMetadataExtractor(
+    private val project: Project,
+    dependency: ResolvedComponentResult,
+    primaryArtifact: File
+) : JarMppDependencyProjectStructureMetadataExtractor(primaryArtifact) {
+
+    private val id: ProjectComponentIdentifier
+
+    init {
+        val id = dependency.id
+        require(id is ProjectComponentIdentifier) { "dependency should resolve to a project" }
+        require(!id.build.isCurrentBuild) { "should be a project from an included build" }
+        this.id = id
+    }
+
+    override fun getProjectStructureMetadata(): KotlinProjectStructureMetadata? =
+        GlobalProjectStructureMetadataStorage.getProjectStructureMetadata(project, id.build.name, id.projectPath)
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MppDependencyProjectStructureMetadataExtractorFactory.kt

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.mpp
+
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier
+import org.gradle.api.artifacts.result.ResolvedComponentResult
+import org.gradle.api.artifacts.result.ResolvedDependencyResult
+import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.toSingleModuleIdentifier
+import org.jetbrains.kotlin.project.model.KotlinModuleIdentifier
+
+internal fun MppDependencyProjectStructureMetadataExtractor.Factory.create(
+    project: Project,
+    resolvedComponentResult: ResolvedComponentResult,
+    configuration: Configuration,
+    resolveViaAvailableAt: Boolean
+): MppDependencyProjectStructureMetadataExtractor? {
+    return create(
+        resolvedMppVariantsProvider = ResolvedMppVariantsProvider.get(project),
+        /*
+        FIXME this loses information about auxiliary module deps
+        TODO check how this code works with multi-capability resolutions,
+         */
+        moduleIdentifier = resolvedComponentResult.toSingleModuleIdentifier(),
+        configuration = configuration,
+        resolveViaAvailableAt = resolveViaAvailableAt,
+        resolvedComponentResult = resolvedComponentResult,
+        project = project
+    )
+}
+
+internal fun MppDependencyProjectStructureMetadataExtractor.Factory.create(
+    project: Project,
+    resolvedComponentResult: ResolvedComponentResult,
+    moduleIdentifier: KotlinModuleIdentifier,
+    configuration: Configuration
+): MppDependencyProjectStructureMetadataExtractor? {
+    return create(
+        resolvedMppVariantsProvider = ResolvedMppVariantsProvider.get(project),
+        moduleIdentifier = moduleIdentifier,
+        configuration = configuration,
+        resolveViaAvailableAt = true,
+        resolvedComponentResult = resolvedComponentResult,
+        project = project
+    )
+}
+
+private fun MppDependencyProjectStructureMetadataExtractor.Factory.create(
+    resolvedMppVariantsProvider: ResolvedMppVariantsProvider,
+    moduleIdentifier: KotlinModuleIdentifier,
+    configuration: Configuration,
+    resolveViaAvailableAt: Boolean,
+    resolvedComponentResult: ResolvedComponentResult,
+    project: Project
+): MppDependencyProjectStructureMetadataExtractor? {
+    var resolvedViaAvailableAt = false
+
+    val metadataArtifact = resolvedMppVariantsProvider.getResolvedArtifactByPlatformModule(
+        moduleIdentifier,
+        configuration
+    ) ?: if (resolveViaAvailableAt) {
+        resolvedMppVariantsProvider.getHostSpecificMetadataArtifactByRootModule(
+            moduleIdentifier, configuration
+        )?.also {
+            resolvedViaAvailableAt = true
+        }
+    } else null
+
+    val actualComponent = if (resolvedViaAvailableAt) {
+        resolvedComponentResult.dependencies.filterIsInstance<ResolvedDependencyResult>().singleOrNull()?.selected
+            ?: resolvedComponentResult
+    } else resolvedComponentResult
+
+    val moduleId = actualComponent.id
+    return when {
+        moduleId is ProjectComponentIdentifier -> when {
+            moduleId.build.isCurrentBuild ->
+                ProjectMppDependencyProjectStructureMetadataExtractor(moduleIdentifier, project.project(moduleId.projectPath))
+            metadataArtifact != null ->
+                IncludedBuildMppDependencyProjectStructureMetadataExtractor(project, actualComponent, metadataArtifact)
+            else -> null
+        }
+        metadataArtifact != null -> JarMppDependencyProjectStructureMetadataExtractor(metadataArtifact)
+        else -> null
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/ProjectMetadataProviderImpl.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.mpp
+
+import org.gradle.api.Project
+import org.gradle.api.file.FileCollection
+import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
+import org.jetbrains.kotlin.gradle.dsl.topLevelExtension
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinPm20ProjectExtension
+import org.jetbrains.kotlin.gradle.targets.native.internal.*
+import org.jetbrains.kotlin.project.model.KotlinModuleIdentifier
+
+internal fun ProjectMetadataProvider(
+    dependencyProject: Project,
+    moduleIdentifier: KotlinModuleIdentifier
+): ProjectMetadataProvider {
+    return ProjectMetadataProviderImpl(dependencyProject, moduleIdentifier)
+}
+
+private class ProjectMetadataProviderImpl(
+    private val dependencyProject: Project,
+    private val moduleIdentifier: KotlinModuleIdentifier
+) : ProjectMetadataProvider() {
+    override fun getSourceSetCompiledMetadata(sourceSetName: String): FileCollection {
+        return when (val projectExtension = dependencyProject.topLevelExtension) {
+            is KotlinMultiplatformExtension -> projectExtension.targets.getByName(KotlinMultiplatformPlugin.METADATA_TARGET_NAME).compilations
+                .firstOrNull { it.name == sourceSetName }
+                ?.output?.classesDirs ?: dependencyProject.files()
+
+            is KotlinPm20ProjectExtension -> {
+                val moduleId = moduleIdentifier
+                val module = projectExtension.modules.single { it.moduleIdentifier == moduleId }
+                val metadataCompilationRegistry = projectExtension.metadataCompilationRegistryByModuleId.getValue(moduleId)
+                metadataCompilationRegistry.byFragment(module.fragments.getByName(sourceSetName)).output.classesDirs
+            }
+            else -> error("unexpected top-level Kotlin extension $projectExtension")
+        }
+    }
+
+    override fun getSourceSetCInteropMetadata(sourceSetName: String, consumer: MetadataConsumer): FileCollection {
+        val multiplatformExtension = dependencyProject.topLevelExtension as? KotlinMultiplatformExtension
+            ?: return dependencyProject.files()
+
+        val commonizeCInteropTask = when (consumer) {
+            MetadataConsumer.Ide -> dependencyProject.copyCommonizeCInteropForIdeTask ?: return dependencyProject.files()
+            MetadataConsumer.Cli -> dependencyProject.commonizeCInteropTask ?: return dependencyProject.files()
+        }
+
+        val sourceSet = multiplatformExtension.sourceSets.findByName(sourceSetName) ?: return dependencyProject.files()
+        val dependent = CInteropCommonizerDependent.from(dependencyProject, sourceSet) ?: return dependencyProject.files()
+        return commonizeCInteropTask.get().commonizedOutputLibraries(dependent)
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/TransformKotlinGranularMetadata.kt

@@ -54,8 +54,8 @@ open class TransformKotlinGranularMetadata
         CompilationSourceSetUtil.compilationsBySourceSets(project)
             .filterKeys { it in sourceSets }
             .entries.associate { (sourceSet, compilations) ->
-            sourceSet.name to compilations.map { it.name }.sorted()
-        }
+                sourceSet.name to compilations.map { it.name }.sorted()
+            }
     }
 
     private val participatingCompilations: Iterable<KotlinCompilation<*>>
@@ -96,18 +96,15 @@ open class TransformKotlinGranularMetadata
         transformation.metadataDependencyResolutions
     }
 
-    private val extractableFilesByResolution: Map<out MetadataDependencyResolution, ExtractableMetadataFiles>
+    @get:Internal
+    internal val filesByResolution: Map<out MetadataDependencyResolution, FileCollection>
         get() = metadataDependencyResolutions
             .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
-            .associateWith { it.getExtractableMetadataFiles(outputsDir) }
-
-    @get:Internal
-    internal val filesByResolution: Map<MetadataDependencyResolution, FileCollection>
-        get() = extractableFilesByResolution.mapValues { (_, value) ->
-            project.files(value.getMetadataFilesPerSourceSet(false).values).builtBy(this)
-        }
-
-    private val extractableFiles by project.provider { extractableFilesByResolution.values }
+            .associateWith { chooseVisibleSourceSets ->
+                chooseVisibleSourceSets.getAllCompiledSourceSetMetadata(
+                    project, outputDirectoryWhenMaterialised = outputsDir, materializeFilesIfNecessary = false
+                ).builtBy(this)
+            }
 
     @TaskAction
     fun transformMetadata() {
@@ -116,7 +113,13 @@ open class TransformKotlinGranularMetadata
         }
         outputsDir.mkdirs()
 
-        extractableFiles.forEach { it.getMetadataFilesPerSourceSet(createFiles = true) }
+        metadataDependencyResolutions
+            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+            .forEach { chooseVisibleSourceSets ->
+                chooseVisibleSourceSets.getAllCompiledSourceSetMetadata(
+                    project, outputDirectoryWhenMaterialised = outputsDir, materializeFilesIfNecessary = true
+                )
+            }
     }
 }
 
@@ -125,5 +128,6 @@ internal class SourceSetResolvedMetadataProvider(
 ) : ResolvedMetadataFilesProvider {
     override val buildDependencies: Iterable<TaskProvider<*>> = listOf(taskProvider)
     override val metadataResolutions: Iterable<MetadataDependencyResolution> by taskProvider.map { it.metadataDependencyResolutions }
-    override val metadataFilesByResolution: Map<MetadataDependencyResolution, FileCollection> by taskProvider.map { it.filesByResolution }
+    override val metadataFilesByResolution: Map<out MetadataDependencyResolution, FileCollection>
+            by taskProvider.map { it.filesByResolution }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/DependencyResolution.kt

@@ -13,9 +13,7 @@ import org.gradle.api.artifacts.result.ResolvedDependencyResult
 import org.gradle.api.capabilities.Capability
 import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
 import org.jetbrains.kotlin.gradle.plugin.mpp.*
-import org.jetbrains.kotlin.gradle.plugin.mpp.getProjectStructureMetadata
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultLanguageSettingsBuilder
-import org.jetbrains.kotlin.gradle.utils.getOrPut
 import org.jetbrains.kotlin.gradle.utils.getOrPutRootProjectProperty
 import org.jetbrains.kotlin.project.model.*
 import java.util.*
@@ -214,3 +212,17 @@ internal fun ComponentIdentifier.matchesModuleIdentifier(id: KotlinModuleIdentif
         }
         else -> false
     }
+
+private fun getProjectStructureMetadata(
+    project: Project,
+    module: ResolvedComponentResult,
+    configuration: Configuration,
+    moduleIdentifier: KotlinModuleIdentifier? = null
+): KotlinProjectStructureMetadata? {
+    val extractor = if (moduleIdentifier != null)
+        MppDependencyProjectStructureMetadataExtractor.create(project, module, moduleIdentifier, configuration)
+    else
+        MppDependencyProjectStructureMetadataExtractor.create(project, module, configuration, resolveViaAvailableAt = true)
+
+    return extractor?.getProjectStructureMetadata()
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/FragmentGranularMetadataResolver.kt

@@ -8,13 +8,11 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20
 import org.gradle.api.Project
 import org.gradle.api.artifacts.result.ResolvedDependencyResult
 import org.jetbrains.kotlin.gradle.plugin.mpp.*
-import org.jetbrains.kotlin.gradle.plugin.mpp.ChooseVisibleSourceSetsImpl
-import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution
-import org.jetbrains.kotlin.gradle.plugin.mpp.getMetadataExtractor
-import org.jetbrains.kotlin.gradle.plugin.mpp.getProjectStructureMetadata
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.Companion.asMetadataProvider
 import org.jetbrains.kotlin.project.model.*
 import org.jetbrains.kotlin.utils.addToStdlib.flattenTo
-import java.util.ArrayDeque
+import java.io.File
+import java.util.*
 
 internal class FragmentGranularMetadataResolver(
     private val requestingFragment: KotlinGradleFragment,
@@ -80,33 +78,46 @@ internal class FragmentGranularMetadataResolver(
                     MetadataDependencyResolution.KeepOriginalDependency(resolvedComponentResult, isResolvedAsProject)
                 }
                 else -> run {
+
                     val metadataSourceComponent = dependencyNode.run { metadataSourceComponent ?: selectedComponent }
 
-                    val metadataExtractor = getMetadataExtractor(project, resolvedComponentResult, configurationToResolve, true)
-
-                    if (dependencyModule is ExternalImportedKotlinModule &&
-                        metadataExtractor is JarArtifactMppDependencyMetadataExtractor &&
-                        chosenFragments != null
-                    ) {
-                        resolveHostSpecificMetadataArtifacts(dependencyModule, chosenFragments, metadataExtractor)
-                    }
-
-                    val projectStructureMetadata = (dependencyModule as? ExternalImportedKotlinModule)?.projectStructureMetadata
-                        ?: checkNotNull(metadataExtractor?.getProjectStructureMetadata())
-
                     val visibleFragmentNames = visibleFragments.map { it.fragmentName }.toSet()
                     val visibleFragmentNamesExcludingVisibleByParents =
-                        visibleFragmentNames
-                            .minus(fragmentsNamesVisibleByParents(metadataSourceComponent.toSingleModuleIdentifier()))
+                        visibleFragmentNames.minus(fragmentsNamesVisibleByParents(metadataSourceComponent.toSingleModuleIdentifier()))
 
-                    ChooseVisibleSourceSetsImpl(
-                        metadataSourceComponent,
-                        isResolvedAsProject,
-                        projectStructureMetadata,
-                        visibleFragmentNames,
-                        visibleFragmentNamesExcludingVisibleByParents,
+                    val metadataExtractor = MppDependencyProjectStructureMetadataExtractor.create(
+                        project, resolvedComponentResult, configurationToResolve, true
+                    ) ?: error("Failed to create 'MppDependencyProjectStructureMetadataExtractor' for ${resolvedComponentResult.id}")
+
+                    val projectStructureMetadata = (dependencyModule as? ExternalImportedKotlinModule)?.projectStructureMetadata
+                        ?: checkNotNull(metadataExtractor.getProjectStructureMetadata())
+
+
+                    val metadataProvider = when (metadataExtractor) {
+                        is ProjectMppDependencyProjectStructureMetadataExtractor -> ProjectMetadataProvider(
+                            dependencyProject = metadataExtractor.dependencyProject,
+                            moduleIdentifier = metadataExtractor.moduleIdentifier
+                        )
+
+                        is JarMppDependencyProjectStructureMetadataExtractor -> CompositeMetadataJar(
+                            moduleIdentifier = ModuleIds.fromComponent(project, metadataSourceComponent).toString(),
+                            projectStructureMetadata = projectStructureMetadata,
+                            primaryArtifactFile = metadataExtractor.primaryArtifactFile,
+                            hostSpecificArtifactsBySourceSet = if (
+                                dependencyModule is ExternalImportedKotlinModule && chosenFragments != null
+                            ) resolveHostSpecificMetadataArtifacts(dependencyModule, chosenFragments) else emptyMap(),
+                        ).asMetadataProvider()
+                    }
+
+                    MetadataDependencyResolution.ChooseVisibleSourceSets(
+                        dependency = metadataSourceComponent,
+                        projectDependency = isResolvedAsProject,
+                        projectStructureMetadata = projectStructureMetadata,
+                        allVisibleSourceSetNames = visibleFragmentNames,
+                        visibleSourceSetNamesExcludingDependsOn = visibleFragmentNamesExcludingVisibleByParents,
+                        visibleTransitiveDependencies =
                         visibleTransitiveDependencies.map { resolvedDependenciesByModuleId.getValue(it.module.moduleIdentifier) }.toSet(),
-                        checkNotNull(metadataExtractor)
+                        metadataProvider = metadataProvider
                     )
                 }
             }
@@ -124,19 +135,18 @@ internal class FragmentGranularMetadataResolver(
 
     private fun fragmentsNamesVisibleByParents(kotlinModuleIdentifier: KotlinModuleIdentifier): MutableSet<String> {
         val parentResolutionsForDependency = parentResultsByModuleIdentifier[kotlinModuleIdentifier].orEmpty()
-        return parentResolutionsForDependency.filterIsInstance<ChooseVisibleSourceSetsImpl>()
+        return parentResolutionsForDependency.filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
             .flatMapTo(mutableSetOf()) { it.allVisibleSourceSetNames }
     }
 
     private fun resolveHostSpecificMetadataArtifacts(
         dependencyModule: ExternalImportedKotlinModule,
         chosenFragments: FragmentResolution.ChosenFragments,
-        metadataExtractor: JarArtifactMppDependencyMetadataExtractor
-    ) {
+    ): Map<String, File> {
         val visibleFragments = chosenFragments.visibleFragments
         val variantResolutions = chosenFragments.variantResolutions
         val hostSpecificFragments = dependencyModule.hostSpecificFragments
-        val hostSpecificFragmentToArtifact = visibleFragments.intersect(hostSpecificFragments).mapNotNull { hostSpecificFragment ->
+        return visibleFragments.intersect(hostSpecificFragments).mapNotNull { hostSpecificFragment ->
             val relevantVariantResolution = variantResolutions
                 .filterIsInstance<VariantResolution.VariantMatch>()
                 // find some of our variants that resolved a dependency's variant containing the fragment
@@ -152,7 +162,6 @@ internal class FragmentGranularMetadataResolver(
                     )
                 hostSpecificArtifact?.let { hostSpecificFragment.fragmentName to it }
             }
-        }
-        metadataExtractor.metadataArtifactBySourceSet.putAll(hostSpecificFragmentToArtifact)
+        }.toMap()
     }
-}
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/KotlinMetadataCompilationData.kt

@@ -17,9 +17,8 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformCommonOptionsImpl
 import org.jetbrains.kotlin.gradle.dsl.pm20Extension
 import org.jetbrains.kotlin.gradle.plugin.*
 import org.jetbrains.kotlin.gradle.plugin.mpp.*
-import org.jetbrains.kotlin.gradle.plugin.sources.DefaultLanguageSettingsBuilder
 import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider
-import org.jetbrains.kotlin.gradle.targets.metadata.createTransformedMetadataClasspath
+import org.jetbrains.kotlin.gradle.targets.metadata.createMetadataDependencyTransformationClasspath
 import org.jetbrains.kotlin.gradle.utils.getValue
 import org.jetbrains.kotlin.gradle.utils.lowerCamelCaseName
 import org.jetbrains.kotlin.konan.target.HostManager
@@ -59,7 +58,7 @@ internal abstract class AbstractKotlinFragmentMetadataCompilationData<T : Kotlin
         get() = compileAllTask.name
 
     override val compileDependencyFiles: FileCollection by project.provider {
-        createTransformedMetadataClasspath(
+        createMetadataDependencyTransformationClasspath(
             project,
             resolvableMetadataConfiguration(fragment.containingModule),
             lazy { fragment.refinesClosure.minus(fragment).map { metadataCompilationRegistry.byFragment(it).output.classesDirs } },

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/TransformKotlinGranularMetadataForFragment.kt

@@ -8,8 +8,8 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.*
-import org.jetbrains.kotlin.gradle.plugin.mpp.ExtractableMetadataFiles
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution
+import org.jetbrains.kotlin.gradle.plugin.mpp.getAllCompiledSourceSetMetadata
 import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider
 import org.jetbrains.kotlin.gradle.utils.getValue
 import java.io.File
@@ -64,18 +64,15 @@ internal open class TransformKotlinGranularMetadataForFragment
         transformation.resolutions
     }
 
-    private val extractableFilesByResolution: Map<out MetadataDependencyResolution, ExtractableMetadataFiles>
+    @get:Internal
+    internal val filesByResolution: Map<out MetadataDependencyResolution, FileCollection>
         get() = metadataDependencyResolutions
             .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
-            .associateWith { it.getExtractableMetadataFiles(outputsDir) }
-
-    @get:Internal
-    internal val filesByResolution: Map<MetadataDependencyResolution, FileCollection>
-        get() = extractableFilesByResolution.mapValues { (_, value) ->
-            project.files(value.getMetadataFilesPerSourceSet(false).values).builtBy(this)
-        }
-
-    private val extractableFiles by project.provider { extractableFilesByResolution.values }
+            .associateWith { chooseVisibleSourceSets ->
+                chooseVisibleSourceSets.getAllCompiledSourceSetMetadata(
+                    project, outputsDir, materializeFilesIfNecessary = false
+                ).builtBy(this)
+            }
 
     @TaskAction
     fun transformMetadata() {
@@ -84,7 +81,11 @@ internal open class TransformKotlinGranularMetadataForFragment
         }
         outputsDir.mkdirs()
 
-        extractableFiles.forEach { it.getMetadataFilesPerSourceSet(createFiles = true) }
+        metadataDependencyResolutions
+            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+            .forEach { chooseVisibleSourceSets ->
+                chooseVisibleSourceSets.getAllCompiledSourceSetMetadata(project, outputsDir, materializeFilesIfNecessary = true)
+            }
     }
 }
 
@@ -93,5 +94,5 @@ internal class FragmentResolvedMetadataProvider(
 ) : ResolvedMetadataFilesProvider {
     override val buildDependencies: Iterable<TaskProvider<*>> = listOf(taskProvider)
     override val metadataResolutions: Iterable<MetadataDependencyResolution> by taskProvider.map { it.metadataDependencyResolutions }
-    override val metadataFilesByResolution: Map<MetadataDependencyResolution, FileCollection> by taskProvider.map { it.filesByResolution }
+    override val metadataFilesByResolution: Map<out MetadataDependencyResolution, FileCollection> by taskProvider.map { it.filesByResolution }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/DefaultKotlinSourceSet.kt

@@ -155,7 +155,6 @@ class DefaultKotlinSourceSet(
         return getDependenciesTransformation(scope)
     }
 
-    @Suppress("unused") // Used in IDE import
     fun getAdditionalVisibleSourceSets(): List<KotlinSourceSet> =
         getVisibleSourceSetsFromAssociateCompilations(project, this)
 
@@ -184,10 +183,14 @@ class DefaultKotlinSourceSet(
                     MetadataDependencyTransformation(group, name, projectPath, null, emptySet(), emptyMap())
 
                 is MetadataDependencyResolution.ChooseVisibleSourceSets -> {
-                    val filesBySourceSet = resolution.getMetadataFilesBySourceSet(
-                        baseDir,
-                        createFiles = true
-                    ).filter { it.value.any { it.exists() } }
+                    val filesBySourceSet = resolution.visibleSourceSetNamesExcludingDependsOn.associateWith { visibleSourceSetName ->
+                        resolution.metadataProvider.getSourceSetCompiledMetadata(
+                            project,
+                            sourceSetName = visibleSourceSetName,
+                            outputDirectoryWhenMaterialised = baseDir,
+                            materializeFilesIfNecessary = true
+                        )
+                    }.filter { (_, files) -> files.any(File::exists) }
 
                     MetadataDependencyTransformation(
                         group, name, projectPath,

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/metadata/KotlinMetadataTargetConfigurator.kt

@@ -113,7 +113,7 @@ class KotlinMetadataTargetConfigurator :
     override fun setupCompilationDependencyFiles(compilation: KotlinCompilation<KotlinCommonOptions>) {
         val project = compilation.target.project
 
-        /** See [createTransformedMetadataClasspath] and its usage. */
+        /** See [createMetadataDependencyTransformationClasspath] and its usage. */
         if (project.isKotlinGranularMetadataEnabled && compilation.name != KotlinCompilation.MAIN_COMPILATION_NAME)
             compilation.compileDependencyFiles = project.files()
         else
@@ -332,10 +332,14 @@ class KotlinMetadataTargetConfigurator :
             listOf(sourceSet)
         ) { }
 
-        compilation.compileDependencyFiles += createTransformedMetadataClasspath(
+        compilation.compileDependencyFiles += createMetadataDependencyTransformationClasspath(
             project.configurations.getByName(ALL_COMPILE_METADATA_CONFIGURATION_NAME),
             compilation
         )
+
+        if (compilation is KotlinSharedNativeCompilation && sourceSet is DefaultKotlinSourceSet) {
+            compilation.compileDependencyFiles += project.createCInteropMetadataDependencyClasspath(sourceSet)
+        }
     }
 
     private fun setupDependencyTransformationForSourceSet(
@@ -418,7 +422,7 @@ class KotlinMetadataTargetConfigurator :
         }
     }
 
-    private fun createTransformedMetadataClasspath(
+    private fun createMetadataDependencyTransformationClasspath(
         fromFiles: Configuration,
         compilation: AbstractKotlinCompilation<*>
     ): FileCollection {
@@ -440,7 +444,7 @@ class KotlinMetadataTargetConfigurator :
             transformationTaskHolders.map { SourceSetResolvedMetadataProvider(it) }
         }
 
-        return createTransformedMetadataClasspath(
+        return createMetadataDependencyTransformationClasspath(
             project,
             fromFiles,
             dependsOnCompilationOutputs,
@@ -492,14 +496,14 @@ internal fun Project.createGenerateProjectStructureMetadataTask(): TaskProvider<
 internal interface ResolvedMetadataFilesProvider {
     val buildDependencies: Iterable<TaskProvider<*>>
     val metadataResolutions: Iterable<MetadataDependencyResolution>
-    val metadataFilesByResolution: Map<MetadataDependencyResolution, FileCollection>
+    val metadataFilesByResolution: Map<out MetadataDependencyResolution, FileCollection>
 }
 
-internal fun createTransformedMetadataClasspath(
+internal fun createMetadataDependencyTransformationClasspath(
     project: Project,
     fromFiles: Configuration,
     parentCompiledMetadataFiles: Lazy<Iterable<FileCollection>>,
-    metadataResolutionProviders: Lazy<Iterable<ResolvedMetadataFilesProvider>>
+    metadataResolutionProviders: Lazy<Iterable<ResolvedMetadataFilesProvider>>,
 ): FileCollection {
     return project.files(
         Callable {
@@ -547,7 +551,7 @@ internal fun createTransformedMetadataClasspath(
 }
 
 internal fun isSharedNativeSourceSet(project: Project, sourceSet: KotlinSourceSet): Boolean {
-    val compilations = CompilationSourceSetUtil.compilationsBySourceSets(project)[sourceSet].orEmpty()
+    val compilations = compilationsBySourceSets(project)[sourceSet].orEmpty()
     return compilations.isNotEmpty() && compilations.all {
         it.platformType == KotlinPlatformType.common || it.platformType == KotlinPlatformType.native
     }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/AbstractCInteropCommonizerTask.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.commonizer.CommonizerOutputFileLayout.base64Hash
 import org.jetbrains.kotlin.commonizer.CommonizerOutputFileLayout.ensureMaxFileNameLength
 import org.jetbrains.kotlin.commonizer.identityString
 import org.jetbrains.kotlin.gradle.utils.filesProvider
+import org.jetbrains.kotlin.gradle.utils.outputFilesProvider
 import java.io.File
 
 internal abstract class AbstractCInteropCommonizerTask : DefaultTask() {
@@ -33,12 +34,10 @@ internal fun AbstractCInteropCommonizerTask.outputDirectory(group: CInteropCommo
 }
 
 internal fun AbstractCInteropCommonizerTask.commonizedOutputLibraries(dependent: CInteropCommonizerDependent): FileCollection {
-    val fileProvider = project.filesProvider {
-        (commonizedOutputDirectory(dependent) ?: return@filesProvider emptySet<File>())
+    return outputFilesProvider {
+        (commonizedOutputDirectory(dependent) ?: return@outputFilesProvider emptySet<File>())
             .listFiles().orEmpty().toSet()
     }
-
-    return fileProvider.builtBy(this)
 }
 
 internal fun AbstractCInteropCommonizerTask.commonizedOutputDirectory(dependent: CInteropCommonizerDependent): File? {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerAllMetadataKlib.kt

@@ -1,25 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.targets.native.internal
-
-import org.gradle.api.Project
-import org.gradle.api.tasks.TaskProvider
-import org.gradle.api.tasks.bundling.Zip
-import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinSharedNativeCompilation
-
-internal fun Project.includeCommonizedCInteropMetadata(metadataKlib: TaskProvider<out Zip>, compilation: KotlinSharedNativeCompilation) {
-    metadataKlib.configure { jar -> includeCommonizedCInteropMetadata(jar, compilation) }
-}
-
-internal fun Project.includeCommonizedCInteropMetadata(metadataKlib: Zip, compilation: KotlinSharedNativeCompilation) {
-    val commonizerTask = commonizeCInteropTask?.get() ?: return
-    val commonizerDependencyToken = CInteropCommonizerDependent.from(compilation) ?: return
-    val outputDirectory = commonizerTask.commonizedOutputDirectory(commonizerDependencyToken) ?: return
-
-    metadataKlib.from(outputDirectory) { spec ->
-        spec.into(compilation.defaultSourceSet.name + "-cinterop")
-    }
-}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerDependencies.kt

@@ -6,9 +6,11 @@
 package org.jetbrains.kotlin.gradle.targets.native.internal
 
 import org.gradle.api.Project
+import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinSharedNativeCompilation
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet
+import org.jetbrains.kotlin.gradle.plugin.whenEvaluated
 import org.jetbrains.kotlin.gradle.utils.filesProvider
 import java.io.File
 
@@ -21,6 +23,7 @@ internal fun Project.setupCInteropCommonizerDependencies() {
 
     kotlin.forAllDefaultKotlinSourceSets { sourceSet ->
         setupCInteropCommonizerDependenciesForIde(sourceSet)
+        setupCInteropTransformCompositeMetadataDependenciesForIde(sourceSet)
     }
 }
 
@@ -50,3 +53,12 @@ private fun Project.setupCInteropCommonizerDependenciesForIde(sourceSet: Default
         }
     })
 }
+
+private fun Project.setupCInteropTransformCompositeMetadataDependenciesForIde(sourceSet: DefaultKotlinSourceSet) {
+    whenEvaluated {
+        if (getCommonizerTarget(sourceSet) !is SharedCommonizerTarget) return@whenEvaluated
+        addIntransitiveMetadataDependencyIfPossible(
+            sourceSet, createCInteropMetadataDependencyClasspathForIde(sourceSet)
+        )
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropMetadataDependencyClasspath.kt

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.native.internal
+
+import org.gradle.api.Project
+import org.gradle.api.file.FileCollection
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider.MetadataConsumer.Cli
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider.MetadataConsumer.Ide
+import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet
+import org.jetbrains.kotlin.gradle.utils.filesProvider
+import java.io.File
+
+internal fun Project.createCInteropMetadataDependencyClasspath(sourceSet: DefaultKotlinSourceSet): FileCollection {
+    return createCInteropMetadataDependencyClasspath(sourceSet, forIde = false)
+}
+
+internal fun Project.createCInteropMetadataDependencyClasspathForIde(sourceSet: DefaultKotlinSourceSet): FileCollection {
+    return createCInteropMetadataDependencyClasspath(sourceSet, forIde = true)
+}
+
+/**
+ * @param forIde: A different task for dependency transformation will be used. This task will not use the regular 'build' directory
+ * as transformation output to ensure IDE still being able to resolve the dependencies even when the project is cleaned.
+ */
+internal fun Project.createCInteropMetadataDependencyClasspath(sourceSet: DefaultKotlinSourceSet, forIde: Boolean): FileCollection {
+    val dependencyTransformationTask = if (forIde) locateOrRegisterCInteropMetadataDependencyTransformationTaskForIde(sourceSet)
+    else locateOrRegisterCInteropMetadataDependencyTransformationTask(sourceSet)
+
+    /*
+    The classpath will be assembled by three independent parts
+    1) C-Interop Metadata which will be downloaded Jar files that get transformed by the transformation task
+    2) C-Interop Metadata directly provided by dependency projects (in the same build)
+    3) C-Interop Metadata from 'associated compilations' / additionalVisible source sets
+       (e.g. 'nativeTest' will be able to access the classpath from 'nativeMain')
+     */
+    return project.files(dependencyTransformationTask.map { it.outputLibraryFiles }) +
+            createCInteropMetadataDependencyClasspathFromProjectDependencies(sourceSet, forIde) +
+            createCInteropMetadataDependencyClasspathFromAssociatedCompilations(sourceSet, forIde)
+}
+
+private fun Project.createCInteropMetadataDependencyClasspathFromProjectDependencies(
+    sourceSet: DefaultKotlinSourceSet,
+    forIde: Boolean
+): FileCollection {
+    return filesProvider {
+        sourceSet.dependencyTransformations.values.flatMap { it.metadataDependencyResolutions }
+            .filterIsInstance<ChooseVisibleSourceSets>()
+            .flatMap { chooseVisibleSourceSets ->
+                /* We only want to access resolutions that provide metadata from dependency projects */
+                val projectMetadataProvider = when (chooseVisibleSourceSets.metadataProvider) {
+                    is ProjectMetadataProvider -> chooseVisibleSourceSets.metadataProvider
+                    is JarMetadataProvider -> return@flatMap emptyList()
+                }
+
+                chooseVisibleSourceSets.visibleSourceSetsProvidingCInterops.map { visibleSourceSetName ->
+                    projectMetadataProvider.getSourceSetCInteropMetadata(visibleSourceSetName, if (forIde) Ide else Cli)
+                }
+            }
+    }
+}
+
+private fun Project.createCInteropMetadataDependencyClasspathFromAssociatedCompilations(
+    sourceSet: DefaultKotlinSourceSet,
+    forIde: Boolean
+): FileCollection {
+    return filesProvider files@{
+        val commonizerTarget = getSharedCommonizerTarget(sourceSet) ?: return@files emptySet<File>()
+
+        /*
+        We will find the 'most suitable' / 'closest matching' source set
+        (like 'nativeTest' -> 'nativeMain', 'appleTest' -> 'appleMain', ...).
+        If no source set is found that matches the commonizer target explicitly, the next "bigger" source set shall be chosen
+         */
+        val (associatedSourceSet, _) = sourceSet.getAdditionalVisibleSourceSets()
+            .filterIsInstance<DefaultKotlinSourceSet>()
+            .mapNotNull { otherSourceSet -> otherSourceSet to (getSharedCommonizerTarget(otherSourceSet) ?: return@mapNotNull null) }
+            .filter { (_, otherCommonizerTarget) -> otherCommonizerTarget.targets.containsAll(commonizerTarget.targets) }
+            .minByOrNull { (_, otherCommonizerTarget) -> otherCommonizerTarget.targets.size } ?: return@files emptySet<File>()
+
+        createCInteropMetadataDependencyClasspath(associatedSourceSet, forIde)
+    }
+}
+
+/**
+ * Names of all source sets that may potentially provide necessary cinterops for this resolution.
+ * This will select 'the most bottom' source sets in [ChooseVisibleSourceSets.allVisibleSourceSetNames]
+ */
+internal val ChooseVisibleSourceSets.visibleSourceSetsProvidingCInterops: Set<String>
+    get() {
+        val dependsOnSourceSets = allVisibleSourceSetNames
+            .flatMap { projectStructureMetadata.sourceSetsDependsOnRelation[it].orEmpty() }
+            .toSet()
+
+        return allVisibleSourceSetNames.filter { it !in dependsOnSourceSets }.toSet()
+    }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropMetadataDependencyTransformationTask.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.native.internal
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier
+import org.gradle.api.file.FileCollection
+import org.gradle.api.tasks.*
+import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
+import org.jetbrains.kotlin.gradle.plugin.mpp.GranularMetadataTransformation
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.JarMetadataProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider
+import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet
+import org.jetbrains.kotlin.gradle.plugin.sources.SourceSetMetadataStorageForIde
+import org.jetbrains.kotlin.gradle.tasks.dependsOn
+import org.jetbrains.kotlin.gradle.tasks.locateOrRegisterTask
+import org.jetbrains.kotlin.gradle.utils.filesProvider
+import org.jetbrains.kotlin.gradle.utils.lowerCamelCaseName
+import org.jetbrains.kotlin.gradle.utils.outputFilesProvider
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
+import java.io.File
+import javax.inject.Inject
+
+internal fun Project.locateOrRegisterCInteropMetadataDependencyTransformationTask(
+    sourceSet: DefaultKotlinSourceSet,
+): TaskProvider<CInteropMetadataDependencyTransformationTask> {
+    return locateOrRegisterTask(
+        lowerCamelCaseName("transform", sourceSet.name, "CInteropDependenciesMetadata"),
+        args = listOf(
+            sourceSet,
+            /* outputDirectory = */
+            project.buildDir.resolve("kotlinSourceSetMetadata").resolve(sourceSet.name + "-cinterop")
+        )
+    )
+}
+
+internal fun Project.locateOrRegisterCInteropMetadataDependencyTransformationTaskForIde(
+    sourceSet: DefaultKotlinSourceSet,
+): TaskProvider<CInteropMetadataDependencyTransformationTask> {
+    return locateOrRegisterTask(
+        lowerCamelCaseName("transform", sourceSet.name, "CInteropDependenciesMetadataForIde"),
+        invokeWhenRegistered = { commonizeTask.dependsOn(this) },
+        args = listOf(
+            sourceSet,
+            /* outputDirectory = */
+            SourceSetMetadataStorageForIde.sourceSetStorage(project, sourceSet.name).resolve("cinterop")
+        ),
+    )
+}
+
+internal open class CInteropMetadataDependencyTransformationTask @Inject constructor(
+    @get:Internal val sourceSet: DefaultKotlinSourceSet,
+    @get:OutputDirectory val outputDirectory: File
+) : DefaultTask() {
+
+    /**
+     * Esoteric output file that every [CInteropMetadataDependencyTransformationTask] will point to.
+     * Sharing this output file will tell Gradle that those tasks cannot run in parallel.
+     * Parallelling this task is not supported, because parallel access to [GranularMetadataTransformation.metadataDependencyResolutions]
+     * is not supported during Gradle's execution phase. This is due to this property's Lazy's lock and
+     * another Lock used by Gradle which would lead to deadlocking.
+     *
+     * This task could potentially be ordered topologically based on dependsOn edges or parent transformations, however
+     * we actually do not care about the actual order chosen by Gradle.
+     */
+    @get:OutputFile
+    protected val mutex: File = project.file(".cinteropMetadataDependencyTransformationMutex")
+
+    @Suppress("unused")
+    @get:InputFiles
+    @get:Classpath
+    protected val inputArtifactFiles: FileCollection = project.filesProvider {
+        sourceSet.dependencyTransformations.values.map { it.configurationToResolve.withoutProjectDependencies() }
+    }
+
+    @get:Internal
+    protected val chooseVisibleSourceSets
+        get() = sourceSet.dependencyTransformations.values
+            .flatMap { it.metadataDependencyResolutions }
+            .filterIsInstance<ChooseVisibleSourceSets>()
+
+    @Suppress("unused")
+    @get:Input
+    protected val dependencyProjectStructureMetadata
+        get() = chooseVisibleSourceSets.map { it.projectStructureMetadata }
+
+    @get:Internal
+    val outputLibraryFiles = outputFilesProvider {
+        outputDirectory.walkTopDown().maxDepth(2).filter { it.isFile && it.extension == KLIB_FILE_EXTENSION }.toList()
+    }
+
+    @TaskAction
+    protected fun transformDependencies() {
+        if (outputDirectory.isDirectory) {
+            outputDirectory.deleteRecursively()
+        }
+
+        if (project.getCommonizerTarget(sourceSet) !is SharedCommonizerTarget) return
+        chooseVisibleSourceSets.flatMap(::materializeMetadata)
+    }
+
+    private fun materializeMetadata(
+        chooseVisibleSourceSets: ChooseVisibleSourceSets
+    ): Set<File> = when (chooseVisibleSourceSets.metadataProvider) {
+        /* Nothing to transform: We will use original commonizer output in such cases */
+        is ProjectMetadataProvider -> emptySet()
+
+        /* Extract/Materialize all cinterop files from composite jar file */
+        is JarMetadataProvider ->
+            chooseVisibleSourceSets.visibleSourceSetsProvidingCInterops.flatMap { visibleSourceSetName ->
+                chooseVisibleSourceSets.metadataProvider.getSourceSetCInteropMetadata(
+                    visibleSourceSetName, outputDirectory, materializeFiles = true
+                )
+            }.toSet()
+    }
+
+    private fun Configuration.withoutProjectDependencies(): FileCollection {
+        return incoming.artifactView { view ->
+            view.componentFilter { componentIdentifier ->
+                componentIdentifier !is ProjectComponentIdentifier
+            }
+        }.files
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/GetCommonizerTargetOfSourceSet.kt

@@ -24,3 +24,7 @@ internal fun Project.getCommonizerTarget(sourceSet: KotlinSourceSet): Commonizer
         else -> SharedCommonizerTarget(compilationTargets.allLeaves())
     }
 }
+
+internal fun Project.getSharedCommonizerTarget(sourceSet: KotlinSourceSet): SharedCommonizerTarget? {
+    return getCommonizerTarget(sourceSet) as? SharedCommonizerTarget
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/providerApiUtils.kt

@@ -6,11 +6,13 @@
 package org.jetbrains.kotlin.gradle.utils
 
 import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.TaskProvider
 import java.io.File
 import kotlin.properties.ReadOnlyProperty
 import kotlin.reflect.KProperty
@@ -91,3 +93,7 @@ internal fun Project.filesProvider(
 ): ConfigurableFileCollection {
     return project.files(project.provider(provider)).builtBy(*buildDependencies)
 }
+
+internal fun <T : Task> T.outputFilesProvider(provider: T.() -> Any): ConfigurableFileCollection {
+    return project.filesProvider(this) { provider() }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/zipUtils.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.utils
+
+import java.io.File
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+import java.util.zip.ZipOutputStream
+
+internal fun copyZipFilePartially(sourceZipFile: File, destinationZipFile: File, path: String) {
+    require(path.endsWith("/")) { "Expected path to end with '/', found '$path'" }
+
+    ZipFile(sourceZipFile).use { zip ->
+        val entries = zip.entries().asSequence()
+            .filter { it.name.startsWith(path) }
+            .filter { !it.isDirectory }.toList()
+
+        if (entries.isEmpty()) return
+
+        ZipOutputStream(destinationZipFile.outputStream()).use { destinationZipOutputStream ->
+            entries.forEach { entry ->
+                // Drop the source set name from the entry path
+                val destinationEntry = ZipEntry(entry.name.substringAfter(path))
+
+                zip.getInputStream(entry).use { inputStream ->
+                    destinationZipOutputStream.putNextEntry(destinationEntry)
+                    inputStream.copyTo(destinationZipOutputStream)
+                    destinationZipOutputStream.closeEntry()
+                }
+            }
+        }
+    }
+}
+
+internal fun ZipFile.listDescendants(zipEntry: ZipEntry): Sequence<ZipEntry> {
+    require(zipEntry.isDirectory)
+    return entries().asSequence().filter { entry ->
+        entry.name != zipEntry.name && entry.name.startsWith(zipEntry.name)
+    }
+}
+
+internal fun ZipFile.listChildren(zipEntry: ZipEntry): Sequence<ZipEntry> {
+    return listDescendants(zipEntry).filter { entry ->
+        entry.name.removePrefix(zipEntry.name).count { it == '/' } == 1
+    }
+}