Implement dependency deduplication in order to reduce memory

consumption during import process

idea/idea-gradle/src/org/jetbrains/kotlin/idea/configuration/KotlinMPPGradleProjectResolver.kt

@@ -298,7 +298,7 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
                             it.kotlinSourceSet = kotlinSourceSet
                         }
                     if (existingSourceSetDataNode == null) {
-                        sourceSetMap[moduleId] = Pair(compilationDataNode, createExternalSourceSet(compilation, compilationData))
+                        sourceSetMap[moduleId] = Pair(compilationDataNode, createExternalSourceSet(compilation, compilationData, mppModel))
                     }
                 }
 
@@ -348,7 +348,7 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
                         it.kotlinSourceSet = kotlinSourceSet
                     }
                 if (existingSourceSetDataNode == null) {
-                    sourceSetMap[moduleId] = Pair(sourceSetDataNode, createExternalSourceSet(sourceSet, sourceSetData))
+                    sourceSetMap[moduleId] = Pair(sourceSetDataNode, createExternalSourceSet(sourceSet, sourceSetData, mppModel))
                 }
             }
 
@@ -417,7 +417,8 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
             val processedModuleIds = HashSet<String>()
             processCompilations(gradleModule, mppModel, ideModule, resolverCtx) { dataNode, compilation ->
                 if (processedModuleIds.add(getKotlinModuleId(gradleModule, compilation, resolverCtx))) {
-                    val substitutedDependencies = substitutor.substituteDependencies(compilation.dependencies)
+                    val substitutedDependencies =
+                        substitutor.substituteDependencies(compilation.dependencies.mapNotNull { mppModel.dependencyMap[it] })
                     buildDependencies(
                         resolverCtx,
                         sourceSetMap,
@@ -494,8 +495,8 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
                 }
                 if (processedModuleIds.add(getKotlinModuleId(gradleModule, sourceSet, resolverCtx))) {
                     val mergedDependencies = LinkedHashSet<KotlinDependency>().apply {
-                        addAll(sourceSet.dependencies)
+                        addAll(sourceSet.dependencies.mapNotNull { mppModel.dependencyMap[it] })
-                        dependeeSourceSets.flatMapTo(this) { it.dependencies }
+                        dependeeSourceSets.flatMapTo(this) { it.dependencies.mapNotNull { mppModel.dependencyMap[it] } }
                     }
                     buildDependencies(
                         resolverCtx,
@@ -629,14 +630,14 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
             return PathUtilRt.suggestFileName(moduleName.toString(), true, false)
         }
 
-        private fun createExternalSourceSet(compilation: KotlinCompilation, compilationData: GradleSourceSetData): ExternalSourceSet {
+        private fun createExternalSourceSet(compilation: KotlinCompilation, compilationData: GradleSourceSetData, mppModel: KotlinMPPGradleModel): ExternalSourceSet {
             return DefaultExternalSourceSet().also { sourceSet ->
                 val effectiveClassesDir = compilation.output.effectiveClassesDir
                 val resourcesDir = compilation.output.resourcesDir
 
                 sourceSet.name = compilation.fullName()
                 sourceSet.targetCompatibility = compilationData.targetCompatibility
-                sourceSet.dependencies += compilation.dependencies
+                sourceSet.dependencies += compilation.dependencies.mapNotNull { mppModel.dependencyMap[it] }
                 //TODO after applying patch to IDEA core uncomment the following line:
                 // sourceSet.isTest = compilation.sourceSets.filter { isTestModule }.isNotEmpty()
                 // It will allow to get rid of hacks with guessing module type in DataServices and obtain properly set productionOnTest flags
@@ -665,15 +666,15 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
         }
 
 
-        private fun createExternalSourceSet(ktSourceSet: KotlinSourceSet, ktSourceSetData: GradleSourceSetData): ExternalSourceSet {
+        private fun createExternalSourceSet(ktSourceSet: KotlinSourceSet, ktSourceSetData: GradleSourceSetData, mppModel: KotlinMPPGradleModel): ExternalSourceSet {
             return DefaultExternalSourceSet().also { sourceSet ->
                 sourceSet.name = ktSourceSet.name
                 sourceSet.targetCompatibility = ktSourceSetData.targetCompatibility
-                sourceSet.dependencies += ktSourceSet.dependencies
+                sourceSet.dependencies += ktSourceSet.dependencies.mapNotNull { mppModel.dependencyMap[it] }
 
                 // BUNCH: 191 Can't use property because there's no getter in 192 and thus it isn't property anymore
                 @Suppress("UsePropertyAccessSyntax")
-                sourceSet.setSources(linkedMapOf(
+                sourceSet.setSources(linkedMapOf<IExternalSystemSourceType, ExternalSourceDirectorySet>(
                     ktSourceSet.sourceType to DefaultExternalSourceDirectorySet().also { dirSet ->
                         dirSet.srcDirs = ktSourceSet.sourceDirs
                     },

idea/kotlin-gradle-tooling/src/KotlinMPPGradleModel.kt

@@ -10,8 +10,28 @@ import org.jetbrains.plugins.gradle.model.ModelFactory
 import java.io.File
 import java.io.Serializable
 
+typealias KotlinDependencyId = Long
 typealias KotlinDependency = ExternalDependency
 
+class KotlinDependencyMapper {
+    private var currentIndex: KotlinDependencyId = 0
+    private val idToDependency = HashMap<KotlinDependencyId, KotlinDependency>()
+    private val dependencyToId = HashMap<KotlinDependency, KotlinDependencyId>()
+
+    fun getDependency(id: KotlinDependencyId) = idToDependency[id]
+
+    fun getId(dependency: KotlinDependency): KotlinDependencyId {
+        return dependencyToId[dependency] ?: let {
+            currentIndex++
+            dependencyToId[dependency] = currentIndex
+            idToDependency[currentIndex] = dependency
+            return currentIndex
+        }
+    }
+
+    fun toDependencyMap(): Map<KotlinDependencyId, KotlinDependency> = idToDependency
+}
+
 fun KotlinDependency.deepCopy(cache: MutableMap<Any, Any>): KotlinDependency {
     val cachedValue = cache[this] as? KotlinDependency
     return if (cachedValue != null) {
@@ -25,7 +45,7 @@ fun KotlinDependency.deepCopy(cache: MutableMap<Any, Any>): KotlinDependency {
 
 interface KotlinModule : Serializable {
     val name: String
-    val dependencies: Set<KotlinDependency>
+    val dependencies: Array<KotlinDependencyId>
     val isTestModule: Boolean
 }
 
@@ -133,6 +153,7 @@ interface ExtraFeatures : Serializable {
 }
 
 interface KotlinMPPGradleModel : Serializable {
+    val dependencyMap: Map<KotlinDependencyId, KotlinDependency>
     val sourceSets: Map<String, KotlinSourceSet>
     val targets: Collection<KotlinTarget>
     val extraFeatures: ExtraFeatures

idea/kotlin-gradle-tooling/src/KotlinMPPGradleModelBuilder.kt

@@ -45,14 +45,21 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
             true,
             SourceSetCachedFinder(project)
         )
-        val sourceSets = buildSourceSets(dependencyResolver, project) ?: return null
+        val dependencyMapper = KotlinDependencyMapper()
+        val sourceSets = buildSourceSets(dependencyResolver, project, dependencyMapper) ?: return null
         val sourceSetMap = sourceSets.map { it.name to it }.toMap()
-        val targets = buildTargets(sourceSetMap, dependencyResolver, project) ?: return null
+        val targets = buildTargets(sourceSetMap, dependencyResolver, project, dependencyMapper) ?: return null
         computeSourceSetsDeferredInfo(sourceSetMap, targets, isHMPPEnabled(project))
         val coroutinesState = getCoroutinesState(project)
         reportUnresolvedDependencies(targets)
         val kotlinNativeHome = KotlinNativeHomeEvaluator.getKotlinNativeHome(project) ?: NO_KOTLIN_NATIVE_HOME
-        return KotlinMPPGradleModelImpl(sourceSetMap, targets, ExtraFeaturesImpl(coroutinesState, isHMPPEnabled(project)), kotlinNativeHome)
+        return KotlinMPPGradleModelImpl(
+            sourceSetMap,
+            targets,
+            ExtraFeaturesImpl(coroutinesState, isHMPPEnabled(project)),
+            kotlinNativeHome,
+            dependencyMapper.toDependencyMap()
+        )
     }
 
     private fun isHMPPEnabled(project: Project): Boolean {
@@ -97,13 +104,13 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
         }
     }
 
-    private fun buildSourceSets(dependencyResolver: DependencyResolver, project: Project): Collection<KotlinSourceSetImpl>? {
+    private fun buildSourceSets(dependencyResolver: DependencyResolver, project: Project, dependencyMapper: KotlinDependencyMapper): Collection<KotlinSourceSetImpl>? {
         val kotlinExt = project.extensions.findByName("kotlin") ?: return null
         val getSourceSets = kotlinExt.javaClass.getMethodOrNull("getSourceSets") ?: return null
         @Suppress("UNCHECKED_CAST")
         val sourceSets =
             (getSourceSets(kotlinExt) as? NamedDomainObjectContainer<Named>)?.asMap?.values ?: emptyList<Named>()
-        val allSourceSets = sourceSets.mapNotNull { buildSourceSet(it, dependencyResolver, project) }
+        val allSourceSets = sourceSets.mapNotNull { buildSourceSet(it, dependencyResolver, project, dependencyMapper) }
         val map = allSourceSets.map { it.name to it }.toMap()
         val dependsOnCache = HashMap<String, Set<String>>()
         return allSourceSets.map { sourceSet ->
@@ -123,7 +130,8 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
     private fun buildSourceSet(
         gradleSourceSet: Named,
         dependencyResolver: DependencyResolver,
-        project: Project
+        project: Project,
+        dependencyMapper: KotlinDependencyMapper
     ): KotlinSourceSetImpl? {
         val sourceSetClass = gradleSourceSet.javaClass
         val getLanguageSettings = sourceSetClass.getMethodOrNull("getLanguageSettings") ?: return null
@@ -136,7 +144,14 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
         val dependencies = buildSourceSetDependencies(gradleSourceSet, dependencyResolver, project)
         @Suppress("UNCHECKED_CAST")
         val dependsOnSourceSets = (getDependsOn(gradleSourceSet) as? Set<Named>)?.mapTo(LinkedHashSet()) { it.name } ?: emptySet<String>()
-        return KotlinSourceSetImpl(gradleSourceSet.name, languageSettings, sourceDirs, resourceDirs, dependencies, dependsOnSourceSets)
+        return KotlinSourceSetImpl(
+            gradleSourceSet.name,
+            languageSettings,
+            sourceDirs,
+            resourceDirs,
+            dependencies.map { dependencyMapper.getId(it) }.distinct().toTypedArray(),
+            dependsOnSourceSets
+        )
     }
 
     private fun buildLanguageSettings(gradleLanguageSettings: Any): KotlinLanguageSettings? {
@@ -200,16 +215,18 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
     private fun buildTargets(
         sourceSetMap: Map<String, KotlinSourceSet>,
         dependencyResolver: DependencyResolver,
-        project: Project
+        project: Project,
+        dependencyMapper: KotlinDependencyMapper
     ): Collection<KotlinTarget>? {
-        return project.getTargets()?.mapNotNull { buildTarget(it, sourceSetMap, dependencyResolver, project) }
+        return project.getTargets()?.mapNotNull { buildTarget(it, sourceSetMap, dependencyResolver, project, dependencyMapper) }
     }
 
     private fun buildTarget(
         gradleTarget: Named,
         sourceSetMap: Map<String, KotlinSourceSet>,
         dependencyResolver: DependencyResolver,
-        project: Project
+        project: Project,
+        dependencyMapper: KotlinDependencyMapper
     ): KotlinTarget? {
         val targetClass = gradleTarget.javaClass
         val getPlatformType = targetClass.getMethodOrNull("getPlatformType") ?: return null
@@ -229,7 +246,7 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
 
         val gradleCompilations = getCompilations(gradleTarget) ?: return null
         val compilations = gradleCompilations.mapNotNull {
-            buildCompilation(it, disambiguationClassifier, sourceSetMap, dependencyResolver, project)
+            buildCompilation(it, disambiguationClassifier, sourceSetMap, dependencyResolver, project, dependencyMapper)
         }
         val jar = buildTargetJar(gradleTarget, project)
         val target = KotlinTargetImpl(gradleTarget.name, targetPresetName, disambiguationClassifier, platform, compilations, jar)
@@ -256,7 +273,9 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
         classifier: String?,
         sourceSetMap: Map<String, KotlinSourceSet>,
         dependencyResolver: DependencyResolver,
-        project: Project
+        project: Project,
+        dependencyMapper: KotlinDependencyMapper
+
     ): KotlinCompilationImpl? {
         val compilationClass = gradleCompilation.javaClass
         val getKotlinSourceSets = compilationClass.getMethodOrNull("getKotlinSourceSets") ?: return null
@@ -267,13 +286,14 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
         val output = buildCompilationOutput(gradleCompilation, compileKotlinTask) ?: return null
         val arguments = buildCompilationArguments(compileKotlinTask) ?: return null
         val dependencyClasspath = buildDependencyClasspath(compileKotlinTask)
-        val dependencies = buildCompilationDependencies(gradleCompilation, classifier, sourceSetMap, dependencyResolver, project)
+        val dependencies =
+            buildCompilationDependencies(gradleCompilation, classifier, sourceSetMap, dependencyResolver, project, dependencyMapper)
         val kotlinTaskProperties = getKotlinTaskProperties(compileKotlinTask)
 
         return KotlinCompilationImpl(
             gradleCompilation.name,
             kotlinSourceSets,
-            dependencies,
+            dependencies.map { dependencyMapper.getId(it) }.distinct().toTypedArray(),
             output,
             arguments,
             dependencyClasspath.toTypedArray(),
@@ -286,7 +306,8 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
         classifier: String?,
         sourceSetMap: Map<String, KotlinSourceSet>,
         dependencyResolver: DependencyResolver,
-        project: Project
+        project: Project,
+        dependencyMapper: KotlinDependencyMapper
     ): Set<KotlinDependency> {
         return LinkedHashSet<KotlinDependency>().apply {
             val transformationBuilder = MetadataDependencyTransformationBuilder(gradleCompilation)
@@ -296,7 +317,10 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
             this += buildDependencies(
                 gradleCompilation, dependencyResolver, "getRuntimeDependencyConfigurationName", "RUNTIME", project, transformationBuilder
             )
-            this += sourceSetMap[compilationFullName(gradleCompilation.name, classifier)]?.dependencies ?: emptySet()
+            this += sourceSetMap[compilationFullName(
+                gradleCompilation.name,
+                classifier
+            )]?.dependencies?.map { dependencyMapper.getDependency(it) }?.filterNotNull() ?: emptySet()
         }
     }
 
@@ -355,8 +379,8 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
         gradleSourceSet: Named,
         dependencyResolver: DependencyResolver,
         project: Project
-    ): Set<KotlinDependency> {
+    ): List<KotlinDependency> {
-        return LinkedHashSet<KotlinDependency>().apply {
+        return ArrayList<KotlinDependency>().apply {
             val transformationBuilder = MetadataDependencyTransformationBuilder(gradleSourceSet)
             this += buildDependencies(
                 gradleSourceSet, dependencyResolver, "getApiMetadataConfigurationName", "COMPILE", project, transformationBuilder

idea/kotlin-gradle-tooling/src/KotlinMPPGradleModelImpl.kt

@@ -6,15 +6,14 @@
 package org.jetbrains.kotlin.gradle
 
 import java.io.File
-import java.util.*
 import kotlin.collections.HashSet
 
-data class KotlinSourceSetImpl(
+class KotlinSourceSetImpl(
     override val name: String,
     override val languageSettings: KotlinLanguageSettings,
     override val sourceDirs: Set<File>,
     override val resourceDirs: Set<File>,
-    override val dependencies: Set<KotlinDependency>,
+    override val dependencies: Array<KotlinDependencyId>,
     override val dependsOnSourceSets: Set<String>,
     val defaultPlatform: KotlinPlatformContainerImpl = KotlinPlatformContainerImpl(),
     val defaultIsTestModule: Boolean = false
@@ -25,7 +24,7 @@ data class KotlinSourceSetImpl(
         KotlinLanguageSettingsImpl(kotlinSourceSet.languageSettings),
         HashSet(kotlinSourceSet.sourceDirs),
         HashSet(kotlinSourceSet.resourceDirs),
-        kotlinSourceSet.dependencies.map { it.deepCopy(cloningCache) }.toSet(),
+        kotlinSourceSet.dependencies,
         HashSet(kotlinSourceSet.dependsOnSourceSets),
         KotlinPlatformContainerImpl(kotlinSourceSet.actualPlatforms),
         kotlinSourceSet.isTestModule
@@ -85,7 +84,7 @@ data class KotlinCompilationArgumentsImpl(
 data class KotlinCompilationImpl(
     override val name: String,
     override val sourceSets: Collection<KotlinSourceSet>,
-    override val dependencies: Set<KotlinDependency>,
+    override val dependencies: Array<KotlinDependencyId>,
     override val output: KotlinCompilationOutput,
     override val arguments: KotlinCompilationArguments,
     override val dependencyClasspath: Array<String>,
@@ -100,7 +99,7 @@ data class KotlinCompilationImpl(
                 cloningCache[initialSourceSet] = it
             }
         }.toList(),
-        kotlinCompilation.dependencies.map { it.deepCopy(cloningCache) }.toSet(),
+        kotlinCompilation.dependencies,
         KotlinCompilationOutputImpl(kotlinCompilation.output),
         KotlinCompilationArgumentsImpl(kotlinCompilation.arguments),
         kotlinCompilation.dependencyClasspath,
@@ -160,7 +159,8 @@ data class KotlinMPPGradleModelImpl(
     override val sourceSets: Map<String, KotlinSourceSet>,
     override val targets: Collection<KotlinTarget>,
     override val extraFeatures: ExtraFeatures,
-    override val kotlinNativeHome: String
+    override val kotlinNativeHome: String,
+    override val dependencyMap: Map<KotlinDependencyId, KotlinDependency>
 ) : KotlinMPPGradleModel {
 
     constructor(mppModel: KotlinMPPGradleModel, cloningCache: MutableMap<Any, Any>) : this(
@@ -176,7 +176,8 @@ data class KotlinMPPGradleModelImpl(
             }
         }.toList(),
         ExtraFeaturesImpl(mppModel.extraFeatures.coroutinesState, mppModel.extraFeatures.isHMPPEnabled),
-        mppModel.kotlinNativeHome
+        mppModel.kotlinNativeHome,
+        mppModel.dependencyMap.map { it.key to it.value.deepCopy(cloningCache) }.toMap()
     )
 }