azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement dependency deduplication in order to reduce memory

Browse Source 
consumption during import process
This commit is contained in:
Andrey Uskov
2019-07-05 10:38:48 +03:00
parent 09c2b2800c
commit 40633768bc
4 changed files with 84 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
idea/idea-gradle/src/org/jetbrains/kotlin/idea/configuration/KotlinMPPGradleProjectResolver.kt
+11 -10
View File
@@ -298,7 +298,7 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
it.kotlinSourceSet = kotlinSourceSet
}
if (existingSourceSetDataNode == null) {
sourceSetMap[moduleId] = Pair(compilationDataNode, createExternalSourceSet(compilation, compilationData))
sourceSetMap[moduleId] = Pair(compilationDataNode, createExternalSourceSet(compilation, compilationData, mppModel))
}
}
@@ -348,7 +348,7 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
it.kotlinSourceSet = kotlinSourceSet
}
if (existingSourceSetDataNode == null) {
sourceSetMap[moduleId] = Pair(sourceSetDataNode, createExternalSourceSet(sourceSet, sourceSetData))
sourceSetMap[moduleId] = Pair(sourceSetDataNode, createExternalSourceSet(sourceSet, sourceSetData, mppModel))
}
}
@@ -417,7 +417,8 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
val processedModuleIds = HashSet<String>()
processCompilations(gradleModule, mppModel, ideModule, resolverCtx) { dataNode, compilation ->
if (processedModuleIds.add(getKotlinModuleId(gradleModule, compilation, resolverCtx))) {
val substitutedDependencies = substitutor.substituteDependencies(compilation.dependencies)
val substitutedDependencies =
substitutor.substituteDependencies(compilation.dependencies.mapNotNull { mppModel.dependencyMap[it] })
buildDependencies(
resolverCtx,
sourceSetMap,
@@ -494,8 +495,8 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
}
if (processedModuleIds.add(getKotlinModuleId(gradleModule, sourceSet, resolverCtx))) {
val mergedDependencies = LinkedHashSet<KotlinDependency>().apply {
addAll(sourceSet.dependencies)
dependeeSourceSets.flatMapTo(this) { it.dependencies }
addAll(sourceSet.dependencies.mapNotNull { mppModel.dependencyMap[it] })
dependeeSourceSets.flatMapTo(this) { it.dependencies.mapNotNull { mppModel.dependencyMap[it] } }
}
buildDependencies(
resolverCtx,
@@ -629,14 +630,14 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
return PathUtilRt.suggestFileName(moduleName.toString(), true, false)
}
private fun createExternalSourceSet(compilation: KotlinCompilation, compilationData: GradleSourceSetData): ExternalSourceSet {
private fun createExternalSourceSet(compilation: KotlinCompilation, compilationData: GradleSourceSetData, mppModel: KotlinMPPGradleModel): ExternalSourceSet {
return DefaultExternalSourceSet().also { sourceSet ->
val effectiveClassesDir = compilation.output.effectiveClassesDir
val resourcesDir = compilation.output.resourcesDir
sourceSet.name = compilation.fullName()
sourceSet.targetCompatibility = compilationData.targetCompatibility
sourceSet.dependencies += compilation.dependencies
sourceSet.dependencies += compilation.dependencies.mapNotNull { mppModel.dependencyMap[it] }
//TODO after applying patch to IDEA core uncomment the following line:
// sourceSet.isTest = compilation.sourceSets.filter { isTestModule }.isNotEmpty()
// It will allow to get rid of hacks with guessing module type in DataServices and obtain properly set productionOnTest flags
@@ -665,15 +666,15 @@ open class KotlinMPPGradleProjectResolver : AbstractProjectResolverExtension() {
}
private fun createExternalSourceSet(ktSourceSet: KotlinSourceSet, ktSourceSetData: GradleSourceSetData): ExternalSourceSet {
private fun createExternalSourceSet(ktSourceSet: KotlinSourceSet, ktSourceSetData: GradleSourceSetData, mppModel: KotlinMPPGradleModel): ExternalSourceSet {
return DefaultExternalSourceSet().also { sourceSet ->
sourceSet.name = ktSourceSet.name
sourceSet.targetCompatibility = ktSourceSetData.targetCompatibility
sourceSet.dependencies += ktSourceSet.dependencies
sourceSet.dependencies += ktSourceSet.dependencies.mapNotNull { mppModel.dependencyMap[it] }
// BUNCH: 191 Can't use property because there's no getter in 192 and thus it isn't property anymore
@Suppress("UsePropertyAccessSyntax")
sourceSet.setSources(linkedMapOf(
sourceSet.setSources(linkedMapOf<IExternalSystemSourceType, ExternalSourceDirectorySet>(
ktSourceSet.sourceType to DefaultExternalSourceDirectorySet().also { dirSet ->
dirSet.srcDirs = ktSourceSet.sourceDirs
},
idea/kotlin-gradle-tooling/src/KotlinMPPGradleModel.kt
+22 -1
View File
@@ -10,8 +10,28 @@ import org.jetbrains.plugins.gradle.model.ModelFactory
import java.io.File
import java.io.Serializable
typealias KotlinDependencyId = Long
typealias KotlinDependency = ExternalDependency
class KotlinDependencyMapper {
private var currentIndex: KotlinDependencyId = 0
private val idToDependency = HashMap<KotlinDependencyId, KotlinDependency>()
private val dependencyToId = HashMap<KotlinDependency, KotlinDependencyId>()
fun getDependency(id: KotlinDependencyId) = idToDependency[id]
fun getId(dependency: KotlinDependency): KotlinDependencyId {
return dependencyToId[dependency] ?: let {
currentIndex++
dependencyToId[dependency] = currentIndex
idToDependency[currentIndex] = dependency
return currentIndex
}
}
fun toDependencyMap(): Map<KotlinDependencyId, KotlinDependency> = idToDependency
}
fun KotlinDependency.deepCopy(cache: MutableMap<Any, Any>): KotlinDependency {
val cachedValue = cache[this] as? KotlinDependency
return if (cachedValue != null) {
@@ -25,7 +45,7 @@ fun KotlinDependency.deepCopy(cache: MutableMap<Any, Any>): KotlinDependency {
interface KotlinModule : Serializable {
val name: String
val dependencies: Set<KotlinDependency>
val dependencies: Array<KotlinDependencyId>
val isTestModule: Boolean
}
@@ -133,6 +153,7 @@ interface ExtraFeatures : Serializable {
}
interface KotlinMPPGradleModel : Serializable {
val dependencyMap: Map<KotlinDependencyId, KotlinDependency>
val sourceSets: Map<String, KotlinSourceSet>
val targets: Collection<KotlinTarget>
val extraFeatures: ExtraFeatures
idea/kotlin-gradle-tooling/src/KotlinMPPGradleModelBuilder.kt
+42 -18
View File
@@ -45,14 +45,21 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
true,
SourceSetCachedFinder(project)
)
val sourceSets = buildSourceSets(dependencyResolver, project) ?: return null
val dependencyMapper = KotlinDependencyMapper()
val sourceSets = buildSourceSets(dependencyResolver, project, dependencyMapper) ?: return null
val sourceSetMap = sourceSets.map { it.name to it }.toMap()
val targets = buildTargets(sourceSetMap, dependencyResolver, project) ?: return null
val targets = buildTargets(sourceSetMap, dependencyResolver, project, dependencyMapper) ?: return null
computeSourceSetsDeferredInfo(sourceSetMap, targets, isHMPPEnabled(project))
val coroutinesState = getCoroutinesState(project)
reportUnresolvedDependencies(targets)
val kotlinNativeHome = KotlinNativeHomeEvaluator.getKotlinNativeHome(project) ?: NO_KOTLIN_NATIVE_HOME
return KotlinMPPGradleModelImpl(sourceSetMap, targets, ExtraFeaturesImpl(coroutinesState, isHMPPEnabled(project)), kotlinNativeHome)
return KotlinMPPGradleModelImpl(
sourceSetMap,
targets,
ExtraFeaturesImpl(coroutinesState, isHMPPEnabled(project)),
kotlinNativeHome,
dependencyMapper.toDependencyMap()
)
}
private fun isHMPPEnabled(project: Project): Boolean {
@@ -97,13 +104,13 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
}
}
private fun buildSourceSets(dependencyResolver: DependencyResolver, project: Project): Collection<KotlinSourceSetImpl>? {
private fun buildSourceSets(dependencyResolver: DependencyResolver, project: Project, dependencyMapper: KotlinDependencyMapper): Collection<KotlinSourceSetImpl>? {
val kotlinExt = project.extensions.findByName("kotlin") ?: return null
val getSourceSets = kotlinExt.javaClass.getMethodOrNull("getSourceSets") ?: return null
@Suppress("UNCHECKED_CAST")
val sourceSets =
(getSourceSets(kotlinExt) as? NamedDomainObjectContainer<Named>)?.asMap?.values ?: emptyList<Named>()
val allSourceSets = sourceSets.mapNotNull { buildSourceSet(it, dependencyResolver, project) }
val allSourceSets = sourceSets.mapNotNull { buildSourceSet(it, dependencyResolver, project, dependencyMapper) }
val map = allSourceSets.map { it.name to it }.toMap()
val dependsOnCache = HashMap<String, Set<String>>()
return allSourceSets.map { sourceSet ->
@@ -123,7 +130,8 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
private fun buildSourceSet(
gradleSourceSet: Named,
dependencyResolver: DependencyResolver,
project: Project
project: Project,
dependencyMapper: KotlinDependencyMapper
): KotlinSourceSetImpl? {
val sourceSetClass = gradleSourceSet.javaClass
val getLanguageSettings = sourceSetClass.getMethodOrNull("getLanguageSettings") ?: return null
@@ -136,7 +144,14 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
val dependencies = buildSourceSetDependencies(gradleSourceSet, dependencyResolver, project)
@Suppress("UNCHECKED_CAST")
val dependsOnSourceSets = (getDependsOn(gradleSourceSet) as? Set<Named>)?.mapTo(LinkedHashSet()) { it.name } ?: emptySet<String>()
return KotlinSourceSetImpl(gradleSourceSet.name, languageSettings, sourceDirs, resourceDirs, dependencies, dependsOnSourceSets)
return KotlinSourceSetImpl(
gradleSourceSet.name,
languageSettings,
sourceDirs,
resourceDirs,
dependencies.map { dependencyMapper.getId(it) }.distinct().toTypedArray(),
dependsOnSourceSets
)
}
private fun buildLanguageSettings(gradleLanguageSettings: Any): KotlinLanguageSettings? {
@@ -200,16 +215,18 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
private fun buildTargets(
sourceSetMap: Map<String, KotlinSourceSet>,
dependencyResolver: DependencyResolver,
project: Project
project: Project,
dependencyMapper: KotlinDependencyMapper
): Collection<KotlinTarget>? {
return project.getTargets()?.mapNotNull { buildTarget(it, sourceSetMap, dependencyResolver, project) }
return project.getTargets()?.mapNotNull { buildTarget(it, sourceSetMap, dependencyResolver, project, dependencyMapper) }
}
private fun buildTarget(
gradleTarget: Named,
sourceSetMap: Map<String, KotlinSourceSet>,
dependencyResolver: DependencyResolver,
project: Project
project: Project,
dependencyMapper: KotlinDependencyMapper
): KotlinTarget? {
val targetClass = gradleTarget.javaClass
val getPlatformType = targetClass.getMethodOrNull("getPlatformType") ?: return null
@@ -229,7 +246,7 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
val gradleCompilations = getCompilations(gradleTarget) ?: return null
val compilations = gradleCompilations.mapNotNull {
buildCompilation(it, disambiguationClassifier, sourceSetMap, dependencyResolver, project)
buildCompilation(it, disambiguationClassifier, sourceSetMap, dependencyResolver, project, dependencyMapper)
}
val jar = buildTargetJar(gradleTarget, project)
val target = KotlinTargetImpl(gradleTarget.name, targetPresetName, disambiguationClassifier, platform, compilations, jar)
@@ -256,7 +273,9 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
classifier: String?,
sourceSetMap: Map<String, KotlinSourceSet>,
dependencyResolver: DependencyResolver,
project: Project
project: Project,
dependencyMapper: KotlinDependencyMapper
): KotlinCompilationImpl? {
val compilationClass = gradleCompilation.javaClass
val getKotlinSourceSets = compilationClass.getMethodOrNull("getKotlinSourceSets") ?: return null
@@ -267,13 +286,14 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
val output = buildCompilationOutput(gradleCompilation, compileKotlinTask) ?: return null
val arguments = buildCompilationArguments(compileKotlinTask) ?: return null
val dependencyClasspath = buildDependencyClasspath(compileKotlinTask)
val dependencies = buildCompilationDependencies(gradleCompilation, classifier, sourceSetMap, dependencyResolver, project)
val dependencies =
buildCompilationDependencies(gradleCompilation, classifier, sourceSetMap, dependencyResolver, project, dependencyMapper)
val kotlinTaskProperties = getKotlinTaskProperties(compileKotlinTask)
return KotlinCompilationImpl(
gradleCompilation.name,
kotlinSourceSets,
dependencies,
dependencies.map { dependencyMapper.getId(it) }.distinct().toTypedArray(),
output,
arguments,
dependencyClasspath.toTypedArray(),
@@ -286,7 +306,8 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
classifier: String?,
sourceSetMap: Map<String, KotlinSourceSet>,
dependencyResolver: DependencyResolver,
project: Project
project: Project,
dependencyMapper: KotlinDependencyMapper
): Set<KotlinDependency> {
return LinkedHashSet<KotlinDependency>().apply {
val transformationBuilder = MetadataDependencyTransformationBuilder(gradleCompilation)
@@ -296,7 +317,10 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
this += buildDependencies(
gradleCompilation, dependencyResolver, "getRuntimeDependencyConfigurationName", "RUNTIME", project, transformationBuilder
)
this += sourceSetMap[compilationFullName(gradleCompilation.name, classifier)]?.dependencies ?: emptySet()
this += sourceSetMap[compilationFullName(
gradleCompilation.name,
classifier
)]?.dependencies?.map { dependencyMapper.getDependency(it) }?.filterNotNull() ?: emptySet()
}
}
@@ -355,8 +379,8 @@ class KotlinMPPGradleModelBuilder : ModelBuilderService {
gradleSourceSet: Named,
dependencyResolver: DependencyResolver,
project: Project
): Set<KotlinDependency> {
return LinkedHashSet<KotlinDependency>().apply {
): List<KotlinDependency> {
return ArrayList<KotlinDependency>().apply {
val transformationBuilder = MetadataDependencyTransformationBuilder(gradleSourceSet)
this += buildDependencies(
gradleSourceSet, dependencyResolver, "getApiMetadataConfigurationName", "COMPILE", project, transformationBuilder
idea/kotlin-gradle-tooling/src/KotlinMPPGradleModelImpl.kt
+9 -8
View File
@@ -6,15 +6,14 @@
package org.jetbrains.kotlin.gradle
import java.io.File
import java.util.*
import kotlin.collections.HashSet
data class KotlinSourceSetImpl(
class KotlinSourceSetImpl(
override val name: String,
override val languageSettings: KotlinLanguageSettings,
override val sourceDirs: Set<File>,
override val resourceDirs: Set<File>,
override val dependencies: Set<KotlinDependency>,
override val dependencies: Array<KotlinDependencyId>,
override val dependsOnSourceSets: Set<String>,
val defaultPlatform: KotlinPlatformContainerImpl = KotlinPlatformContainerImpl(),
val defaultIsTestModule: Boolean = false
@@ -25,7 +24,7 @@ data class KotlinSourceSetImpl(
KotlinLanguageSettingsImpl(kotlinSourceSet.languageSettings),
HashSet(kotlinSourceSet.sourceDirs),
HashSet(kotlinSourceSet.resourceDirs),
kotlinSourceSet.dependencies.map { it.deepCopy(cloningCache) }.toSet(),
kotlinSourceSet.dependencies,
HashSet(kotlinSourceSet.dependsOnSourceSets),
KotlinPlatformContainerImpl(kotlinSourceSet.actualPlatforms),
kotlinSourceSet.isTestModule
@@ -85,7 +84,7 @@ data class KotlinCompilationArgumentsImpl(
data class KotlinCompilationImpl(
override val name: String,
override val sourceSets: Collection<KotlinSourceSet>,
override val dependencies: Set<KotlinDependency>,
override val dependencies: Array<KotlinDependencyId>,
override val output: KotlinCompilationOutput,
override val arguments: KotlinCompilationArguments,
override val dependencyClasspath: Array<String>,
@@ -100,7 +99,7 @@ data class KotlinCompilationImpl(
cloningCache[initialSourceSet] = it
}
}.toList(),
kotlinCompilation.dependencies.map { it.deepCopy(cloningCache) }.toSet(),
kotlinCompilation.dependencies,
KotlinCompilationOutputImpl(kotlinCompilation.output),
KotlinCompilationArgumentsImpl(kotlinCompilation.arguments),
kotlinCompilation.dependencyClasspath,
@@ -160,7 +159,8 @@ data class KotlinMPPGradleModelImpl(
override val sourceSets: Map<String, KotlinSourceSet>,
override val targets: Collection<KotlinTarget>,
override val extraFeatures: ExtraFeatures,
override val kotlinNativeHome: String
override val kotlinNativeHome: String,
override val dependencyMap: Map<KotlinDependencyId, KotlinDependency>
) : KotlinMPPGradleModel {
constructor(mppModel: KotlinMPPGradleModel, cloningCache: MutableMap<Any, Any>) : this(
@@ -176,7 +176,8 @@ data class KotlinMPPGradleModelImpl(
}
}.toList(),
ExtraFeaturesImpl(mppModel.extraFeatures.coroutinesState, mppModel.extraFeatures.isHMPPEnabled),
mppModel.kotlinNativeHome
mppModel.kotlinNativeHome,
mppModel.dependencyMap.map { it.key to it.value.deepCopy(cloningCache) }.toMap()
)
}