From f93562299c339925bdc76e534463c44137bf6750 Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Mon, 8 Feb 2016 16:05:55 +0300 Subject: [PATCH] Reuse code from build-common KT-8487 --- .../incremental/IncrementalCacheImpl.kt | 13 +-- .../jetbrains/kotlin/incremental/buildUtil.kt | 105 ++++++++++++++---- .../kotlin/jps/build/KotlinBuilder.kt | 93 ++-------------- .../incremental/JpsIncrementalCacheImpl.kt | 2 +- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 32 ++---- 5 files changed, 104 insertions(+), 141 deletions(-) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt index d8e0bed463c..dbe875954de 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt @@ -92,12 +92,11 @@ open class IncrementalCacheImpl( private val dependents = arrayListOf>() private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory: $target" } } - // TODO: review - val dependentsWithThis: Sequence> - get() = sequenceOf(this).plus(dependents.asSequence()) - - internal val dependentCaches: Iterable> - get() = dependents + val thisWithDependentCaches: Iterable> by lazy { + val result = arrayListOf(this) + result.addAll(dependents) + result + } override fun registerInline(fromPath: String, jvmSignature: String, toPath: String) { } @@ -578,7 +577,7 @@ open class IncrementalCacheImpl( val removedFqNames = removedClasses.map { it.fqNameForClassNameWithoutDollars }.toSet() - for (cache in dependentsWithThis) { + for (cache in thisWithDependentCaches) { val parentsFqNames = hashSetOf() val childrenFqNames = hashSetOf() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt index f7ba342aff3..741e4ca0ebe 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt @@ -174,36 +174,94 @@ fun OutputItemsCollectorImpl.generatedFiles( } } -fun CompilationResult.dirtyLookups( - caches: Sequence> -): Iterable = - changes.asIterable().flatMap { change -> - when (change) { - is ChangeInfo.SignatureChanged -> { - val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, caches) - fqNames.map { - val scope = it.parent().asString() - val name = it.shortName().identifier - LookupSymbol(name, scope) - } - } - is ChangeInfo.MembersChanged -> { - val scopes = withSubtypes(change.fqName, caches).map { it.asString() } - change.names.flatMap { name -> scopes.map { scope -> LookupSymbol(name, scope) } } - } - else -> listOf() +data class DirtyData( + val dirtyLookupSymbols: Iterable, + val dirtyClassesFqNames: Iterable +) + +fun CompilationResult.getDirtyData( + caches: Iterable>, + log: (String)->Unit +): DirtyData { + val dirtyLookupSymbols = HashSet() + val dirtyClassesFqNames = HashSet() + + for (change in changes) { + log("Process $change") + + if (change is ChangeInfo.SignatureChanged) { + val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, caches) + + for (classFqName in fqNames) { + assert(!classFqName.isRoot) { "$classFqName is root when processing $change" } + + val scope = classFqName.parent().asString() + val name = classFqName.shortName().identifier + dirtyLookupSymbols.add(LookupSymbol(name, scope)) } } + else if (change is ChangeInfo.MembersChanged) { + val fqNames = withSubtypes(change.fqName, caches) + // need to recompile subtypes because changed member might break override + dirtyClassesFqNames.addAll(fqNames) + for (name in change.names) { + for (fqName in fqNames) { + dirtyLookupSymbols.add(LookupSymbol(name, fqName.asString())) + } + } + } + } + + return DirtyData(dirtyLookupSymbols, dirtyClassesFqNames) +} + +fun mapLookupSymbolsToFiles( + lookupStorage: LookupStorage, + lookupSymbols: Iterable, + log: (String)->Unit, + excludes: Set = emptySet() +): Set { + val dirtyFiles = HashSet() + + for (lookup in lookupSymbols) { + val affectedFiles = lookupStorage.get(lookup).map(::File).filter { it !in excludes } + log("${lookup.scope}#${lookup.name} caused recompilation of: $affectedFiles") + dirtyFiles.addAll(affectedFiles) + } + + return dirtyFiles +} + +fun mapClassesFqNamesToFiles( + caches: Iterable>, + classesFqNames: Iterable, + log: (String)->Unit, + excludes: Set = emptySet() +): Set { + val dirtyFiles = HashSet() + + for (cache in caches) { + for (dirtyClassFqName in classesFqNames) { + val srcFile = cache.getSourceFileIfClass(dirtyClassFqName) + if (srcFile == null || srcFile in excludes) continue + + log("Class $dirtyClassFqName caused recompilation of: $srcFile") + dirtyFiles.add(srcFile) + } + } + + return dirtyFiles +} private fun File.isJavaFile() = extension.equals(JavaFileType.INSTANCE.defaultExtension, ignoreCase = true) private fun findSrcDirRoot(file: File, roots: Iterable): File? = roots.firstOrNull { FileUtil.isAncestor(it, file, false) } -private fun withSubtypes( +private fun withSubtypes( typeFqName: FqName, - caches: Sequence> + caches: Iterable> ): Set { val types = LinkedList(listOf(typeFqName)) val subtypes = hashSetOf() @@ -211,9 +269,10 @@ private fun withSubtypes( while (types.isNotEmpty()) { val unprocessedType = types.pollFirst() - caches.flatMap { it.getSubtypesOf(unprocessedType) } - .filter { it !in subtypes } - .forEach { types.addLast(it) } + caches.asSequence() + .flatMap { it.getSubtypesOf(unprocessedType) } + .filter { it !in subtypes } + .forEach { types.addLast(it) } subtypes.add(unprocessedType) } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 60fb5928698..f4aec197e4b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -62,7 +62,6 @@ import org.jetbrains.kotlin.jps.incremental.* import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents import org.jetbrains.kotlin.modules.TargetId -import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.CompilationCanceledException import org.jetbrains.kotlin.progress.CompilationCanceledStatus import org.jetbrains.kotlin.utils.JsLibraryUtils @@ -773,88 +772,20 @@ private fun CompilationResult.doProcessChangesUsingLookups( compiledFiles: Set, dataManager: BuildDataManager, fsOperations: FSOperationsHelper, - caches: Collection> + caches: Iterable> ) { - val dirtyLookupSymbols = HashSet() - val dirtyClassesFqNames = HashSet() val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider) - val allCaches: Sequence> = caches.asSequence().flatMap { it.dependentsWithThis } + val allCaches = caches.flatMap { it.thisWithDependentCaches } + val logAction = { logStr: String -> KotlinBuilder.LOG.debug(logStr) } - KotlinBuilder.LOG.debug("Start processing changes") - - for (change in changes) { - KotlinBuilder.LOG.debug("Process $change") - - if (change is ChangeInfo.SignatureChanged) { - val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, allCaches) - - for (classFqName in fqNames) { - assert(!classFqName.isRoot) { "classFqName is root when processing $change" } - - val scope = classFqName.parent().asString() - val name = classFqName.shortName().identifier - dirtyLookupSymbols.add(LookupSymbol(name, scope)) - } - } - else if (change is ChangeInfo.MembersChanged) { - val fqNames = withSubtypes(change.fqName, allCaches) - // need to recompile subtypes because changed member might break override - dirtyClassesFqNames.addAll(fqNames) - - change.names.forAllPairs(fqNames) { name, fqName -> - dirtyLookupSymbols.add(LookupSymbol(name, fqName.asString())) - } - } - } - - val dirtyFiles = HashSet() - - for (lookup in dirtyLookupSymbols) { - val affectedFiles = lookupStorage.get(lookup).map(::File) - - KotlinBuilder.LOG.debug { "${lookup.scope}#${lookup.name} caused recompilation of: $affectedFiles" } - - dirtyFiles.addAll(affectedFiles) - } - - for (cache in allCaches) { - for (dirtyClassFqName in dirtyClassesFqNames) { - val srcFile = cache.getSourceFileIfClass(dirtyClassFqName) ?: continue - dirtyFiles.add(srcFile) - } - } + logAction("Start processing changes") + val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(allCaches, logAction) + val dirtyFiles = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction) + + mapClassesFqNamesToFiles(allCaches, dirtyClassFqNames, logAction) fsOperations.markFiles(dirtyFiles.asIterable(), excludeFiles = compiledFiles) - KotlinBuilder.LOG.debug("End of processing changes") -} -/** - * Returns type with its subtypes transitively - * - * For example: - * open class A - * open class B : A() - * class C : B() - * withSubtypes(A) will return [A, B, C] - */ -private fun withSubtypes( - typeFqName: FqName, - caches: Sequence> -): Set { - val types = LinkedList(listOf(typeFqName)) - val subtypes = hashSetOf() - - while (types.isNotEmpty()) { - val unprocessedType = types.pollFirst() - - caches.flatMap { it.getSubtypesOf(unprocessedType) } - .filter { it !in subtypes } - .forEach { types.addLast(it) } - - subtypes.add(unprocessedType) - } - - return subtypes + logAction("End of processing changes") } private fun getLookupTracker(project: JpsProject): LookupTracker { @@ -948,14 +879,6 @@ private fun hasKotlinDirtyOrRemovedFiles( return chunk.targets.any { KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, it).isNotEmpty() } } -private inline fun Iterable.forAllPairs(other: Iterable, fn: (T, R)->Unit) { - for (t in this) { - for (r in other) { - fn(t, r) - } - } -} - private inline fun Logger.debug(message: ()->String) { if (isDebugEnabled) { debug(message()) diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCacheImpl.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCacheImpl.kt index 8c199778276..1f8782e6712 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCacheImpl.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCacheImpl.kt @@ -60,7 +60,7 @@ class JpsIncrementalCacheImpl( for ((className, functions) in dirtyInlineFunctionsMap.getEntries()) { val classFilePath = getClassFilePath(className.internalName) - for (cache in dependentsWithThis) { + for (cache in thisWithDependentCaches) { val targetFiles = functions.flatMap { (cache as JpsIncrementalCacheImpl).inlinedTo[classFilePath, it] } result.addAll(targetFiles) } diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index eb0d9a3f66d..5db785f0dc9 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -238,6 +238,7 @@ open class KotlinCompile() : AbstractKotlinCompile() { var currentRemoved = removed val allGeneratedFiles = hashSetOf>() val compiledSourcesSet = hashSetOf() + val logAction = { logStr: String -> logger.kotlinInfo(logStr) } fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl { val cacheDir = File(cachesBaseDir, "increCache.${target.name}") @@ -255,15 +256,6 @@ open class KotlinCompile() : AbstractKotlinCompile() { innerClasses.flatMap { it.findLookupSymbols() } } - fun Iterable.files(filesFilter: (File) -> Boolean = { true }, logAction: (LookupSymbol, Iterable) -> Unit = { l,fs -> }): Iterable = - flatMap { lookup -> - val files = lookupStorage.get(lookup).map(::File).filter(filesFilter) - if (files.any()) { - logAction(lookup, files) - } - files - } - fun dirtyLookupSymbolsFromRemovedKotlinFiles(): List { val removedKotlinFiles = removed.filter { it.isKotlinFile() } return if (removedKotlinFiles.isNotEmpty()) @@ -298,11 +290,7 @@ open class KotlinCompile() : AbstractKotlinCompile() { if (lookupSymbols.any()) { val kotlinModifiedFilesSet = modifiedKotlinFiles.toHashSet() - val dirtyFilesFromLookups = lookupSymbols.files( - filesFilter = { it !in kotlinModifiedFilesSet }, - logAction = { lookup, files -> - logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}") - }) + val dirtyFilesFromLookups = mapLookupSymbolsToFiles(lookupStorage, lookupSymbols, logAction, excludes = kotlinModifiedFilesSet) modifiedKotlinFiles.addAll(dirtyFilesFromLookups) } @@ -428,9 +416,9 @@ open class KotlinCompile() : AbstractKotlinCompile() { val (exitCode, generatedFiles) = compileChanged(targets, existingSource.toSet(), outputDir, args, ::getIncrementalCache, lookupTracker) allGeneratedFiles.addAll(generatedFiles) - val changes = updateIncrementalCaches(targets, generatedFiles, - compiledWithErrors = exitCode != ExitCode.OK, - getIncrementalCache = { caches[it]!! }) + val compilationResult = updateIncrementalCaches(targets, generatedFiles, + compiledWithErrors = exitCode != ExitCode.OK, + getIncrementalCache = { caches[it]!! }) lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved) allCachesVersions().forEach { it.saveIfNeeded() } @@ -439,14 +427,8 @@ open class KotlinCompile() : AbstractKotlinCompile() { if (!isIncrementalDecided) break; compiledSourcesSet.addAll(sourcesToCompile) - - val dirtyLookups = changes.dirtyLookups(caches.values.asSequence()) - val newDirtyFiles = dirtyLookups.files( - filesFilter = { it !in compiledSourcesSet }, - logAction = { lookup, files -> - logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}") - }) - sourcesToCompile = newDirtyFiles.filter { it in sources }.toSet() + val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(caches.values, logAction) + sourcesToCompile = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction, excludes = compiledSourcesSet) if (currentRemoved.any()) { currentRemoved = listOf()