Reuse code from build-common

KT-8487

build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt

@@ -92,12 +92,11 @@ open class IncrementalCacheImpl<Target>(
     private val dependents = arrayListOf<IncrementalCacheImpl<Target>>()
     private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory: $target" } }
 
-    // TODO: review
+    val thisWithDependentCaches: Iterable<IncrementalCacheImpl<Target>> by lazy {
-    val dependentsWithThis: Sequence<IncrementalCacheImpl<Target>>
+        val result = arrayListOf(this)
-        get() = sequenceOf(this).plus(dependents.asSequence())
+        result.addAll(dependents)
-
+        result
-    internal val dependentCaches: Iterable<IncrementalCacheImpl<Target>>
+    }
-        get() = dependents
 
     override fun registerInline(fromPath: String, jvmSignature: String, toPath: String) {
     }
@@ -578,7 +577,7 @@ open class IncrementalCacheImpl<Target>(
 
         val removedFqNames = removedClasses.map { it.fqNameForClassNameWithoutDollars }.toSet()
 
-        for (cache in dependentsWithThis) {
+        for (cache in thisWithDependentCaches) {
             val parentsFqNames = hashSetOf<FqName>()
             val childrenFqNames = hashSetOf<FqName>()
 

build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt

@@ -174,36 +174,94 @@ fun<Target> OutputItemsCollectorImpl.generatedFiles(
     }
 }
 
-fun<Target> CompilationResult.dirtyLookups(
+data class DirtyData(
-        caches: Sequence<IncrementalCacheImpl<TargetId>>
+        val dirtyLookupSymbols: Iterable<LookupSymbol>,
-): Iterable<LookupSymbol> =
+        val dirtyClassesFqNames: Iterable<FqName>
-        changes.asIterable().flatMap { change ->
+)
-            when (change) {
+
-                is ChangeInfo.SignatureChanged -> {
+fun <Target> CompilationResult.getDirtyData(
-                    val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, caches)
+        caches: Iterable<IncrementalCacheImpl<Target>>,
-                    fqNames.map {
+        log: (String)->Unit
-                        val scope = it.parent().asString()
+): DirtyData {
-                        val name = it.shortName().identifier
+    val dirtyLookupSymbols = HashSet<LookupSymbol>()
-                        LookupSymbol(name, scope)
+    val dirtyClassesFqNames = HashSet<FqName>()
-                    }
+
-                }
+    for (change in changes) {
-                is ChangeInfo.MembersChanged -> {
+        log("Process $change")
-                    val scopes = withSubtypes(change.fqName, caches).map { it.asString() }
+
-                    change.names.flatMap { name -> scopes.map { scope -> LookupSymbol(name, scope) } }
+        if (change is ChangeInfo.SignatureChanged) {
-                }
+            val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, caches)
-                else -> listOf<LookupSymbol>()
+
+            for (classFqName in fqNames) {
+                assert(!classFqName.isRoot) { "$classFqName is root when processing $change" }
+
+                val scope = classFqName.parent().asString()
+                val name = classFqName.shortName().identifier
+                dirtyLookupSymbols.add(LookupSymbol(name, scope))
             }
         }
+        else if (change is ChangeInfo.MembersChanged) {
+            val fqNames = withSubtypes(change.fqName, caches)
+            // need to recompile subtypes because changed member might break override
+            dirtyClassesFqNames.addAll(fqNames)
 
+            for (name in change.names) {
+                for (fqName in fqNames) {
+                    dirtyLookupSymbols.add(LookupSymbol(name, fqName.asString()))
+                }
+            }
+        }
+    }
+
+    return DirtyData(dirtyLookupSymbols, dirtyClassesFqNames)
+}
+
+fun mapLookupSymbolsToFiles(
+        lookupStorage: LookupStorage,
+        lookupSymbols: Iterable<LookupSymbol>,
+        log: (String)->Unit,
+        excludes: Set<File> = emptySet()
+): Set<File> {
+    val dirtyFiles = HashSet<File>()
+
+    for (lookup in lookupSymbols) {
+        val affectedFiles = lookupStorage.get(lookup).map(::File).filter { it !in excludes }
+        log("${lookup.scope}#${lookup.name} caused recompilation of: $affectedFiles")
+        dirtyFiles.addAll(affectedFiles)
+    }
+
+    return dirtyFiles
+}
+
+fun <Target> mapClassesFqNamesToFiles(
+        caches: Iterable<IncrementalCacheImpl<Target>>,
+        classesFqNames: Iterable<FqName>,
+        log: (String)->Unit,
+        excludes: Set<File> = emptySet()
+): Set<File> {
+    val dirtyFiles = HashSet<File>()
+
+    for (cache in caches) {
+        for (dirtyClassFqName in classesFqNames) {
+            val srcFile = cache.getSourceFileIfClass(dirtyClassFqName)
+            if (srcFile == null || srcFile in excludes) continue
+
+            log("Class $dirtyClassFqName caused recompilation of: $srcFile")
+            dirtyFiles.add(srcFile)
+        }
+    }
+
+    return dirtyFiles
+}
 
 private fun File.isJavaFile() = extension.equals(JavaFileType.INSTANCE.defaultExtension, ignoreCase = true)
 
 private fun findSrcDirRoot(file: File, roots: Iterable<File>): File? =
         roots.firstOrNull { FileUtil.isAncestor(it, file, false) }
 
-private fun<TargetId> withSubtypes(
+private fun <Target> withSubtypes(
         typeFqName: FqName,
-        caches: Sequence<IncrementalCacheImpl<TargetId>>
+        caches: Iterable<IncrementalCacheImpl<Target>>
 ): Set<FqName> {
     val types = LinkedList(listOf(typeFqName))
     val subtypes = hashSetOf<FqName>()
@@ -211,9 +269,10 @@ private fun<TargetId> withSubtypes(
     while (types.isNotEmpty()) {
         val unprocessedType = types.pollFirst()
 
-        caches.flatMap { it.getSubtypesOf(unprocessedType) }
+        caches.asSequence()
-                .filter { it !in subtypes }
+              .flatMap { it.getSubtypesOf(unprocessedType) }
-                .forEach { types.addLast(it) }
+              .filter { it !in subtypes }
+              .forEach { types.addLast(it) }
 
         subtypes.add(unprocessedType)
     }

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -62,7 +62,6 @@ import org.jetbrains.kotlin.jps.incremental.*
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.modules.TargetId
-import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.CompilationCanceledException
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import org.jetbrains.kotlin.utils.JsLibraryUtils
@@ -773,88 +772,20 @@ private fun CompilationResult.doProcessChangesUsingLookups(
         compiledFiles: Set<File>,
         dataManager: BuildDataManager,
         fsOperations: FSOperationsHelper,
-        caches: Collection<IncrementalCacheImpl<*>>
+        caches: Iterable<IncrementalCacheImpl<ModuleBuildTarget>>
 ) {
-    val dirtyLookupSymbols = HashSet<LookupSymbol>()
-    val dirtyClassesFqNames = HashSet<FqName>()
     val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider)
-    val allCaches: Sequence<IncrementalCacheImpl<*>> = caches.asSequence().flatMap { it.dependentsWithThis }
+    val allCaches = caches.flatMap { it.thisWithDependentCaches }
+    val logAction = { logStr: String -> KotlinBuilder.LOG.debug(logStr) }
 
-    KotlinBuilder.LOG.debug("Start processing changes")
+    logAction("Start processing changes")
-
-    for (change in changes) {
-        KotlinBuilder.LOG.debug("Process $change")
-
-        if (change is ChangeInfo.SignatureChanged) {
-            val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, allCaches)
-
-            for (classFqName in fqNames) {
-                assert(!classFqName.isRoot) { "classFqName is root when processing $change" }
-
-                val scope = classFqName.parent().asString()
-                val name = classFqName.shortName().identifier
-                dirtyLookupSymbols.add(LookupSymbol(name, scope))
-            }
-        }
-        else if (change is ChangeInfo.MembersChanged) {
-            val fqNames = withSubtypes(change.fqName, allCaches)
-            // need to recompile subtypes because changed member might break override
-            dirtyClassesFqNames.addAll(fqNames)
-
-            change.names.forAllPairs(fqNames) { name, fqName ->
-                dirtyLookupSymbols.add(LookupSymbol(name, fqName.asString()))
-            }
-        }
-    }
-
-    val dirtyFiles = HashSet<File>()
-
-    for (lookup in dirtyLookupSymbols) {
-        val affectedFiles = lookupStorage.get(lookup).map(::File)
-
-        KotlinBuilder.LOG.debug { "${lookup.scope}#${lookup.name} caused recompilation of: $affectedFiles" }
-
-        dirtyFiles.addAll(affectedFiles)
-    }
-
-    for (cache in allCaches) {
-        for (dirtyClassFqName in dirtyClassesFqNames) {
-            val srcFile = cache.getSourceFileIfClass(dirtyClassFqName) ?: continue
-            dirtyFiles.add(srcFile)
-        }
-    }
 
+    val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(allCaches, logAction)
+    val dirtyFiles = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction) +
+                     mapClassesFqNamesToFiles(allCaches, dirtyClassFqNames, logAction)
     fsOperations.markFiles(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
-    KotlinBuilder.LOG.debug("End of processing changes")
-}
 
-/**
+    logAction("End of processing changes")
- * Returns type with its subtypes transitively
- *
- * For example:
- *    open class A
- *    open class B : A()
- *    class C : B()
- * withSubtypes(A) will return [A, B, C]
- */
-private fun withSubtypes(
-        typeFqName: FqName,
-        caches: Sequence<IncrementalCacheImpl<*>>
-): Set<FqName> {
-    val types = LinkedList(listOf(typeFqName))
-    val subtypes = hashSetOf<FqName>()
-
-    while (types.isNotEmpty()) {
-        val unprocessedType = types.pollFirst()
-
-        caches.flatMap { it.getSubtypesOf(unprocessedType) }
-              .filter { it !in subtypes }
-              .forEach { types.addLast(it) }
-
-        subtypes.add(unprocessedType)
-    }
-
-    return subtypes
 }
 
 private fun getLookupTracker(project: JpsProject): LookupTracker {
@@ -948,14 +879,6 @@ private fun hasKotlinDirtyOrRemovedFiles(
     return chunk.targets.any { KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, it).isNotEmpty() }
 }
 
-private inline fun <T, R> Iterable<T>.forAllPairs(other: Iterable<R>, fn: (T, R)->Unit) {
-    for (t in this) {
-        for (r in other) {
-            fn(t, r)
-        }
-    }
-}
-
 private inline fun Logger.debug(message: ()->String) {
     if (isDebugEnabled) {
         debug(message())

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCacheImpl.kt

@@ -60,7 +60,7 @@ class JpsIncrementalCacheImpl(
         for ((className, functions) in dirtyInlineFunctionsMap.getEntries()) {
             val classFilePath = getClassFilePath(className.internalName)
 
-            for (cache in dependentsWithThis) {
+            for (cache in thisWithDependentCaches) {
                 val targetFiles = functions.flatMap { (cache as JpsIncrementalCacheImpl).inlinedTo[classFilePath, it] }
                 result.addAll(targetFiles)
             }

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -238,6 +238,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
         var currentRemoved = removed
         val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
         val compiledSourcesSet = hashSetOf<File>()
+        val logAction = { logStr: String -> logger.kotlinInfo(logStr) }
 
         fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl {
             val cacheDir = File(cachesBaseDir, "increCache.${target.name}")
@@ -255,15 +256,6 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
                     innerClasses.flatMap { it.findLookupSymbols() }
         }
 
-        fun Iterable<LookupSymbol>.files(filesFilter: (File) -> Boolean = { true }, logAction: (LookupSymbol, Iterable<File>) -> Unit = { l,fs -> }): Iterable<File> =
-            flatMap { lookup ->
-                val files = lookupStorage.get(lookup).map(::File).filter(filesFilter)
-                if (files.any()) {
-                    logAction(lookup, files)
-                }
-                files
-            }
-
         fun dirtyLookupSymbolsFromRemovedKotlinFiles(): List<LookupSymbol> {
             val removedKotlinFiles = removed.filter { it.isKotlinFile() }
             return if (removedKotlinFiles.isNotEmpty())
@@ -298,11 +290,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
 
             if (lookupSymbols.any()) {
                 val kotlinModifiedFilesSet = modifiedKotlinFiles.toHashSet()
-                val dirtyFilesFromLookups = lookupSymbols.files(
+                val dirtyFilesFromLookups = mapLookupSymbolsToFiles(lookupStorage, lookupSymbols, logAction, excludes = kotlinModifiedFilesSet)
-                                filesFilter = { it !in kotlinModifiedFilesSet },
-                                logAction = { lookup, files ->
-                                    logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}")
-                                })
                 modifiedKotlinFiles.addAll(dirtyFilesFromLookups)
             }
 
@@ -428,9 +416,9 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             val (exitCode, generatedFiles) = compileChanged(targets, existingSource.toSet(), outputDir, args, ::getIncrementalCache, lookupTracker)
 
             allGeneratedFiles.addAll(generatedFiles)
-            val changes = updateIncrementalCaches(targets, generatedFiles,
+            val compilationResult = updateIncrementalCaches(targets, generatedFiles,
-                                                  compiledWithErrors = exitCode != ExitCode.OK,
+                                                            compiledWithErrors = exitCode != ExitCode.OK,
-                                                  getIncrementalCache = { caches[it]!! })
+                                                            getIncrementalCache = { caches[it]!! })
 
             lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved)
             allCachesVersions().forEach { it.saveIfNeeded() }
@@ -439,14 +427,8 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             if (!isIncrementalDecided) break;
 
             compiledSourcesSet.addAll(sourcesToCompile)
-
+            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(caches.values, logAction)
-            val dirtyLookups = changes.dirtyLookups<TargetId>(caches.values.asSequence())
+            sourcesToCompile = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction, excludes = compiledSourcesSet)
-            val newDirtyFiles = dirtyLookups.files(
-                    filesFilter = { it !in compiledSourcesSet },
-                    logAction = { lookup, files ->
-                        logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}")
-                    })
-            sourcesToCompile = newDirtyFiles.filter { it in sources }.toSet()
 
             if (currentRemoved.any()) {
                 currentRemoved = listOf()