Fix updating complementary files map in case of compilation errors

#KT-27868 Fixed

Previously given dirtyFiles was removed from complementaryFilesMap
exactly on call of clearComplementaryFilesMapping. This causes fail
of next build in case of compilation error of previous build on JPS
(since complementary files not known on second build). The right way
to do it is removing (replacing) them only after successful
build.

This was working on Gradle since Gradle rebuilds whole module (project)
in case of build error.

build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt

@@ -37,8 +37,8 @@ interface IncrementalCacheCommon {
     fun getSourceFileIfClass(fqName: FqName): File?
     fun markDirty(removedAndCompiledSources: Collection<File>)
     fun clearCacheForRemovedClasses(changesCollector: ChangesCollector)
-    fun clearComplementaryFilesMapping(dirtyFiles: Collection<File>): Collection<File>
+    fun getComplementaryFilesRecursive(dirtyFiles: Collection<File>): Collection<File>
-    fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl)
+    fun updateComplementaryFiles(dirtyFiles: Collection<File>, expectActualTracker: ExpectActualTrackerImpl)
     fun dump(): String
 }
 
@@ -51,14 +51,17 @@ abstract class AbstractIncrementalCache<ClassName>(workingDir: File) : BasicMaps
         private val SUPERTYPES = "supertypes"
         private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
         private val COMPLEMENTARY_FILES = "complementary-files"
-        @JvmStatic protected val SOURCE_TO_CLASSES = "source-to-classes"
+        @JvmStatic
-        @JvmStatic protected val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
+        protected val SOURCE_TO_CLASSES = "source-to-classes"
+        @JvmStatic
+        protected val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
     }
 
     private val dependents = arrayListOf<AbstractIncrementalCache<ClassName>>()
     fun addDependentCache(cache: AbstractIncrementalCache<ClassName>) {
         dependents.add(cache)
     }
+
     override val thisWithDependentCaches: Iterable<AbstractIncrementalCache<ClassName>> by lazy {
         val result = arrayListOf(this)
         result.addAll(dependents)
@@ -83,10 +86,10 @@ abstract class AbstractIncrementalCache<ClassName>(workingDir: File) : BasicMaps
         files.flatMapTo(HashSet()) { sourceToClassesMap.getFqNames(it) }
 
     override fun getSubtypesOf(className: FqName): Sequence<FqName> =
-            subtypesMap[className].asSequence()
+        subtypesMap[className].asSequence()
 
     override fun getSourceFileIfClass(fqName: FqName): File? =
-            classFqNameToSourceMap[fqName]
+        classFqNameToSourceMap[fqName]
 
     override fun markDirty(removedAndCompiledSources: Collection<File>) {
         for (sourceFile in removedAndCompiledSources) {
@@ -102,8 +105,8 @@ abstract class AbstractIncrementalCache<ClassName>(workingDir: File) : BasicMaps
     protected fun addToClassStorage(proto: ProtoBuf.Class, nameResolver: NameResolver, srcFile: File) {
         val supertypes = proto.supertypes(TypeTable(proto.typeTable))
         val parents = supertypes.map { nameResolver.getClassId(it.className).asSingleFqName() }
-                .filter { it.asString() != "kotlin.Any" }
+            .filter { it.asString() != "kotlin.Any" }
-                .toSet()
+            .toSet()
         val child = nameResolver.getClassId(proto.fqName).asSingleFqName()
 
         parents.forEach { subtypesMap.add(it, child) }
@@ -150,13 +153,14 @@ abstract class AbstractIncrementalCache<ClassName>(workingDir: File) : BasicMaps
         removedFqNames.forEach { classFqNameToSourceMap.remove(it) }
     }
 
-    protected class ClassFqNameToSourceMap(storageFile: File) : BasicStringMap<String>(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) {
+    protected class ClassFqNameToSourceMap(storageFile: File) :
+        BasicStringMap<String>(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) {
         operator fun set(fqName: FqName, sourceFile: File) {
             storage[fqName.asString()] = sourceFile.canonicalPath
         }
 
         operator fun get(fqName: FqName): File? =
-                storage[fqName.asString()]?.let(::File)
+            storage[fqName.asString()]?.let(::File)
 
         fun remove(fqName: FqName) {
             storage.remove(fqName.asString())
@@ -165,18 +169,24 @@ abstract class AbstractIncrementalCache<ClassName>(workingDir: File) : BasicMaps
         override fun dumpValue(value: String) = value
     }
 
-    override fun clearComplementaryFilesMapping(dirtyFiles: Collection<File>): Collection<File> {
+    override fun getComplementaryFilesRecursive(dirtyFiles: Collection<File>): Collection<File> {
         val complementaryFiles = HashSet<File>()
         val filesQueue = ArrayDeque(dirtyFiles)
         while (filesQueue.isNotEmpty()) {
             val file = filesQueue.pollFirst()
-            complementaryFilesMap.remove(file).filterTo(filesQueue) { complementaryFiles.add(it) }
+            complementaryFilesMap[file].forEach {
+                if (complementaryFiles.add(it)) filesQueue.add(it)
+            }
         }
         complementaryFiles.removeAll(dirtyFiles)
         return complementaryFiles
     }
 
-    override fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl) {
+    override fun updateComplementaryFiles(dirtyFiles: Collection<File>, expectActualTracker: ExpectActualTrackerImpl) {
+        dirtyFiles.forEach {
+            complementaryFilesMap.remove(it)
+        }
+
         val actualToExpect = hashMapOf<File, MutableSet<File>>()
         for ((expect, actuals) in expectActualTracker.expectToActualMap) {
             for (actual in actuals) {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -198,7 +198,7 @@ abstract class IncrementalCompilerRunner<
         var exitCode = ExitCode.OK
 
         while (dirtySources.any() || runWithNoDirtyKotlinSources(caches)) {
-            val complementaryFiles = caches.platformCache.clearComplementaryFilesMapping(dirtySources)
+            val complementaryFiles = caches.platformCache.getComplementaryFilesRecursive(dirtySources)
             dirtySources.addAll(complementaryFiles)
             caches.platformCache.markDirty(dirtySources)
             caches.inputsCache.removeOutputForSourceFiles(dirtySources)
@@ -237,7 +237,7 @@ abstract class IncrementalCompilerRunner<
                 }
             }
 
-            caches.platformCache.registerComplementaryFiles(expectActualTracker)
+            caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
             caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
             caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
             val changesCollector = ChangesCollector()

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -468,7 +468,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
 
             for ((target, files) in generatedFiles) {
                 val kotlinModuleBuilderTarget = kotlinContext.targetsBinding[target]!!
-                kotlinModuleBuilderTarget.updateCaches(incrementalCaches[kotlinModuleBuilderTarget]!!, files, changesCollector, environment)
+                kotlinModuleBuilderTarget.updateCaches(
+                    kotlinDirtyFilesHolder,
+                    incrementalCaches[kotlinModuleBuilderTarget]!!,
+                    files,
+                    changesCollector,
+                    environment
+                )
             }
 
             updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder)
@@ -512,7 +518,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
                 val targetDirtyFiles = dirtyFilesHolder.byTarget[jpsTarget]
 
                 if (cache != null && targetDirtyFiles != null) {
-                    val complementaryFiles = cache.clearComplementaryFilesMapping(
+                    val complementaryFiles = cache.getComplementaryFilesRecursive(
                         targetDirtyFiles.dirty.keys + targetDirtyFiles.removed
                     )
 

jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt

@@ -211,12 +211,13 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
         JpsIncrementalJsCache(jpsModuleBuildTarget, paths)
 
     override fun updateCaches(
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
         jpsIncrementalCache: JpsIncrementalCache,
         files: List<GeneratedFile>,
         changesCollector: ChangesCollector,
         environment: JpsCompilerEnvironment
     ) {
-        super.updateCaches(jpsIncrementalCache, files, changesCollector, environment)
+        super.updateCaches(dirtyFilesHolder, jpsIncrementalCache, files, changesCollector, environment)
 
         val incrementalResults = environment.services[IncrementalResultsConsumer::class.java] as IncrementalResultsConsumerImpl
 

jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt

@@ -254,12 +254,13 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB
     }
 
     override fun updateCaches(
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
         jpsIncrementalCache: JpsIncrementalCache,
         files: List<GeneratedFile>,
         changesCollector: ChangesCollector,
         environment: JpsCompilerEnvironment
     ) {
-        super.updateCaches(jpsIncrementalCache, files, changesCollector, environment)
+        super.updateCaches(dirtyFilesHolder, jpsIncrementalCache, files, changesCollector, environment)
 
         updateIncrementalCache(files, jpsIncrementalCache as IncrementalJvmCache, changesCollector, null)
     }

jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt

@@ -27,11 +27,11 @@ import org.jetbrains.kotlin.incremental.ChangesCollector
 import org.jetbrains.kotlin.incremental.ExpectActualTrackerImpl
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.jps.build.*
 import org.jetbrains.kotlin.jps.incremental.CacheAttributesDiff
+import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
 import org.jetbrains.kotlin.jps.incremental.loadDiff
 import org.jetbrains.kotlin.jps.incremental.localCacheVersionManager
-import org.jetbrains.kotlin.jps.build.*
-import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
 import org.jetbrains.kotlin.jps.model.productionOutputFilePath
 import org.jetbrains.kotlin.jps.model.testOutputFilePath
 import org.jetbrains.kotlin.modules.TargetId
@@ -246,13 +246,17 @@ abstract class KotlinModuleBuildTarget<BuildMetaInfoType : BuildMetaInfo> intern
     }
 
     open fun updateCaches(
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
         jpsIncrementalCache: JpsIncrementalCache,
         files: List<GeneratedFile>,
         changesCollector: ChangesCollector,
         environment: JpsCompilerEnvironment
     ) {
+        val changedAndRemovedFiles = dirtyFilesHolder.getDirtyFiles(jpsModuleBuildTarget).keys +
+                dirtyFilesHolder.getRemovedFiles(jpsModuleBuildTarget)
         val expectActualTracker = environment.services[ExpectActualTracker::class.java] as ExpectActualTrackerImpl
-        jpsIncrementalCache.registerComplementaryFiles(expectActualTracker)
+
+        jpsIncrementalCache.updateComplementaryFiles(changedAndRemovedFiles,expectActualTracker)
     }
 
     open fun makeServices(