From f70d01f657d8c449aee6cb2b4f45d19967787a4f Mon Sep 17 00:00:00 2001 From: Sergey Rostov Date: Tue, 20 Nov 2018 13:41:18 +0300 Subject: [PATCH] Fix updating complementary files map in case of compilation errors #KT-27868 Fixed Previously given dirtyFiles was removed from complementaryFilesMap exactly on call of clearComplementaryFilesMapping. This causes fail of next build in case of compilation error of previous build on JPS (since complementary files not known on second build). The right way to do it is removing (replacing) them only after successful build. This was working on Gradle since Gradle rebuilds whole module (project) in case of build error. --- .../incremental/AbstractIncrementalCache.kt | 36 ++++++++++++------- .../incremental/IncrementalCompilerRunner.kt | 4 +-- .../kotlin/jps/build/KotlinBuilder.kt | 10 ++++-- .../jps/targets/KotlinJsModuleBuildTarget.kt | 3 +- .../jps/targets/KotlinJvmModuleBuildTarget.kt | 3 +- .../jps/targets/KotlinModuleBuildTarget.kt | 10 ++++-- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt index a4da1c255e2..14969438dc0 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt @@ -37,8 +37,8 @@ interface IncrementalCacheCommon { fun getSourceFileIfClass(fqName: FqName): File? fun markDirty(removedAndCompiledSources: Collection) fun clearCacheForRemovedClasses(changesCollector: ChangesCollector) - fun clearComplementaryFilesMapping(dirtyFiles: Collection): Collection - fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl) + fun getComplementaryFilesRecursive(dirtyFiles: Collection): Collection + fun updateComplementaryFiles(dirtyFiles: Collection, expectActualTracker: ExpectActualTrackerImpl) fun dump(): String } @@ -51,14 +51,17 @@ abstract class AbstractIncrementalCache(workingDir: File) : BasicMaps private val SUPERTYPES = "supertypes" private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source" private val COMPLEMENTARY_FILES = "complementary-files" - @JvmStatic protected val SOURCE_TO_CLASSES = "source-to-classes" - @JvmStatic protected val DIRTY_OUTPUT_CLASSES = "dirty-output-classes" + @JvmStatic + protected val SOURCE_TO_CLASSES = "source-to-classes" + @JvmStatic + protected val DIRTY_OUTPUT_CLASSES = "dirty-output-classes" } private val dependents = arrayListOf>() fun addDependentCache(cache: AbstractIncrementalCache) { dependents.add(cache) } + override val thisWithDependentCaches: Iterable> by lazy { val result = arrayListOf(this) result.addAll(dependents) @@ -83,10 +86,10 @@ abstract class AbstractIncrementalCache(workingDir: File) : BasicMaps files.flatMapTo(HashSet()) { sourceToClassesMap.getFqNames(it) } override fun getSubtypesOf(className: FqName): Sequence = - subtypesMap[className].asSequence() + subtypesMap[className].asSequence() override fun getSourceFileIfClass(fqName: FqName): File? = - classFqNameToSourceMap[fqName] + classFqNameToSourceMap[fqName] override fun markDirty(removedAndCompiledSources: Collection) { for (sourceFile in removedAndCompiledSources) { @@ -102,8 +105,8 @@ abstract class AbstractIncrementalCache(workingDir: File) : BasicMaps protected fun addToClassStorage(proto: ProtoBuf.Class, nameResolver: NameResolver, srcFile: File) { val supertypes = proto.supertypes(TypeTable(proto.typeTable)) val parents = supertypes.map { nameResolver.getClassId(it.className).asSingleFqName() } - .filter { it.asString() != "kotlin.Any" } - .toSet() + .filter { it.asString() != "kotlin.Any" } + .toSet() val child = nameResolver.getClassId(proto.fqName).asSingleFqName() parents.forEach { subtypesMap.add(it, child) } @@ -150,13 +153,14 @@ abstract class AbstractIncrementalCache(workingDir: File) : BasicMaps removedFqNames.forEach { classFqNameToSourceMap.remove(it) } } - protected class ClassFqNameToSourceMap(storageFile: File) : BasicStringMap(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) { + protected class ClassFqNameToSourceMap(storageFile: File) : + BasicStringMap(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) { operator fun set(fqName: FqName, sourceFile: File) { storage[fqName.asString()] = sourceFile.canonicalPath } operator fun get(fqName: FqName): File? = - storage[fqName.asString()]?.let(::File) + storage[fqName.asString()]?.let(::File) fun remove(fqName: FqName) { storage.remove(fqName.asString()) @@ -165,18 +169,24 @@ abstract class AbstractIncrementalCache(workingDir: File) : BasicMaps override fun dumpValue(value: String) = value } - override fun clearComplementaryFilesMapping(dirtyFiles: Collection): Collection { + override fun getComplementaryFilesRecursive(dirtyFiles: Collection): Collection { val complementaryFiles = HashSet() val filesQueue = ArrayDeque(dirtyFiles) while (filesQueue.isNotEmpty()) { val file = filesQueue.pollFirst() - complementaryFilesMap.remove(file).filterTo(filesQueue) { complementaryFiles.add(it) } + complementaryFilesMap[file].forEach { + if (complementaryFiles.add(it)) filesQueue.add(it) + } } complementaryFiles.removeAll(dirtyFiles) return complementaryFiles } - override fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl) { + override fun updateComplementaryFiles(dirtyFiles: Collection, expectActualTracker: ExpectActualTrackerImpl) { + dirtyFiles.forEach { + complementaryFilesMap.remove(it) + } + val actualToExpect = hashMapOf>() for ((expect, actuals) in expectActualTracker.expectToActualMap) { for (actual in actuals) { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 7d9ce1b9524..7ec129e9d0e 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -198,7 +198,7 @@ abstract class IncrementalCompilerRunner< var exitCode = ExitCode.OK while (dirtySources.any() || runWithNoDirtyKotlinSources(caches)) { - val complementaryFiles = caches.platformCache.clearComplementaryFilesMapping(dirtySources) + val complementaryFiles = caches.platformCache.getComplementaryFilesRecursive(dirtySources) dirtySources.addAll(complementaryFiles) caches.platformCache.markDirty(dirtySources) caches.inputsCache.removeOutputForSourceFiles(dirtySources) @@ -237,7 +237,7 @@ abstract class IncrementalCompilerRunner< } } - caches.platformCache.registerComplementaryFiles(expectActualTracker) + caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker) caches.inputsCache.registerOutputForSourceFiles(generatedFiles) caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) val changesCollector = ChangesCollector() diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 6270b13249c..aa6bc4be271 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -468,7 +468,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { for ((target, files) in generatedFiles) { val kotlinModuleBuilderTarget = kotlinContext.targetsBinding[target]!! - kotlinModuleBuilderTarget.updateCaches(incrementalCaches[kotlinModuleBuilderTarget]!!, files, changesCollector, environment) + kotlinModuleBuilderTarget.updateCaches( + kotlinDirtyFilesHolder, + incrementalCaches[kotlinModuleBuilderTarget]!!, + files, + changesCollector, + environment + ) } updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder) @@ -512,7 +518,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val targetDirtyFiles = dirtyFilesHolder.byTarget[jpsTarget] if (cache != null && targetDirtyFiles != null) { - val complementaryFiles = cache.clearComplementaryFilesMapping( + val complementaryFiles = cache.getComplementaryFilesRecursive( targetDirtyFiles.dirty.keys + targetDirtyFiles.removed ) diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt index f75403afbe3..9b805a8e6a6 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt @@ -211,12 +211,13 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu JpsIncrementalJsCache(jpsModuleBuildTarget, paths) override fun updateCaches( + dirtyFilesHolder: KotlinDirtySourceFilesHolder, jpsIncrementalCache: JpsIncrementalCache, files: List, changesCollector: ChangesCollector, environment: JpsCompilerEnvironment ) { - super.updateCaches(jpsIncrementalCache, files, changesCollector, environment) + super.updateCaches(dirtyFilesHolder, jpsIncrementalCache, files, changesCollector, environment) val incrementalResults = environment.services[IncrementalResultsConsumer::class.java] as IncrementalResultsConsumerImpl diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt index 8c719a3901d..e5c5653819b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt @@ -254,12 +254,13 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB } override fun updateCaches( + dirtyFilesHolder: KotlinDirtySourceFilesHolder, jpsIncrementalCache: JpsIncrementalCache, files: List, changesCollector: ChangesCollector, environment: JpsCompilerEnvironment ) { - super.updateCaches(jpsIncrementalCache, files, changesCollector, environment) + super.updateCaches(dirtyFilesHolder, jpsIncrementalCache, files, changesCollector, environment) updateIncrementalCache(files, jpsIncrementalCache as IncrementalJvmCache, changesCollector, null) } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt index 3e66c6a8662..1f463220131 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt @@ -27,11 +27,11 @@ import org.jetbrains.kotlin.incremental.ChangesCollector import org.jetbrains.kotlin.incremental.ExpectActualTrackerImpl import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.jps.build.* import org.jetbrains.kotlin.jps.incremental.CacheAttributesDiff +import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache import org.jetbrains.kotlin.jps.incremental.loadDiff import org.jetbrains.kotlin.jps.incremental.localCacheVersionManager -import org.jetbrains.kotlin.jps.build.* -import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache import org.jetbrains.kotlin.jps.model.productionOutputFilePath import org.jetbrains.kotlin.jps.model.testOutputFilePath import org.jetbrains.kotlin.modules.TargetId @@ -246,13 +246,17 @@ abstract class KotlinModuleBuildTarget intern } open fun updateCaches( + dirtyFilesHolder: KotlinDirtySourceFilesHolder, jpsIncrementalCache: JpsIncrementalCache, files: List, changesCollector: ChangesCollector, environment: JpsCompilerEnvironment ) { + val changedAndRemovedFiles = dirtyFilesHolder.getDirtyFiles(jpsModuleBuildTarget).keys + + dirtyFilesHolder.getRemovedFiles(jpsModuleBuildTarget) val expectActualTracker = environment.services[ExpectActualTracker::class.java] as ExpectActualTrackerImpl - jpsIncrementalCache.registerComplementaryFiles(expectActualTracker) + + jpsIncrementalCache.updateComplementaryFiles(changedAndRemovedFiles,expectActualTracker) } open fun makeServices(