From 19089f8b17c64876420e290acd361450e34da54b Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Tue, 1 Dec 2015 20:06:03 +0300 Subject: [PATCH] Prevent processing changes if all kotlin in chunk was rebuilt --- .../kotlin/jps/build/KotlinBuilder.kt | 62 ++++++++++--------- .../jetbrains/kotlin/jps/build/MarkerFile.kt | 60 ++++++++++++++++++ ...tractIncrementalCacheVersionChangedTest.kt | 3 +- .../clearedHasKotlin/build.log | 6 ++ .../data-container-version-build.log | 6 ++ .../module1Modified/build.log | 4 ++ .../data-container-version-build.log | 12 ---- .../moduleWithConstantModified/build.log | 4 ++ .../moduleWithInlineModified/build.log | 4 ++ .../incrementalOffOn/build.log | 9 --- 10 files changed, 118 insertions(+), 52 deletions(-) create mode 100644 jps-plugin/src/org/jetbrains/kotlin/jps/build/MarkerFile.kt diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 76e90390538..fe2d5aea5c3 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -29,7 +29,6 @@ import org.jetbrains.jps.builders.impl.TargetOutputIndexImpl import org.jetbrains.jps.builders.java.JavaBuilderUtil import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor import org.jetbrains.jps.builders.java.dependencyView.Mappings -import org.jetbrains.jps.builders.storage.BuildDataPaths import org.jetbrains.jps.incremental.* import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.* import org.jetbrains.jps.incremental.java.JavaBuilder @@ -73,26 +72,9 @@ import java.util.* public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { companion object { - private val HAS_KOTLIN_FILE_NAME = "has-kotlin.txt" - public val KOTLIN_BUILDER_NAME: String = "Kotlin Builder" public val LOOKUP_TRACKER: JpsElementChildRoleBase> = JpsElementChildRoleBase.create("lookup tracker") val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder") - - private fun hasKotlin(target: ModuleBuildTarget, paths: BuildDataPaths): Boolean { - val hasKotlinFile = File(paths.getTargetDataRoot(target), HAS_KOTLIN_FILE_NAME) - return hasKotlinFile.exists() - } - - private fun setHasKotlin(target: ModuleBuildTarget, paths: BuildDataPaths) { - val hasKotlinFile = File(paths.getTargetDataRoot(target), HAS_KOTLIN_FILE_NAME) - hasKotlinFile.createNewFile() - } - - fun clearHasKotlin(target: ModuleBuildTarget, paths: BuildDataPaths) { - val hasKotlinFile = File(paths.getTargetDataRoot(target), HAS_KOTLIN_FILE_NAME) - hasKotlinFile.delete() - } } private val statisticsLogger = TeamcityStatisticsLogger() @@ -164,22 +146,25 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR val projectDescriptor = context.projectDescriptor val dataManager = projectDescriptor.dataManager val targets = chunk.targets - val isFullRebuild = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context) + val hasKotlin = HasKotlinMarker(dataManager) + val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager) + val isChunkRebuilding = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context) + || targets.any { rebuildAfterCacheVersionChanged[it] == true } - if (!isFullRebuild && - targets.any { hasKotlin(it, dataManager.dataPaths) } && + if (!isChunkRebuilding && + targets.any { hasKotlin[it] == true } && shouldRebuildBecauseVersionChanged(context, dataManager, targets, fsOperations) ) { - fsOperations.markChunk(recursively = true) - val targetsWithDependents = getIncrementalCaches(chunk, context).keys - targetsWithDependents.forEach { clearHasKotlin(it, dataManager.dataPaths) } + targets.forEach { rebuildAfterCacheVersionChanged[it] = true } return CHUNK_REBUILD_REQUIRED } - if (hasKotlinDirtyOrRemovedFiles(dirtyFilesHolder, chunk)) { - targets.forEach { setHasKotlin(it, dataManager.dataPaths) } - } - else { + if (!hasKotlinDirtyOrRemovedFiles(dirtyFilesHolder, chunk)) { + if (isChunkRebuilding) { + targets.forEach { hasKotlin[it] = false } + } + + targets.forEach { rebuildAfterCacheVersionChanged.clean(it) } return NOTHING_DONE } @@ -226,6 +211,15 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR registerOutputItems(outputConsumer, generatedFiles) saveVersions(context, chunk) + if (targets.any { hasKotlin[it] == null }) { + fsOperations.markChunk(excludeFiles = filesToCompile.values().toSet()) + } + + for (target in targets) { + hasKotlin[target] = true + rebuildAfterCacheVersionChanged.clean(target) + } + if (JpsUtils.isJsKotlinModule(chunk.representativeTarget())) { copyJsLibraryFilesIfNeeded(chunk, project) return OK @@ -243,7 +237,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR val changesInfo = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedFiles) updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile) - if (isFullRebuild) { + if (isChunkRebuilding) { return OK } @@ -332,6 +326,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR val actions = allVersions.map { it.checkVersion() }.toSet().sorted() val buildTargetIndex = context.projectDescriptor.buildTargetIndex val allTargets = buildTargetIndex.allTargets.filterIsInstance().toSet() + val hasKotlin = HasKotlinMarker(dataManager) + val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager) for (status in actions) { when (status) { @@ -347,6 +343,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR for (target in allTargets) { dataManager.getKotlinCache(target).clean() + rebuildAfterCacheVersionChanged[target] = true } dataManager.getStorage(KotlinDataContainerTarget, LookupStorageProvider).clean() @@ -355,7 +352,12 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR } CacheVersion.Action.REBUILD_CHUNK -> { LOG.info("Clearing caches for " + targets.joinToString { it.presentableName }) - targets.forEach { dataManager.getKotlinCache(it).clean() } + + for (target in targets) { + dataManager.getKotlinCache(target).clean() + hasKotlin.clean(target) + } + return true } CacheVersion.Action.CLEAN_NORMAL_CACHES -> { diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/MarkerFile.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/MarkerFile.kt new file mode 100644 index 00000000000..a3b5cca9291 --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/MarkerFile.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.builders.storage.BuildDataPaths +import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.jps.incremental.storage.BuildDataManager +import org.jetbrains.kotlin.jps.incremental.CACHE_DIRECTORY_NAME +import java.io.File + +private val HAS_KOTLIN_MARKER_FILE_NAME = "has-kotlin-marker.txt" +private val REBUILD_AFTER_CACHE_VERSION_CHANGE_MARKER = "rebuild-after-cache-version-change-marker.txt" + +abstract class MarkerFile(private val fileName: String, private val paths: BuildDataPaths) { + operator fun get(target: ModuleBuildTarget): Boolean? { + val file = target.markerFile + + if (!file.exists()) return null + + return file.readText().toBoolean() + } + + operator fun set(target: ModuleBuildTarget, value: Boolean) { + val file = target.markerFile + + if (!file.exists()) { + file.parentFile.mkdirs() + file.createNewFile() + } + + file.writeText(value.toString()) + } + + fun clean(target: ModuleBuildTarget) { + target.markerFile.delete() + } + + private val ModuleBuildTarget.markerFile: File + get() { + val directory = File(paths.getTargetDataRoot(this), CACHE_DIRECTORY_NAME) + return File(directory, fileName) + } +} + +class HasKotlinMarker(dataManager: BuildDataManager) : MarkerFile(HAS_KOTLIN_MARKER_FILE_NAME, dataManager.dataPaths) +class RebuildAfterCacheVersionChangeMarker(dataManager: BuildDataManager) : MarkerFile(REBUILD_AFTER_CACHE_VERSION_CHANGE_MARKER, dataManager.dataPaths) diff --git a/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt b/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt index 498829ae793..536f6d91219 100644 --- a/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt +++ b/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt @@ -24,9 +24,10 @@ abstract class AbstractIncrementalCacheVersionChangedTest : AbstractIncrementalJ val modifiedFiles = modifications.filterIsInstance().map { it.path } val paths = projectDescriptor.dataManager.dataPaths val targets = projectDescriptor.allModuleTargets + val hasKotlin = HasKotlinMarker(projectDescriptor.dataManager) if (modifiedFiles.any { it.endsWith("clear-has-kotlin") }) { - targets.forEach { KotlinBuilder.clearHasKotlin(it, paths) } + targets.forEach { hasKotlin.clean(it) } } if (modifiedFiles.none { it.endsWith("do-not-change-cache-versions") }) { diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/build.log index 83489248281..5fdce9ad8e7 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/build.log @@ -11,4 +11,10 @@ out/production/module1/A.class End of files Compiling files: module1/src/module1_A.kt +End of files +Cleaning output files: +out/production/module1/B.class +End of files +Compiling files: +module1/src/module1_B.kt End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/data-container-version-build.log index 83489248281..5fdce9ad8e7 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/clearedHasKotlin/data-container-version-build.log @@ -11,4 +11,10 @@ out/production/module1/A.class End of files Compiling files: module1/src/module1_A.kt +End of files +Cleaning output files: +out/production/module1/B.class +End of files +Compiling files: +module1/src/module1_B.kt End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log index 3c6cd9b049b..7300f1cf983 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log @@ -23,12 +23,16 @@ Compiling files: module1/src/module1_A.java End of files Cleaning output files: +out/production/module2/B.class out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/Module2_bKt.class End of files Compiling files: module2/src/module2_b.kt End of files +Compiling files: +module2/src/module2_B.java +End of files Cleaning output files: out/production/module3/C.class out/production/module3/META-INF/module3.kotlin_module diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log index b16aeb5f638..2ab0048b937 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log @@ -13,15 +13,9 @@ out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class out/production/module1/module1/Module1_aKt.class End of files -Cleaning output files: -out/production/module1/A.class -End of files Compiling files: module1/src/module1_a.kt End of files -Compiling files: -module1/src/module1_A.java -End of files Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/Module2_bKt.class @@ -33,12 +27,6 @@ Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/module3/Module3_cKt.class End of files -Cleaning output files: -out/production/module3/C.class -End of files Compiling files: module3/src/module3_c.kt -End of files -Compiling files: -module3/src/module3_C.java End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log index d9e25129990..07ee39331d0 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log @@ -7,7 +7,11 @@ module1/src/module1_A.kt End of files Cleaning output files: out/production/module2/b/B.class +out/production/module2/b/C.class End of files Compiling files: module2/src/module2_B.kt +End of files +Compiling files: +module2/src/module2_C.java End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log index 2883783148a..c9cf56e79b6 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log @@ -8,7 +8,11 @@ module1/src/module1_A.kt End of files Cleaning output files: out/production/module2/b/B.class +out/production/module2/b/C.class End of files Compiling files: module2/src/module2_B.kt +End of files +Compiling files: +module2/src/module2_C.java End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log index bf4fea0aa71..dc9b74c5701 100644 --- a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log +++ b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log @@ -18,17 +18,8 @@ End of files Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/foo/Module2_aKt.class -out/production/module2/foo/Module2_bKt.class -out/production/module2/foo/Module2_cKt.class -End of files -Compiling files: -module2/src/module2_a.kt -module2/src/module2_b.kt -module2/src/module2_c.kt End of files Cleaning output files: -out/production/module2/META-INF/module2.kotlin_module -out/production/module2/foo/Module2_aKt.class out/production/module2/foo/Module2_bKt.class out/production/module2/foo/Module2_cKt.class End of files