From 143ab5dad295ac858999b82784b5db5559393f76 Mon Sep 17 00:00:00 2001 From: Evgeny Gerashchenko Date: Tue, 3 Feb 2015 17:18:17 +0300 Subject: [PATCH] Simplified process of filtering out obsolete package parts and loading descriptors from incremental caches. Now when file is deleted or dirty (compiled right now), its old package part is used. Current package of file doesn't matter. Original commit: f071802e19da0676229f094da24cf14393b15176 --- .../kotlin/jps/build/KotlinBuilder.kt | 11 ++++ .../jps/incremental/IncrementalCacheImpl.kt | 65 ++++++++++++------- .../build/IncrementalJpsTestGenerated.java | 12 ++++ .../build.log | 30 +++++++++ .../fun.kt | 2 + .../other.kt | 3 + .../other.kt.delete.1 | 0 .../usage.kt | 3 + .../usage.kt.new.1 | 3 + .../usage.kt.new.2 | 3 + .../build.log | 30 +++++++++ .../fun.kt | 2 + .../other.kt | 3 + .../other.kt.new.1 | 5 ++ .../usage.kt | 3 + .../usage.kt.new.1 | 3 + .../usage.kt.new.2 | 3 + 17 files changed, 156 insertions(+), 25 deletions(-) create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/build.log create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/fun.kt create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt.delete.1 create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.1 create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.2 create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/build.log create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/fun.kt create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt.new.1 create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.1 create mode 100644 jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.2 diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 3c3aa7470ab..cf73a90692f 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -105,6 +105,13 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR messageCollector.report(INFO, "Kotlin JPS plugin version " + KotlinVersion.VERSION, NO_LOCATION) val incrementalCaches = chunk.getTargets().keysToMap { dataManager.getKotlinCache(it) } + + for (target in chunk.getTargets()) { + val removedFiles = dirtyFilesHolder.getRemovedFiles(target) + val cache = incrementalCaches[target]!! + removedFiles.forEach { cache.fileIsDeleted(File(it)) } + } + val environment = createCompileEnvironment(incrementalCaches) if (!environment.success()) { environment.reportErrorsTo(messageCollector) @@ -274,6 +281,10 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR return DO_NOTHING } + if (!compilationErrors) { + incrementalCaches.values().forEach { it.clearRemovedPackageParts () } + } + for ((target, cache) in incrementalCaches) { cache.clearCacheForRemovedFiles( KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target), diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index 8056f2536a2..831473cf1b4 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -45,6 +45,8 @@ import org.jetbrains.kotlin.load.kotlin.header.isCompatibleClassKind import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.builders.BuildTarget import org.jetbrains.jps.builders.storage.BuildDataPaths +import com.intellij.util.io.BooleanDataDescriptor +import java.util.ArrayList val INLINE_ANNOTATION_DESC = "Lkotlin/inline;" @@ -54,7 +56,7 @@ private val CACHE_DIRECTORY_NAME = "kotlin" class CacheFormatVersion(targetDataRoot: File) { class object { // Change this when incremental cache format changes - private val INCREMENTAL_CACHE_OWN_VERSION = 1 + private val INCREMENTAL_CACHE_OWN_VERSION = 2 private val CACHE_FORMAT_VERSION: Int = INCREMENTAL_CACHE_OWN_VERSION * 1000000 + JvmAbi.VERSION val FORMAT_VERSION_FILE_PATH: String = "$CACHE_DIRECTORY_NAME/format-version.txt" } @@ -84,6 +86,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen val CONSTANTS_MAP = "constants.tab" val INLINE_FUNCTIONS = "inline-functions.tab" val PACKAGE_PARTS = "package-parts.tab" + val REMOVED_PACKAGE_PARTS = "removed-package-parts.tab" } private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME) @@ -91,8 +94,9 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen private val constantsMap = ConstantsMap() private val inlineFunctionsMap = InlineFunctionsMap() private val packagePartMap = PackagePartMap() + private val removedPackagePartMap = RemovedPackagePartMap() - private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap) + private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap, removedPackagePartMap) private val cacheFormatVersion = CacheFormatVersion(targetDataRoot) @@ -150,9 +154,21 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return DO_NOTHING } - public fun clearCacheForRemovedFiles(removedSourceFiles: Collection, outDirectory: File, compilationSuccessful: Boolean) { - removedSourceFiles.forEach { packagePartMap.remove(it) } + public fun fileIsDeleted(sourceFile: File) { + val wasPackagePart = packagePartMap[sourceFile] + packagePartMap.remove(sourceFile) + if (wasPackagePart != null) { + removedPackagePartMap.add(wasPackagePart) + } + } + public fun clearRemovedPackageParts() { + removedPackagePartMap.clear() + } + + // TODO transitional function + deprecated("transitional function") + public fun clearCacheForRemovedFiles(removedSourceFiles: Collection, outDirectory: File, compilationSuccessful: Boolean) { if (compilationSuccessful) { inlineFunctionsMap.clearOutdated(outDirectory) constantsMap.clearOutdated(outDirectory) @@ -160,8 +176,8 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen } } - public override fun getRemovedPackageParts(sourceFilesToCompileAndFqNames: Map): Collection { - return packagePartMap.getRemovedPackageParts(sourceFilesToCompileAndFqNames) + public override fun getObsoletePackageParts(sourceFilesToCompile: Collection): Collection { + return (removedPackagePartMap.getRemovedParts() + sourceFilesToCompile.map { packagePartMap[it] }.filterNotNull()).toSet() } public override fun getPackageData(fqName: String): ByteArray? { @@ -465,29 +481,28 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen storage.remove(sourceFile.getAbsolutePath()) } - public fun getRemovedPackageParts(compiledSourceFilesToFqName: Map): Collection { - val result = HashSet() + public fun get(sourceFile: File): String? { + return storage[sourceFile.getAbsolutePath()] + } + } - storage.processKeysWithExistingMapping { key -> - val sourceFile = File(key!!) + private inner class RemovedPackagePartMap : BasicMap() { + override fun createMap(): PersistentHashMap = PersistentHashMap( + File(baseDir, REMOVED_PACKAGE_PARTS), + EnumeratorStringDescriptor(), + BooleanDataDescriptor.INSTANCE + ) - val packagePartClassName = storage[key]!! - if (!sourceFile.exists()) { - result.add(packagePartClassName) - } - else { - if (sourceFile in compiledSourceFilesToFqName) { - val previousPackageFqName = JvmClassName.byInternalName(packagePartClassName).getPackageFqName() - if (compiledSourceFilesToFqName[sourceFile] != previousPackageFqName.asString()) { - result.add(packagePartClassName) - } - } - } + public fun add(name: String) { + storage.put(name, true) + } - true - } + public fun getRemovedParts(): Collection { + return storage.getAllKeysWithExistingMapping() + } - return result + public fun clear() { + storage.getAllKeysWithExistingMapping().forEach { storage.remove(it) } } } diff --git a/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalJpsTestGenerated.java b/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalJpsTestGenerated.java index 9db726d6502..1dfdbd8ab87 100644 --- a/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalJpsTestGenerated.java +++ b/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalJpsTestGenerated.java @@ -183,6 +183,18 @@ public class IncrementalJpsTestGenerated extends AbstractIncrementalJpsTest { doTest(fileName); } + @TestMetadata("compilationErrorThenFixedWithPhantomPart") + public void testCompilationErrorThenFixedWithPhantomPart() throws Exception { + String fileName = JetTestUtils.navigationMetadata("jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/"); + doTest(fileName); + } + + @TestMetadata("compilationErrorThenFixedWithPhantomPart2") + public void testCompilationErrorThenFixedWithPhantomPart2() throws Exception { + String fileName = JetTestUtils.navigationMetadata("jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/"); + doTest(fileName); + } + @TestMetadata("conflictingPlatformDeclarations") public void testConflictingPlatformDeclarations() throws Exception { String fileName = JetTestUtils.navigationMetadata("jps-plugin/testData/incremental/pureKotlin/conflictingPlatformDeclarations/"); diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/build.log b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/build.log new file mode 100644 index 00000000000..ba859e438ea --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/build.log @@ -0,0 +1,30 @@ +Cleaning output files: +out/production/module/_DefaultPackage$other$*.class +out/production/module/_DefaultPackage.class +End of files +Cleaning output files: +out/production/module/_DefaultPackage$usage$*.class +End of files +Compiling files: +src/usage.kt +End of files +COMPILATION FAILED +Expecting an expression + + +Cleaning output files: +out/production/module/_DefaultPackage$fun$*.class +End of files +Compiling files: +src/fun.kt +src/usage.kt +End of files +Cleaning output files: +out/production/module/_DefaultPackage$fun$*.class +out/production/module/_DefaultPackage$usage$*.class +out/production/module/_DefaultPackage.class +End of files +Compiling files: +src/fun.kt +src/usage.kt +End of files \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/fun.kt b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/fun.kt new file mode 100644 index 00000000000..99cec0dbea1 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/fun.kt @@ -0,0 +1,2 @@ +fun f() { +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt new file mode 100644 index 00000000000..81a9dfacc10 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt @@ -0,0 +1,3 @@ +fun other() { + +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt.delete.1 b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt.delete.1 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt new file mode 100644 index 00000000000..05a6473592e --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt @@ -0,0 +1,3 @@ +fun main(args: Array) { + f() +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.1 b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.1 new file mode 100644 index 00000000000..c7eec515ee8 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.1 @@ -0,0 +1,3 @@ +fun main(args: Array) { + f( +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.2 b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.2 new file mode 100644 index 00000000000..05a6473592e --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.2 @@ -0,0 +1,3 @@ +fun main(args: Array) { + f() +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/build.log b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/build.log new file mode 100644 index 00000000000..dae6fabaaff --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/build.log @@ -0,0 +1,30 @@ +Cleaning output files: +out/production/module/_DefaultPackage$other$*.class +out/production/module/_DefaultPackage$usage$*.class +out/production/module/_DefaultPackage.class +End of files +Compiling files: +src/other.kt +src/usage.kt +End of files +COMPILATION FAILED +Expecting an expression + + +Cleaning output files: +out/production/module/_DefaultPackage$fun$*.class +End of files +Compiling files: +src/fun.kt +src/other.kt +src/usage.kt +End of files +Cleaning output files: +out/production/module/_DefaultPackage$fun$*.class +out/production/module/_DefaultPackage$usage$*.class +out/production/module/_DefaultPackage.class +End of files +Compiling files: +src/fun.kt +src/usage.kt +End of files \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/fun.kt b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/fun.kt new file mode 100644 index 00000000000..99cec0dbea1 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/fun.kt @@ -0,0 +1,2 @@ +fun f() { +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt new file mode 100644 index 00000000000..81a9dfacc10 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt @@ -0,0 +1,3 @@ +fun other() { + +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt.new.1 b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt.new.1 new file mode 100644 index 00000000000..7eddc99ab2b --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt.new.1 @@ -0,0 +1,5 @@ +package new + +fun other() { + +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt new file mode 100644 index 00000000000..05a6473592e --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt @@ -0,0 +1,3 @@ +fun main(args: Array) { + f() +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.1 b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.1 new file mode 100644 index 00000000000..c7eec515ee8 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.1 @@ -0,0 +1,3 @@ +fun main(args: Array) { + f( +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.2 b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.2 new file mode 100644 index 00000000000..05a6473592e --- /dev/null +++ b/jps/jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.2 @@ -0,0 +1,3 @@ +fun main(args: Array) { + f() +} \ No newline at end of file