Clearing incremental cache for removed files without scanning output directory.

Original commit: 89f3d21ba4

jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -106,12 +106,6 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
 
         val incrementalCaches = chunk.getTargets().keysToMap { dataManager.getKotlinCache(it) }
 
-        for (target in chunk.getTargets()) {
-            val removedFiles = dirtyFilesHolder.getRemovedFiles(target)
-            val cache = incrementalCaches[target]!!
-            removedFiles.forEach { cache.fileIsDeleted(File(it)) }
-        }
-
         val environment = createCompileEnvironment(incrementalCaches)
         if (!environment.success()) {
             environment.reportErrorsTo(messageCollector)
@@ -129,6 +123,14 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
             compileToJs(chunk, commonArguments, environment, messageCollector, project)
         }
         else {
+            if (IncrementalCompilation.ENABLED) {
+                for (target in chunk.getTargets()) {
+                    val cache = incrementalCaches[target]!!
+                    val removedAndDirtyFiles = filesToCompile[target] + dirtyFilesHolder.getRemovedFiles(target).map { File(it) }
+                    cache.markOutputClassesDirty(removedAndDirtyFiles)
+                }
+            }
+
             compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, filesToCompile, messageCollector)
         }
 
@@ -146,7 +148,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
             recompilationDecision = DO_NOTHING
         }
         else {
-            recompilationDecision = updateKotlinIncrementalCache(compilationErrors, dirtyFilesHolder, incrementalCaches, outputsItemsAndTargets)
+            recompilationDecision = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, outputsItemsAndTargets)
             updateJavaMappings(chunk, compilationErrors, context, dirtyFilesHolder, filesToCompile, outputsItemsAndTargets)
         }
 
@@ -273,7 +275,6 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
 
     private fun updateKotlinIncrementalCache(
             compilationErrors: Boolean,
-            dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
             incrementalCaches: Map<ModuleBuildTarget, IncrementalCacheImpl>,
             outputsItemsAndTargets: List<Pair<SimpleOutputItem, ModuleBuildTarget>>
     ): IncrementalCacheImpl.RecompilationDecision {
@@ -281,23 +282,16 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
             return DO_NOTHING
         }
 
-        if (!compilationErrors) {
-            incrementalCaches.values().forEach { it.clearRemovedPackageParts () }
-        }
-
-        for ((target, cache) in incrementalCaches) {
-            cache.clearCacheForRemovedFiles(
-                    KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target),
-                    target.getOutputDir()!!,
-                    !compilationErrors
-            )
-        }
-
         var recompilationDecision = DO_NOTHING
         for ((outputItem, target) in outputsItemsAndTargets) {
             val newDecision = incrementalCaches[target]!!.saveFileToCache(outputItem.getSourceFiles(), outputItem.getOutputFile())
             recompilationDecision = recompilationDecision.merge(newDecision)
         }
+
+        if (!compilationErrors) {
+            incrementalCaches.values().forEach { it.clearCacheForRemovedClasses() }
+        }
+
         return recompilationDecision
     }
 

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -49,6 +49,7 @@ import com.intellij.util.io.BooleanDataDescriptor
 import java.util.ArrayList
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.utils.Printer
+import java.io.DataInputStream
 
 val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
 
@@ -88,7 +89,8 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         val CONSTANTS_MAP = "constants.tab"
         val INLINE_FUNCTIONS = "inline-functions.tab"
         val PACKAGE_PARTS = "package-parts.tab"
-        val REMOVED_PACKAGE_PARTS = "removed-package-parts.tab"
+        val SOURCE_TO_CLASSES = "source-to-classes.tab"
+        val DIRTY_OUTPUT_CLASSES = "dirty-output-classes.tab"
     }
 
     private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME)
@@ -96,9 +98,10 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
     private val constantsMap = ConstantsMap()
     private val inlineFunctionsMap = InlineFunctionsMap()
     private val packagePartMap = PackagePartMap()
-    private val removedPackagePartMap = RemovedPackagePartMap()
+    private val sourceToClassesMap = SourceToClassesMap()
+    private val dirtyOutputClassesMap = DirtyOutputClassesMap()
 
-    private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap, removedPackagePartMap)
+    private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap, sourceToClassesMap, dirtyOutputClassesMap)
 
     private val cacheFormatVersion = CacheFormatVersion(targetDataRoot)
 
@@ -107,6 +110,15 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         return maps.map { it.dump() }.join("\n\n")
     }
 
+    public fun markOutputClassesDirty(removedAndCompiledSources: List<File>) {
+        for (sourceFile in removedAndCompiledSources) {
+            val classes = sourceToClassesMap[sourceFile]
+            classes.forEach { dirtyOutputClassesMap.markDirty(it.getInternalName()) }
+
+            sourceToClassesMap.clearOutputsForSource(sourceFile)
+        }
+    }
+
     public fun saveFileToCache(sourceFiles: Collection<File>, classFile: File): RecompilationDecision {
         if (classFile.extension.toLowerCase() != "class") return DO_NOTHING
 
@@ -119,6 +131,9 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         val className = JvmClassName.byClassId(kotlinClass.getClassId())
         val header = kotlinClass.getClassHeader()
 
+        dirtyOutputClassesMap.notDirty(className.getInternalName())
+        sourceFiles.forEach { sourceToClassesMap.addSourceToClass(it, className) }
+
         val annotationDataEncoded = header.annotationData
         if (annotationDataEncoded != null) {
             val data = BitEncoding.decodeBytes(annotationDataEncoded)
@@ -147,7 +162,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         if (header.syntheticClassKind == JvmAnnotationNames.KotlinSyntheticClass.Kind.PACKAGE_PART) {
             assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" }
 
-            packagePartMap.putPackagePartSourceData(sourceFiles.first(), className)
+            packagePartMap.addPackagePart(className)
             val inlinesChanged = inlineFunctionsMap.process(className, fileBytes)
             val constantsChanged = constantsMap.process(className, fileBytes)
 
@@ -161,30 +176,19 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         return DO_NOTHING
     }
 
-    public fun fileIsDeleted(sourceFile: File) {
+    public fun clearCacheForRemovedClasses() {
-        val wasPackagePart = packagePartMap[sourceFile]
+        for (internalClassName in dirtyOutputClassesMap.getDirtyOutputClasses()) {
-        packagePartMap.remove(sourceFile)
+            val className = JvmClassName.byInternalName(internalClassName)
-        if (wasPackagePart != null) {
+            protoMap.remove(className)
-            removedPackagePartMap.add(wasPackagePart)
+            packagePartMap.remove(className)
-        }
+            constantsMap.remove(className)
-    }
+            inlineFunctionsMap.remove(className)
-
-    public fun clearRemovedPackageParts() {
-        removedPackagePartMap.clear()
-    }
-
-    // TODO transitional function
-    deprecated("transitional function")
-    public fun clearCacheForRemovedFiles(removedSourceFiles: Collection<File>, outDirectory: File, compilationSuccessful: Boolean) {
-        if (compilationSuccessful) {
-            inlineFunctionsMap.clearOutdated(outDirectory)
-            constantsMap.clearOutdated(outDirectory)
-            protoMap.clearOutdated(outDirectory)
         }
+        dirtyOutputClassesMap.clear()
     }
 
     public override fun getObsoletePackageParts(sourceFilesToCompile: Collection<File>): Collection<String> {
-        return (removedPackagePartMap.getRemovedParts() + sourceFilesToCompile.map { packagePartMap[it] }.filterNotNull()).toSet()
+        return dirtyOutputClassesMap.getDirtyOutputClasses().filter { packagePartMap.isPackagePart(JvmClassName.byInternalName(it)) }
     }
 
     public override fun getPackageData(fqName: String): ByteArray? {
@@ -260,29 +264,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         protected abstract fun dumpValue(value: V): String
     }
 
-    private abstract class ClassFileBasedMap<V> : BasicMap<V>() {
+    private inner class ProtoMap : BasicMap<ByteArray>() {
-
-        // TODO may be too expensive, because it traverses all files in out directory
-        public fun clearOutdated(outDirectory: File) {
-            val keysToRemove = HashSet<String>()
-
-            storage.processKeysWithExistingMapping { key ->
-                val className = JvmClassName.byInternalName(key!!)
-                val classFile = File(outDirectory, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
-                if (!classFile.exists()) {
-                    keysToRemove.add(key)
-                }
-
-                true
-            }
-
-            for (key in keysToRemove) {
-                storage.remove(key)
-            }
-        }
-    }
-
-    private inner class ProtoMap : ClassFileBasedMap<ByteArray>() {
         override fun createMap(): PersistentHashMap<String, ByteArray> = PersistentHashMap(
                 File(baseDir, PROTO_MAP),
                 EnumeratorStringDescriptor(),
@@ -303,12 +285,16 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
             return storage[className.getInternalName()]
         }
 
+        public fun remove(className: JvmClassName) {
+            storage.remove(className.getInternalName())
+        }
+
         override fun dumpValue(value: ByteArray): String {
             return java.lang.Long.toHexString(value.md5())
         }
     }
 
-    private inner class ConstantsMap : ClassFileBasedMap<Map<String, Any>>() {
+    private inner class ConstantsMap : BasicMap<Map<String, Any>>() {
         override fun createMap(): PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
                 File(baseDir, CONSTANTS_MAP),
                 EnumeratorStringDescriptor(),
@@ -351,6 +337,10 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
             return true
         }
 
+        public fun remove(className: JvmClassName) {
+            put(className, null)
+        }
+
         override fun dumpValue(value: Map<String, Any>): String {
             return StringBuilder {
                 append("{")
@@ -423,7 +413,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         }
     }
 
-    private inner class InlineFunctionsMap : ClassFileBasedMap<Map<String, Long>>() {
+    private inner class InlineFunctionsMap : BasicMap<Map<String, Long>>() {
         override fun createMap(): PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
                 File(baseDir, INLINE_FUNCTIONS),
                 EnumeratorStringDescriptor(),
@@ -482,6 +472,10 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
             return true
         }
 
+        public fun remove(className: JvmClassName) {
+            put(className, null)
+        }
+
         override fun dumpValue(value: Map<String, Long>): String {
             return StringBuilder {
                 append("{")
@@ -521,42 +515,66 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
 
     }
 
-
+    private inner class PackagePartMap : BasicMap<Boolean>() {
-    private inner class PackagePartMap : BasicMap<String>() {
-        // Format of serialization to string: <source file path>  -->  <package part JVM internal name>
-        override fun createMap(): PersistentHashMap<String, String> = PersistentHashMap(
-                File(baseDir, PACKAGE_PARTS),
-                EnumeratorStringDescriptor(),
-                EnumeratorStringDescriptor()
-        )
-
-        public fun putPackagePartSourceData(sourceFile: File, className: JvmClassName) {
-            storage.put(sourceFile.getAbsolutePath(), className.getInternalName())
-        }
-
-        public fun remove(sourceFile: File) {
-            storage.remove(sourceFile.getAbsolutePath())
-        }
-
-        public fun get(sourceFile: File): String? {
-            return storage[sourceFile.getAbsolutePath()]
-        }
-
-        override fun dumpValue(value: String) = value
-    }
-
-    private inner class RemovedPackagePartMap : BasicMap<Boolean>() {
         override fun createMap(): PersistentHashMap<String, Boolean> = PersistentHashMap(
-                File(baseDir, REMOVED_PACKAGE_PARTS),
+                File(baseDir, PACKAGE_PARTS),
                 EnumeratorStringDescriptor(),
                 BooleanDataDescriptor.INSTANCE
         )
 
-        public fun add(name: String) {
+        public fun addPackagePart(className: JvmClassName) {
-            storage.put(name, true)
+            storage.put(className.getInternalName(), true)
         }
 
-        public fun getRemovedParts(): Collection<String> {
+        public fun remove(className: JvmClassName) {
+            storage.remove(className.getInternalName())
+        }
+
+        public fun isPackagePart(className: JvmClassName): Boolean {
+            return storage.containsMapping(className.getInternalName())
+        }
+
+        override fun dumpValue(value: Boolean) = ""
+    }
+
+    private inner class SourceToClassesMap : BasicMap<List<String>>() {
+        override fun createMap(): PersistentHashMap<String, List<String>> = PersistentHashMap(
+                File(baseDir, SOURCE_TO_CLASSES),
+                EnumeratorStringDescriptor(),
+                StringListExternalizer
+        )
+
+        public fun clearOutputsForSource(sourceFile: File) {
+            storage.remove(sourceFile.getAbsolutePath())
+        }
+
+        public fun addSourceToClass(sourceFile: File, className: JvmClassName) {
+            storage.appendData(sourceFile.getAbsolutePath(), { out -> IOUtil.writeUTF(out, className.getInternalName()) })
+        }
+
+        public fun get(sourceFile: File): Collection<JvmClassName> {
+            return storage[sourceFile.getAbsolutePath()].orEmpty().map { JvmClassName.byInternalName(it) }
+        }
+
+        override fun dumpValue(value: List<String>) = value.toString()
+    }
+
+    private inner class DirtyOutputClassesMap : BasicMap<Boolean>() {
+        override fun createMap(): PersistentHashMap<String, Boolean> = PersistentHashMap(
+                File(baseDir, DIRTY_OUTPUT_CLASSES),
+                EnumeratorStringDescriptor(),
+                BooleanDataDescriptor.INSTANCE
+        )
+
+        public fun markDirty(className: String) {
+            storage.put(className, true)
+        }
+
+        public fun notDirty(className: String) {
+            storage.remove(className)
+        }
+
+        public fun getDirtyOutputClasses(): Collection<String> {
             return storage.getAllKeysWithExistingMapping()
         }
 
@@ -615,3 +633,17 @@ private object ByteArrayExternalizer : DataExternalizer<ByteArray> {
         return buf
     }
 }
+
+private object StringListExternalizer : DataExternalizer<List<String>> {
+    override fun save(out: DataOutput, value: List<String>) {
+        value.forEach { IOUtil.writeUTF(out, it) }
+    }
+
+    override fun read(`in`: DataInput): List<String> {
+        val result = ArrayList<String>()
+        while ((`in` as DataInputStream).available() > 0) {
+            result.add(IOUtil.readUTF(`in`))
+        }
+        return result
+    }
+}