Simplified process of filtering out obsolete package parts and loading descriptors from incremental caches.

Now when file is deleted or dirty (compiled right now), its old package part is used. Current package of file doesn't matter.

compiler/backend/src/org/jetbrains/kotlin/codegen/KotlinCodegenFacade.java

@@ -62,8 +62,8 @@ public class KotlinCodegenFacade {
             packageFqNameToFiles.putValue(file.getPackageFqName(), file);
         }
 
-        Set<FqName> removedPackageFiles = new HashSet<FqName>(state.getPackagesWithRemovedFiles());
-        for (FqName fqName : Sets.union(removedPackageFiles, packageFqNameToFiles.keySet())) {
+        Set<FqName> packagesWithObsoleteParts = new HashSet<FqName>(state.getPackagesWithObsoleteParts());
+        for (FqName fqName : Sets.union(packagesWithObsoleteParts, packageFqNameToFiles.keySet())) {
             generatePackage(state, fqName, packageFqNameToFiles.get(fqName), errorHandler);
         }
 

compiler/backend/src/org/jetbrains/kotlin/codegen/state/GenerationState.java

@@ -111,7 +111,7 @@ public class GenerationState {
     private final ModuleDescriptor module;
 
     @NotNull
-    private final Collection<FqName> packagesWithRemovedFiles;
+    private final Collection<FqName> packagesWithObsoleteParts;
 
     @Nullable
     private final String moduleId; // for PackageCodegen in incremental compilation mode
@@ -142,7 +142,7 @@ public class GenerationState {
             GenerateClassFilter generateClassFilter,
             boolean disableInline,
             boolean disableOptimization,
-            @Nullable Collection<FqName> packagesWithRemovedFiles,
+            @Nullable Collection<FqName> packagesWithObsoleteParts,
             @Nullable String moduleId,
             @NotNull DiagnosticSink diagnostics,
             @Nullable File outDirectory
@@ -152,7 +152,7 @@ public class GenerationState {
         this.module = module;
         this.files = files;
         this.moduleId = moduleId;
-        this.packagesWithRemovedFiles = packagesWithRemovedFiles == null ? Collections.<FqName>emptySet() : packagesWithRemovedFiles;
+        this.packagesWithObsoleteParts = packagesWithObsoleteParts == null ? Collections.<FqName>emptySet() : packagesWithObsoleteParts;
         this.classBuilderMode = builderFactory.getClassBuilderMode();
         this.disableInline = disableInline;
 
@@ -287,8 +287,8 @@ public class GenerationState {
     }
 
     @NotNull
-    public Collection<FqName> getPackagesWithRemovedFiles() {
-        return packagesWithRemovedFiles;
+    public Collection<FqName> getPackagesWithObsoleteParts() {
+        return packagesWithObsoleteParts;
     }
 
     @Nullable

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.java

@@ -338,13 +338,13 @@ public class KotlinToJVMBytecodeCompiler {
         CompilerConfiguration configuration = environment.getConfiguration();
         IncrementalCacheProvider incrementalCacheProvider = configuration.get(JVMConfigurationKeys.INCREMENTAL_CACHE_PROVIDER);
 
-        Collection<FqName> packagesWithRemovedFiles;
+        Collection<FqName> packagesWithObsoleteParts;
         if (moduleId == null || incrementalCacheProvider == null) {
-            packagesWithRemovedFiles = null;
+            packagesWithObsoleteParts = null;
         }
         else {
             IncrementalCache incrementalCache = incrementalCacheProvider.getIncrementalCache(moduleId);
-            packagesWithRemovedFiles = IncrementalPackage.getPackagesWithRemovedFiles(incrementalCache, environment.getSourceFiles());
+            packagesWithObsoleteParts = IncrementalPackage.getPackagesWithObsoleteParts(incrementalCache, environment.getSourceFiles());
         }
         BindingTraceContext diagnosticHolder = new BindingTraceContext();
         GenerationState generationState = new GenerationState(
@@ -359,7 +359,7 @@ public class KotlinToJVMBytecodeCompiler {
                 GenerationState.GenerateClassFilter.GENERATE_ALL,
                 configuration.get(JVMConfigurationKeys.DISABLE_INLINE, false),
                 configuration.get(JVMConfigurationKeys.DISABLE_OPTIMIZATION, false),
-                packagesWithRemovedFiles,
+                packagesWithObsoleteParts,
                 moduleId,
                 diagnosticHolder,
                 outputDirectory

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalCacheUtil.kt

@@ -24,21 +24,13 @@ import java.util.HashMap
 import java.io.File
 import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils
 
-public fun IncrementalCache.getPackagesWithRemovedFiles(sourceFilesToCompile: Collection<JetFile>): Collection<FqName> {
-    return getRemovedPackageParts(sourceFilesToCompile).map { it.getPackageFqName() }
+// TODO JetFiles are redundant
+public fun IncrementalCache.getPackagesWithObsoleteParts(sourceFilesToCompile: Collection<JetFile>): Collection<FqName> {
+    return getObsoletePackageParts(sourceFilesToCompile).map { it.getPackageFqName() }
 }
 
-public fun IncrementalCache.getRemovedPackageParts(sourceFilesToCompile: Collection<JetFile>): Collection<JvmClassName> {
-    val sourceFilesToFqName = HashMap<File, String?>()
-    for (sourceFile in sourceFilesToCompile) {
-        sourceFilesToFqName[File(sourceFile.getVirtualFile()!!.getPath())] =
-                if (PackagePartClassUtils.fileHasCallables(sourceFile))
-                    sourceFile.getPackageFqName().asString()
-                else
-                    null
-    }
-
-    return getRemovedPackageParts(sourceFilesToFqName).map { JvmClassName.byInternalName(it) }
+public fun IncrementalCache.getObsoletePackageParts(sourceFilesToCompile: Collection<JetFile>): Collection<JvmClassName> {
+    return getObsoletePackageParts(sourceFilesToCompile.map { File(it.getVirtualFile()!!.getPath()) }).map { JvmClassName.byInternalName(it) }
 }
 
 public fun IncrementalCache.getPackageData(fqName: FqName): ByteArray? {

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackageFragmentProvider.kt

@@ -47,10 +47,7 @@ public class IncrementalPackageFragmentProvider(
         val moduleId: String
 ) : PackageFragmentProvider {
 
-    val packagePartsToNotLoadFromCache = (
-            sourceFiles.map { PackagePartClassUtils.getPackagePartInternalName(it) }
-                    + incrementalCache.getRemovedPackageParts(sourceFiles).map { it.getInternalName() }
-            ).toSet()
+    val obsoletePackageParts = incrementalCache.getObsoletePackageParts(sourceFiles).map { it.getInternalName() }.toSet()
     val fqNameToSubFqNames = MultiMap<FqName, FqName>()
     val fqNameToPackageFragment = HashMap<FqName, PackageFragmentDescriptor>()
     val fqNamesToLoad: Set<FqName>
@@ -70,10 +67,7 @@ public class IncrementalPackageFragmentProvider(
             fqNameToPackageFragment[fqName] = IncrementalPackageFragment(fqName)
         }
 
-        fqNamesToLoad = (
-                PackagePartClassUtils.getPackageFilesWithCallables(sourceFiles).map { it.getPackageFqName() }
-                + incrementalCache.getPackagesWithRemovedFiles(sourceFiles)
-        ).toSet()
+        fqNamesToLoad = incrementalCache.getPackagesWithObsoleteParts(sourceFiles).toSet()
 
         fqNamesToLoad.forEach { createPackageFragment(it) }
     }
@@ -119,7 +113,7 @@ public class IncrementalPackageFragmentProvider(
                             if (member.hasExtension(JvmProtoBuf.implClassName)) {
                                 val shortName = packageData.getNameResolver().getName(member.getExtension(JvmProtoBuf.implClassName)!!)
                                 val internalName = JvmClassName.byFqNameWithoutInnerClasses(fqName.child(shortName)).getInternalName()
-                                internalName !in packagePartsToNotLoadFromCache
+                                internalName !in obsoletePackageParts
                             }
                             else {
                                 true

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/cache/IncrementalCache.kt

@@ -19,10 +19,7 @@ package org.jetbrains.kotlin.load.kotlin.incremental.cache
 import java.io.File
 
 public trait IncrementalCache {
-    public fun getRemovedPackageParts(
-            // null value means source file has no top-level callables (won't produce package part)
-            sourceFilesToCompileAndFqNames: Map<File, String?>
-    ): Collection<String>
+    public fun getObsoletePackageParts(sourceFilesToCompile: Collection<File>): Collection<String>
 
     public fun getPackageData(fqName: String): ByteArray?
 

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -105,6 +105,13 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
         messageCollector.report(INFO, "Kotlin JPS plugin version " + KotlinVersion.VERSION, NO_LOCATION)
 
         val incrementalCaches = chunk.getTargets().keysToMap { dataManager.getKotlinCache(it) }
+
+        for (target in chunk.getTargets()) {
+            val removedFiles = dirtyFilesHolder.getRemovedFiles(target)
+            val cache = incrementalCaches[target]!!
+            removedFiles.forEach { cache.fileIsDeleted(File(it)) }
+        }
+
         val environment = createCompileEnvironment(incrementalCaches)
         if (!environment.success()) {
             environment.reportErrorsTo(messageCollector)
@@ -274,6 +281,10 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
             return DO_NOTHING
         }
 
+        if (!compilationErrors) {
+            incrementalCaches.values().forEach { it.clearRemovedPackageParts () }
+        }
+
         for ((target, cache) in incrementalCaches) {
             cache.clearCacheForRemovedFiles(
                     KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target),

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -45,6 +45,8 @@ import org.jetbrains.kotlin.load.kotlin.header.isCompatibleClassKind
 import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.builders.BuildTarget
 import org.jetbrains.jps.builders.storage.BuildDataPaths
+import com.intellij.util.io.BooleanDataDescriptor
+import java.util.ArrayList
 
 val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
 
@@ -54,7 +56,7 @@ private val CACHE_DIRECTORY_NAME = "kotlin"
 class CacheFormatVersion(targetDataRoot: File) {
     class object {
         // Change this when incremental cache format changes
-        private val INCREMENTAL_CACHE_OWN_VERSION = 1
+        private val INCREMENTAL_CACHE_OWN_VERSION = 2
         private val CACHE_FORMAT_VERSION: Int = INCREMENTAL_CACHE_OWN_VERSION * 1000000 + JvmAbi.VERSION
         val FORMAT_VERSION_FILE_PATH: String = "$CACHE_DIRECTORY_NAME/format-version.txt"
     }
@@ -84,6 +86,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         val CONSTANTS_MAP = "constants.tab"
         val INLINE_FUNCTIONS = "inline-functions.tab"
         val PACKAGE_PARTS = "package-parts.tab"
+        val REMOVED_PACKAGE_PARTS = "removed-package-parts.tab"
     }
 
     private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME)
@@ -91,8 +94,9 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
     private val constantsMap = ConstantsMap()
     private val inlineFunctionsMap = InlineFunctionsMap()
     private val packagePartMap = PackagePartMap()
+    private val removedPackagePartMap = RemovedPackagePartMap()
 
-    private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap)
+    private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap, removedPackagePartMap)
 
     private val cacheFormatVersion = CacheFormatVersion(targetDataRoot)
 
@@ -150,9 +154,21 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         return DO_NOTHING
     }
 
-    public fun clearCacheForRemovedFiles(removedSourceFiles: Collection<File>, outDirectory: File, compilationSuccessful: Boolean) {
-        removedSourceFiles.forEach { packagePartMap.remove(it) }
+    public fun fileIsDeleted(sourceFile: File) {
+        val wasPackagePart = packagePartMap[sourceFile]
+        packagePartMap.remove(sourceFile)
+        if (wasPackagePart != null) {
+            removedPackagePartMap.add(wasPackagePart)
+        }
+    }
 
+    public fun clearRemovedPackageParts() {
+        removedPackagePartMap.clear()
+    }
+
+    // TODO transitional function
+    deprecated("transitional function")
+    public fun clearCacheForRemovedFiles(removedSourceFiles: Collection<File>, outDirectory: File, compilationSuccessful: Boolean) {
         if (compilationSuccessful) {
             inlineFunctionsMap.clearOutdated(outDirectory)
             constantsMap.clearOutdated(outDirectory)
@@ -160,8 +176,8 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
         }
     }
 
-    public override fun getRemovedPackageParts(sourceFilesToCompileAndFqNames: Map<File, String?>): Collection<String> {
-        return packagePartMap.getRemovedPackageParts(sourceFilesToCompileAndFqNames)
+    public override fun getObsoletePackageParts(sourceFilesToCompile: Collection<File>): Collection<String> {
+        return (removedPackagePartMap.getRemovedParts() + sourceFilesToCompile.map { packagePartMap[it] }.filterNotNull()).toSet()
     }
 
     public override fun getPackageData(fqName: String): ByteArray? {
@@ -465,29 +481,28 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen
             storage.remove(sourceFile.getAbsolutePath())
         }
 
-        public fun getRemovedPackageParts(compiledSourceFilesToFqName: Map<File, String?>): Collection<String> {
-            val result = HashSet<String>()
+        public fun get(sourceFile: File): String? {
+            return storage[sourceFile.getAbsolutePath()]
+        }
+    }
 
-            storage.processKeysWithExistingMapping { key ->
-                val sourceFile = File(key!!)
+    private inner class RemovedPackagePartMap : BasicMap<Boolean>() {
+        override fun createMap(): PersistentHashMap<String, Boolean> = PersistentHashMap(
+                File(baseDir, REMOVED_PACKAGE_PARTS),
+                EnumeratorStringDescriptor(),
+                BooleanDataDescriptor.INSTANCE
+        )
 
-                val packagePartClassName = storage[key]!!
-                if (!sourceFile.exists()) {
-                    result.add(packagePartClassName)
-                }
-                else {
-                    if (sourceFile in compiledSourceFilesToFqName) {
-                        val previousPackageFqName = JvmClassName.byInternalName(packagePartClassName).getPackageFqName()
-                        if (compiledSourceFilesToFqName[sourceFile] != previousPackageFqName.asString()) {
-                            result.add(packagePartClassName)
-                        }
-                    }
-                }
+        public fun add(name: String) {
+            storage.put(name, true)
+        }
 
-                true
-            }
+        public fun getRemovedParts(): Collection<String> {
+            return storage.getAllKeysWithExistingMapping()
+        }
 
-            return result
+        public fun clear() {
+            storage.getAllKeysWithExistingMapping().forEach { storage.remove(it) }
         }
     }
 

jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalJpsTestGenerated.java

@@ -183,6 +183,18 @@ public class IncrementalJpsTestGenerated extends AbstractIncrementalJpsTest {
             doTest(fileName);
         }
 
+        @TestMetadata("compilationErrorThenFixedWithPhantomPart")
+        public void testCompilationErrorThenFixedWithPhantomPart() throws Exception {
+            String fileName = JetTestUtils.navigationMetadata("jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/");
+            doTest(fileName);
+        }
+
+        @TestMetadata("compilationErrorThenFixedWithPhantomPart2")
+        public void testCompilationErrorThenFixedWithPhantomPart2() throws Exception {
+            String fileName = JetTestUtils.navigationMetadata("jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/");
+            doTest(fileName);
+        }
+
         @TestMetadata("conflictingPlatformDeclarations")
         public void testConflictingPlatformDeclarations() throws Exception {
             String fileName = JetTestUtils.navigationMetadata("jps-plugin/testData/incremental/pureKotlin/conflictingPlatformDeclarations/");

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/build.log

@@ -0,0 +1,30 @@
+Cleaning output files:
+out/production/module/_DefaultPackage$other$*.class
+out/production/module/_DefaultPackage.class
+End of files
+Cleaning output files:
+out/production/module/_DefaultPackage$usage$*.class
+End of files
+Compiling files:
+src/usage.kt
+End of files
+COMPILATION FAILED
+Expecting an expression
+
+
+Cleaning output files:
+out/production/module/_DefaultPackage$fun$*.class
+End of files
+Compiling files:
+src/fun.kt
+src/usage.kt
+End of files
+Cleaning output files:
+out/production/module/_DefaultPackage$fun$*.class
+out/production/module/_DefaultPackage$usage$*.class
+out/production/module/_DefaultPackage.class
+End of files
+Compiling files:
+src/fun.kt
+src/usage.kt
+End of files

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/fun.kt

@@ -0,0 +1,2 @@
+fun f() {
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/other.kt

@@ -0,0 +1,3 @@
+fun other() {
+
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    f()
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.1

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    f(
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart/usage.kt.new.2

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    f()
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/build.log

@@ -0,0 +1,30 @@
+Cleaning output files:
+out/production/module/_DefaultPackage$other$*.class
+out/production/module/_DefaultPackage$usage$*.class
+out/production/module/_DefaultPackage.class
+End of files
+Compiling files:
+src/other.kt
+src/usage.kt
+End of files
+COMPILATION FAILED
+Expecting an expression
+
+
+Cleaning output files:
+out/production/module/_DefaultPackage$fun$*.class
+End of files
+Compiling files:
+src/fun.kt
+src/other.kt
+src/usage.kt
+End of files
+Cleaning output files:
+out/production/module/_DefaultPackage$fun$*.class
+out/production/module/_DefaultPackage$usage$*.class
+out/production/module/_DefaultPackage.class
+End of files
+Compiling files:
+src/fun.kt
+src/usage.kt
+End of files

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/fun.kt

@@ -0,0 +1,2 @@
+fun f() {
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt

@@ -0,0 +1,3 @@
+fun other() {
+
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/other.kt.new.1

@@ -0,0 +1,5 @@
+package new
+
+fun other() {
+
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    f()
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.1

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    f(
+}

jps-plugin/testData/incremental/pureKotlin/compilationErrorThenFixedWithPhantomPart2/usage.kt.new.2

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    f()
+}