Integrate components of the new incremental compilation

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -205,14 +205,17 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
         updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile)
 
         val caches = filesToCompile.keySet().map { incrementalCaches[it]!! }
-        processChanges(context, chunk, allCompiledFiles, caches, changesInfo)
+        processChanges(context, chunk, filesToCompile.values(), allCompiledFiles, dataManager, caches, changesInfo)
+
         return ADDITIONAL_PASS_REQUIRED
     }
 
-    fun processChanges(
+    private fun processChanges(
             context: CompileContext,
             chunk: ModuleChunk,
+            compiledFiles: Collection<File>,
             allCompiledFiles: MutableSet<File>,
+            dataManager: BuildDataManager,
             caches: List<IncrementalCacheImpl>,
             changesInfo: ChangesInfo
     ) {
@@ -250,7 +253,34 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
             }
         }
 
-        changesInfo.doProcessChanges()
+        fun ChangesInfo.doProcessChangesUsingLookups() {
+            val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, LookupStorageProvider)
+
+            // TODO group by fqName?
+            for (change in changes) {
+
+                if (change !is ChangeInfo.MembersChanged) continue
+
+                val files = change.names
+                        .flatMap { lookupStorage.get(LookupSymbol(it, change.fqName.asString())) }
+                        .asSequence()
+                        .map { File(it) }
+                        .filter { it !in compiledFiles && it.exists() }
+
+                files.forEach {
+                    FSOperations.markDirty(context, CompilationRound.NEXT, it)
+                }
+            }
+
+            caches.forEach { it.cleanDirtyInlineFunctions() }
+        }
+
+        if (IncrementalCompilation.isExperimental()) {
+            changesInfo.doProcessChangesUsingLookups()
+        }
+        else {
+            changesInfo.doProcessChanges()
+        }
     }
 
     private fun doCompileModuleChunk(

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.jps.incremental
 
+import com.google.protobuf.MessageLite
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.util.io.BooleanDataDescriptor
 import com.intellij.util.io.EnumeratorStringDescriptor
@@ -28,6 +29,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider
 import org.jetbrains.jps.incremental.ModuleBuildTarget
 import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.incremental.storage.PathStringDescriptor
+import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.inline.inlineFunctionsJvmNames
 import org.jetbrains.kotlin.jps.build.GeneratedJvmClass
 import org.jetbrains.kotlin.jps.build.KotlinBuilder
@@ -36,8 +38,12 @@ import org.jetbrains.kotlin.load.kotlin.ModuleMapping
 import org.jetbrains.kotlin.load.kotlin.header.*
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.NameResolver
 import org.jetbrains.kotlin.serialization.jvm.BitEncoding
+import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
 import org.jetbrains.org.objectweb.asm.*
 import java.io.File
 import java.security.MessageDigest
@@ -122,10 +128,14 @@ public class IncrementalCacheImpl(
             dependents.forEach(::addFilesAffectedByChangedInlineFuns)
         }
 
-        dirtyInlineFunctionsMap.clean()
+        cleanDirtyInlineFunctions()
         return result.map { File(it) }
     }
 
+    public fun cleanDirtyInlineFunctions() {
+        dirtyInlineFunctionsMap.clean()
+    }
+
     override fun getClassFilePath(internalClassName: String): String {
         return File(outputDir, "$internalClassName.class").canonicalPath
     }
@@ -160,9 +170,11 @@ public class IncrementalCacheImpl(
                 assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" }
                 packagePartMap.addPackagePart(className)
 
-                protoMap.process(kotlinClass, isPackage = true) +
+                val isPackage = true
+
+                protoMap.process(kotlinClass, isPackage) +
                 constantsMap.process(kotlinClass) +
-                inlineFunctionsMap.process(kotlinClass)
+                inlineFunctionsMap.process(kotlinClass, isPackage)
             }
             header.isCompatibleMultifileClassKind() -> {
                 val partNames = kotlinClass.classHeader.filePartClassNames?.toList()
@@ -171,21 +183,25 @@ public class IncrementalCacheImpl(
 
                 // TODO NO_CHANGES? (delegates only, see package facade)
                 constantsMap.process(kotlinClass) +
-                inlineFunctionsMap.process(kotlinClass)
+                inlineFunctionsMap.process(kotlinClass, isPackage = true)
             }
             header.isCompatibleMultifileClassPartKind() -> {
                 assert(sourceFiles.size() == 1) { "Multifile class part from several source files: $sourceFiles" }
                 packagePartMap.addPackagePart(className)
                 multifileClassPartMap.add(className.internalName, header.multifileClassName!!)
 
-                protoMap.process(kotlinClass, isPackage = true) +
+                val isPackage = true
+
+                protoMap.process(kotlinClass, isPackage) +
                 constantsMap.process(kotlinClass) +
-                inlineFunctionsMap.process(kotlinClass)
+                inlineFunctionsMap.process(kotlinClass, isPackage)
             }
             header.isCompatibleClassKind() && !header.isLocalClass -> {
-                protoMap.process(kotlinClass, isPackage = false) +
+                val isPackage = false
+
+                protoMap.process(kotlinClass, isPackage) +
                 constantsMap.process(kotlinClass) +
-                inlineFunctionsMap.process(kotlinClass)
+                inlineFunctionsMap.process(kotlinClass, isPackage)
             }
             else -> ChangesInfo.NO_CHANGES
         }
@@ -201,12 +217,57 @@ public class IncrementalCacheImpl(
     }
 
     public fun clearCacheForRemovedClasses(): ChangesInfo {
+
+        fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>) =
+                members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
+
+        fun createChangeInfo(className: JvmClassName): ChangeInfo? {
+            if (className.internalName == MODULE_MAPPING_FILE_NAME) return null
+
+            val mapValue = protoMap.get(className) ?: return null
+
+            return when {
+                mapValue.isPackageFacade -> {
+                    val packageData = JvmProtoBufUtil.readPackageDataFrom(mapValue.bytes, mapValue.strings)
+
+                    val memberNames =
+                            packageData.packageProto.getNonPrivateNames(
+                                    packageData.nameResolver,
+                                    ProtoBuf.Package::getFunctionList,
+                                    ProtoBuf.Package::getPropertyList
+                            )
+
+                    ChangeInfo.Removed(className.packageFqName, memberNames)
+                }
+                else -> {
+                    val classData = JvmProtoBufUtil.readClassDataFrom(mapValue.bytes, mapValue.strings)
+
+                    val memberNames =
+                            classData.classProto.getNonPrivateNames(
+                                    classData.nameResolver,
+                                    ProtoBuf.Class::getConstructorList,
+                                    ProtoBuf.Class::getFunctionList,
+                                    ProtoBuf.Class::getPropertyList
+                            ) +
+                            classData.classProto.enumEntryList.map { classData.nameResolver.getString(it) }.toSet()
+
+                    ChangeInfo.Removed(className.fqNameForClassNameWithoutDollars, memberNames)
+                }
+            }
+        }
+
         val dirtyClasses = dirtyOutputClassesMap
                                 .getDirtyOutputClasses()
                                 .map(JvmClassName::byInternalName)
                                 .toList()
 
-        val changesInfo = dirtyClasses.fold(ChangesInfo.NO_CHANGES) { info, className ->
+        val changes =
+                if (IncrementalCompilation.isExperimental())
+                    dirtyClasses.map { createChangeInfo(it) }.asSequence().filterNotNull()
+                else
+                    emptySequence<ChangeInfo>()
+
+        val changesInfo = dirtyClasses.fold(ChangesInfo(changes = changes)) { info, className ->
             val newInfo = ChangesInfo(protoChanged = className in protoMap,
                                       constantsChanged = className in constantsMap)
             newInfo.logIfSomethingChanged(className)
@@ -268,10 +329,10 @@ public class IncrementalCacheImpl(
 
     private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
 
-        public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): ChangesInfo {
+        public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): ChangesInfo {
             val header = kotlinClass.classHeader
             val bytes = BitEncoding.decodeBytes(header.annotationData!!)
-            return put(kotlinClass.className, bytes, header.strings!!, isPackage, checkChangesIsOpenPart)
+            return put(kotlinClass.className, bytes, header.strings!!, isPackage, checkChangesIsOpenPart = true)
         }
 
         public fun process(className: JvmClassName, data: ByteArray, strings: Array<String>, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo {
@@ -288,13 +349,27 @@ public class IncrementalCacheImpl(
             if (oldData == null ||
                 !Arrays.equals(bytes, oldData.bytes) ||
                 !Arrays.equals(strings, oldData.strings) ||
-                isPackage != oldData.isPackageFacade) {
+                isPackage != oldData.isPackageFacade
+            ) {
                 storage[key] = data
             }
 
-            return ChangesInfo(protoChanged = oldData == null ||
+            if (oldData == null || !checkChangesIsOpenPart) return ChangesInfo(protoChanged = true)
-                                              !checkChangesIsOpenPart ||
+
-                                              difference(oldData, data) != DifferenceKind.NONE)
+            val diff = difference(oldData, data)
+
+            if (!IncrementalCompilation.isExperimental()) return ChangesInfo(protoChanged = diff != DifferenceKind.NONE)
+
+            val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars
+
+            val changes =
+                    when (diff) {
+                        is DifferenceKind.NONE -> emptySequence<ChangeInfo>()
+                        is DifferenceKind.CLASS_SIGNATURE -> sequenceOf(ChangeInfo.SignatureChanged(fqName))
+                        is DifferenceKind.MEMBERS -> sequenceOf(ChangeInfo.MembersChanged(fqName, diff.names))
+                    }
+
+            return ChangesInfo(protoChanged = diff != DifferenceKind.NONE, changes = changes)
         }
 
         public fun contains(className: JvmClassName): Boolean =
@@ -388,11 +463,11 @@ public class IncrementalCacheImpl(
             return result
         }
 
-        public fun process(kotlinClass: LocalFileKotlinClass): ChangesInfo {
+        public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): ChangesInfo {
-            return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.fileContents))
+            return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.fileContents), isPackage)
         }
 
-        private fun put(className: JvmClassName, newMap: Map<String, Long>): ChangesInfo {
+        private fun put(className: JvmClassName, newMap: Map<String, Long>, isPackage: Boolean): ChangesInfo {
             val internalName = className.internalName
             val oldMap = storage[internalName] ?: emptyMap()
 
@@ -419,8 +494,19 @@ public class IncrementalCacheImpl(
                 dirtyInlineFunctionsMap.put(className, changed.toList())
             }
 
+            val changes =
+                    if (IncrementalCompilation.isExperimental()) {
+                        val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars
+                        // TODO get name in better way instead of using substringBefore
+                        (added.asSequence() + changed.asSequence()).map { ChangeInfo.MembersChanged(fqName, listOf(it.substringBefore("("))) }
+                    }
+                    else {
+                        emptySequence<ChangeInfo>()
+                    }
+
             return ChangesInfo(inlineChanged = changed.isNotEmpty(),
-                               inlineAdded = added.isNotEmpty())
+                               inlineAdded = added.isNotEmpty(),
+                               changes = changes)
         }
 
         public fun remove(className: JvmClassName) {
@@ -559,11 +645,18 @@ public class IncrementalCacheImpl(
     }
 }
 
+sealed class ChangeInfo(val fqName: FqName) {
+    open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName)
+    class Removed(fqName: FqName, names: Collection<String>) : MembersChanged(fqName, names)
+    class SignatureChanged(fqName: FqName) : ChangeInfo(fqName)
+}
+
 data class ChangesInfo(
-        public val protoChanged: Boolean = false,
+        val protoChanged: Boolean = false,
-        public val constantsChanged: Boolean = false,
+        val constantsChanged: Boolean = false,
-        public val inlineChanged: Boolean = false,
+        val inlineChanged: Boolean = false,
-        public val inlineAdded: Boolean = false
+        val inlineAdded: Boolean = false,
+        val changes: Sequence<ChangeInfo> = emptySequence()
 ) {
     companion object {
         public val NO_CHANGES: ChangesInfo = ChangesInfo()
@@ -573,7 +666,8 @@ data class ChangesInfo(
             ChangesInfo(protoChanged || other.protoChanged,
                         constantsChanged || other.constantsChanged,
                         inlineChanged || other.inlineChanged,
-                        inlineAdded || other.inlineAdded)
+                        inlineAdded || other.inlineAdded,
+                        changes + other.changes)
 }
 
 
@@ -607,9 +701,6 @@ private fun ByteArray.md5(): Long {
            )
 }
 
-private val File.normalizedPath: String
-    get() = FileUtil.toSystemIndependentName(canonicalPath)
-
 @TestOnly
 private fun <K : Comparable<K>, V> Map<K, V>.dumpMap(dumpValue: (V)->String): String =
         StringBuilder {

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt

@@ -55,6 +55,16 @@ class LookupStorage(private val targetDataDir: File) : BasicMapsOwner() {
         }
     }
 
+    fun get(lookupSymbol: LookupSymbol): Collection<String> {
+        val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
+        val fileIds = lookupMap[key] ?: return emptySet()
+
+        return fileIds.map {
+            // null means it's outdated
+            idToFile[it]?.path
+        }.filterNotNull()
+    }
+
     public fun add(lookupSymbol: LookupSymbol, containingPaths: Collection<String>) {
         val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
         val fileIds = containingPaths.map { addFileIfNeeded(File(it)) }.toHashSet()

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt

@@ -44,6 +44,26 @@ public fun difference(oldData: ProtoMapValue, newData: ProtoMapValue): Differenc
     return differenceObject.difference()
 }
 
+internal val MessageLite.isPrivate: Boolean
+    get() = Visibilities.isPrivate(Deserialization.visibility(
+            when (this) {
+                is ProtoBuf.Constructor -> Flags.VISIBILITY.get(flags)
+                is ProtoBuf.Function -> Flags.VISIBILITY.get(flags)
+                is ProtoBuf.Property -> Flags.VISIBILITY.get(flags)
+                else -> error("Unknown message: $this")
+            }))
+
+private fun MessageLite.name(nameResolver: NameResolver): String {
+    return when (this) {
+        is ProtoBuf.Constructor -> "<init>"
+        is ProtoBuf.Function -> nameResolver.getString(name)
+        is ProtoBuf.Property -> nameResolver.getString(name)
+        else -> error("Unknown message: $this")
+    }
+}
+
+internal fun List<MessageLite>.names(nameResolver: NameResolver): List<String> = map { it.name(nameResolver) }
+
 private abstract class DifferenceCalculator() {
     protected abstract val oldNameResolver: NameResolver
     protected abstract val newNameResolver: NameResolver
@@ -57,9 +77,6 @@ private abstract class DifferenceCalculator() {
     protected fun calcDifferenceForMembers(oldList: List<MessageLite>, newList: List<MessageLite>): Collection<String> {
         val result = hashSetOf<String>()
 
-        fun List<MessageLite>.names(nameResolver: NameResolver): List<String> =
-                map { it.name(nameResolver) }
-
         val oldMap =
                 oldList.groupBy { it.getHashCode({ compareObject.oldGetIndexOfString(it) }, { compareObject.oldGetIndexOfClassId(it) }) }
         val newMap =
@@ -114,15 +131,6 @@ private abstract class DifferenceCalculator() {
         return HashSetUtil.symmetricDifference(oldNames, newNames)
     }
 
-    protected val MessageLite.isPrivate: Boolean
-        get() = Visibilities.isPrivate(Deserialization.visibility(
-                when (this) {
-                    is ProtoBuf.Constructor -> Flags.VISIBILITY.get(flags)
-                    is ProtoBuf.Function -> Flags.VISIBILITY.get(flags)
-                    is ProtoBuf.Property -> Flags.VISIBILITY.get(flags)
-                    else -> error("Unknown message: $this")
-                }))
-
     private fun MessageLite.getHashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (Int) -> Int): Int {
         return when (this) {
             is ProtoBuf.Constructor -> hashCode(stringIndexes, fqNameIndexes)
@@ -132,15 +140,6 @@ private abstract class DifferenceCalculator() {
         }
     }
 
-    private fun MessageLite.name(nameResolver: NameResolver): String {
-        return when (this) {
-            is ProtoBuf.Constructor -> "<init>"
-            is ProtoBuf.Function -> nameResolver.getString(name)
-            is ProtoBuf.Property -> nameResolver.getString(name)
-            else -> error("Unknown message: $this")
-        }
-    }
-
     private fun ProtoCompareGenerated.checkEquals(old: MessageLite, new: MessageLite): Boolean {
         return when {
             old is ProtoBuf.Constructor && new is ProtoBuf.Constructor -> checkEquals(old, new)