KT-34862 use abi snapshot instead of build history files

Use jar snapshot instead build history file to avoid using time stamps and support remote gradle cache

build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt

@@ -17,6 +17,7 @@ enum class BuildAttributeKind : Serializable {
 
 enum class BuildAttribute(val kind: BuildAttributeKind) : Serializable {
     NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON),
+    NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON),
     CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON),
     UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON),
     JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON),

build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt

@@ -9,6 +9,7 @@ import java.io.Serializable
 
 @Suppress("Reformat")
 enum class BuildTime(val parent: BuildTime? = null) : Serializable {
+    GRADLE_TASK_ACTION,
     GRADLE_TASK,
         CLEAR_OUTPUT(GRADLE_TASK),
         BACKUP_OUTPUT(GRADLE_TASK),
@@ -20,6 +21,10 @@ enum class BuildTime(val parent: BuildTime? = null) : Serializable {
             NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILER),
             NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILER),
             INCREMENTAL_COMPILATION(RUN_COMPILER),
+                STORE_BUILD_INFO(INCREMENTAL_COMPILATION),
+                JAR_SNAPSHOT(INCREMENTAL_COMPILATION),
+                    SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT),
+                    IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT),
                 IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION),
                     IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET),
                         IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES),

build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt

@@ -30,6 +30,28 @@ class ChangesCollector {
     private val changedMembers = hashMapOf<FqName, MutableSet<String>>()
     private val areSubclassesAffected = hashMapOf<FqName, Boolean>()
 
+    //TODO for test only: ProtoData or ProtoBuf
+    private val storage = hashMapOf<FqName, ProtoData>()
+    private val removed = ArrayList<FqName>()
+
+    //TODO change to immutable map
+    fun protoDataChanges(): Map<FqName, ProtoData> = storage
+    fun protoDataRemoved(): List<FqName> = removed
+
+    companion object {
+        fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>) =
+            members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
+
+        fun ClassProtoData.getNonPrivateMemberNames(): Set<String> {
+            return proto.getNonPrivateNames(
+                nameResolver,
+                ProtoBuf.Class::getConstructorList,
+                ProtoBuf.Class::getFunctionList,
+                ProtoBuf.Class::getPropertyList
+            ) + proto.enumEntryList.map { nameResolver.getString(it.name) }
+        }
+    }
+
     fun changes(): List<ChangeInfo> {
         val changes = arrayListOf<ChangeInfo>()
 
@@ -57,7 +79,7 @@ class ChangesCollector {
     }
 
     private fun <T, R> MutableMap<T, MutableSet<R>>.getSet(key: T) =
-            getOrPut(key) { HashSet() }
+        getOrPut(key) { HashSet() }
 
     private fun collectChangedMember(scope: FqName, name: String) {
         changedMembers.getSet(scope).add(name)
@@ -79,11 +101,35 @@ class ChangesCollector {
         }
     }
 
-    fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?, collectAllMembersForNewClass: Boolean = false) {
+    fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?, collectAllMembersForNewClass: Boolean = false, packageProtoKey: String? = null) {
         if (oldData == null && newData == null) {
             throw IllegalStateException("Old and new value are null")
         }
 
+        if (newData != null) {
+            when (newData) {
+                is ClassProtoData -> {
+                    val fqName = newData.nameResolver.getClassId(newData.proto.fqName).asSingleFqName()
+                    storage[fqName] = newData
+                }
+                is PackagePartProtoData -> {
+                    //TODO fqName is not unique. It's package and can be present in both java and kotlin
+                    val fqName = newData.packageFqName
+                    storage[packageProtoKey?.let { FqName(it) } ?: fqName] = newData
+                }
+            }
+        } else {
+            when (oldData) {
+                is ClassProtoData -> {
+                    removed.add(oldData.nameResolver.getClassId(oldData.proto.fqName).asSingleFqName())
+                }
+                is PackagePartProtoData -> {
+                    //TODO fqName is not unique. It's package and can be present in both java and kotlin
+                    removed.add(packageProtoKey?.let { FqName(it) } ?: oldData.packageFqName)
+                }
+            }
+        }
+
         if (oldData == null) {
             newData!!.collectAll(isRemoved = false, isAdded = true, collectAllMembersForNewClass = collectAllMembersForNewClass)
             return
@@ -125,8 +171,8 @@ class ChangesCollector {
         }
     }
 
-    private fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>): Set<String> =
-            members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
+    fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>) =
+        members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
 
     //TODO remember all sealed parent classes
     private fun ProtoData.collectAll(isRemoved: Boolean, isAdded: Boolean, collectAllMembersForNewClass: Boolean = false) =
@@ -137,16 +183,15 @@ class ChangesCollector {
 
     private fun PackagePartProtoData.collectAllFromPackage(isRemoved: Boolean) {
         val memberNames =
-                proto.getNonPrivateNames(
-                        nameResolver,
-                        ProtoBuf.Package::getFunctionList,
-                        ProtoBuf.Package::getPropertyList
-                )
+            proto.getNonPrivateNames(
+                nameResolver,
+                ProtoBuf.Package::getFunctionList,
+                ProtoBuf.Package::getPropertyList
+            )
 
         if (isRemoved) {
             collectRemovedMembers(packageFqName, memberNames)
-        }
-        else {
+        } else {
             collectChangedMembers(packageFqName, memberNames)
         }
     }
@@ -161,8 +206,7 @@ class ChangesCollector {
             val collectMember = if (isRemoved) this@ChangesCollector::collectRemovedMember else this@ChangesCollector::collectChangedMember
             collectMember(classFqName.parent(), classFqName.shortName().asString())
             memberNames.forEach { collectMember(classFqName, it) }
-        }
-        else {
+        } else {
             if (!isRemoved && collectAllMembersForNewClass) {
                 val memberNames = getNonPrivateMemberNames()
                 memberNames.forEach { this@ChangesCollector.collectChangedMember(classFqName, it) }
@@ -189,15 +233,6 @@ class ChangesCollector {
         addChangedParents(fqName, changedParentsFqNames)
     }
 
-    private fun ClassProtoData.getNonPrivateMemberNames(): Set<String> {
-        return proto.getNonPrivateNames(
-                nameResolver,
-                ProtoBuf.Class::getConstructorList,
-                ProtoBuf.Class::getFunctionList,
-                ProtoBuf.Class::getPropertyList
-        ) + proto.enumEntryList.map { nameResolver.getString(it.name) }
-    }
-
     fun collectMemberIfValueWasChanged(scope: FqName, name: String, oldValue: Any?, newValue: Any?) {
         if (oldValue == null && newValue == null) {
             throw IllegalStateException("Old and new value are null for $scope#$name")
@@ -205,8 +240,7 @@ class ChangesCollector {
 
         if (oldValue != null && newValue == null) {
             collectRemovedMember(scope, name)
-        }
-        else if (oldValue != newValue) {
+        } else if (oldValue != newValue) {
             collectChangedMember(scope, name)
         }
     }

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt

@@ -185,7 +185,7 @@ open class IncrementalJvmCache(
         sourceToClassesMap.add(source, jvmClassName)
         val (proto, nameResolver) = serializedJavaClass.toProtoData()
         addToClassStorage(proto, nameResolver, source)
-
+//        collector.addJavaProto(ClassProtoData(proto, nameResolver))
         dirtyOutputClassesMap.notDirty(jvmClassName)
     }
 
@@ -308,7 +308,7 @@ open class IncrementalJvmCache(
             storage[key] = newData
 
             val packageFqName = kotlinClass.className.packageFqName
-            changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName))
+            changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName), packageProtoKey = key)
         }
 
         operator fun contains(className: JvmClassName): Boolean =

build-common/src/org/jetbrains/kotlin/incremental/IncrementalModuleInfo.kt

@@ -12,7 +12,8 @@ data class IncrementalModuleEntry(
     private val projectPath: String,
     val name: String,
     val buildDir: File,
-    val buildHistoryFile: File
+    val buildHistoryFile: File,
+    val abiSnapshot: File
 ) : Serializable {
     companion object {
         private const val serialVersionUID = 0L
@@ -26,7 +27,9 @@ class IncrementalModuleInfo(
     val nameToModules: Map<String, Set<IncrementalModuleEntry>>,
     val jarToClassListFile: Map<File, File>,
     // only for js and mpp
-    val jarToModule: Map<File, IncrementalModuleEntry>
+    val jarToModule: Map<File, IncrementalModuleEntry>,
+    //for JVM only
+    val jarToAbiSnapshot: Map<File, File>
 ) : Serializable {
     companion object {
         private const val serialVersionUID = 1L

build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt

@@ -41,7 +41,7 @@ open class LookupStorage(
     private val countersFile = "counters".storageFile
     private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter))
     private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter))
-    private val lookupMap = registerMap(LookupMap("lookups".storageFile))
+    val lookupMap = registerMap(LookupMap("lookups".storageFile))
 
     @Volatile
     private var size: Int = 0

build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt

@@ -255,22 +255,23 @@ fun withSubtypes(
     typeFqName: FqName,
     caches: Iterable<IncrementalCacheCommon>
 ): Set<FqName> {
-    val types = LinkedHashSet(listOf(typeFqName))
-    val subtypes = hashSetOf<FqName>()
+    val typesToProccess = LinkedHashSet(listOf(typeFqName))
+    val proccessedTypes = hashSetOf<FqName>()
 
-    while (types.isNotEmpty()) {
-        val iterator = types.iterator()
+
+    while (typesToProccess.isNotEmpty()) {
+        val iterator = typesToProccess.iterator()
         val unprocessedType = iterator.next()
         iterator.remove()
 
         caches.asSequence()
             .flatMap { it.getSubtypesOf(unprocessedType) }
-            .filter { it !in subtypes }
-            .forEach { types.add(it) }
+            .filter { it !in proccessedTypes }
+            .forEach { typesToProccess.add(it) }
 
-        subtypes.add(unprocessedType)
+        proccessedTypes.add(unprocessedType)
     }
 
-    return subtypes
+    return proccessedTypes
 }
 

build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt

@@ -18,7 +18,7 @@ package org.jetbrains.kotlin.incremental.storage
 
 import java.io.File
 
-internal class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) {
+class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) {
     override fun dumpKey(key: LookupSymbolKey): String = key.toString()
 
     override fun dumpValue(value: Collection<Int>): String = value.toString()

build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt

@@ -21,23 +21,49 @@ import com.intellij.util.io.DataExternalizer
 import com.intellij.util.io.EnumeratorStringDescriptor
 import com.intellij.util.io.IOUtil
 import com.intellij.util.io.KeyDescriptor
+import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import java.io.DataInput
 import java.io.DataInputStream
 import java.io.DataOutput
 import java.io.File
 import java.util.*
 
+/**
+ * Storage versioning:
+ * 0 - only name and value hashes are saved
+ * 1 - name and scope are saved
+ */
 object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
     override fun read(input: DataInput): LookupSymbolKey {
-        val first = input.readInt()
-        val second = input.readInt()
-
-        return LookupSymbolKey(first, second)
+        val version = input.readByte()
+        return when (version.toInt()) {
+            0 -> {
+                val name = input.readUTF()
+                val scope = input.readUTF()
+                LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope)
+            }
+            1 -> {
+                val first = input.readInt()
+                val second = input.readInt()
+                LookupSymbolKey(first, second, "", "")
+            }
+            else -> throw RuntimeException("Unknown version of LookupSymbolKeyDescriptor=${version}")
+        }
     }
 
+    private val storeFullFqName = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value.toBooleanLenient() ?: false
+
     override fun save(output: DataOutput, value: LookupSymbolKey) {
-        output.writeInt(value.nameHash)
-        output.writeInt(value.scopeHash)
+        if (storeFullFqName) {
+            output.writeByte(0)
+            output.writeUTF(value.name)
+            output.writeUTF(value.scope)
+        } else {
+            output.writeByte(1)
+            output.writeInt(value.nameHash)
+            output.writeInt(value.scopeHash)
+        }
     }
 
     override fun getHashCode(value: LookupSymbolKey): Int = value.hashCode()

build-common/src/org/jetbrains/kotlin/incremental/storage/values.kt

@@ -16,8 +16,8 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
-data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int) : Comparable<LookupSymbolKey> {
-    constructor(name: String, scope: String) : this(name.hashCode(), scope.hashCode())
+data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int, val name:String, val scope:String) : Comparable<LookupSymbolKey> {
+    constructor(name: String, scope: String) : this(name.hashCode(), scope.hashCode(), name, scope)
 
     override fun compareTo(other: LookupSymbolKey): Int {
         val nameCmp = nameHash.compareTo(other.nameHash)
@@ -26,6 +26,26 @@ data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int) : Comparable<L
 
         return scopeHash.compareTo(other.scopeHash)
     }
+
+    override fun hashCode(): Int {
+        var result = nameHash
+        result = 31 * result + scopeHash
+        return result
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as LookupSymbolKey
+
+        if (nameHash != other.nameHash) return false
+        if (scopeHash != other.scopeHash) return false
+
+        return true
+    }
+
+
 }
 
 data class ProtoMapValue(val isPackageFacade: Boolean, val bytes: ByteArray, val strings: Array<String>)

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt

@@ -35,6 +35,7 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess
     DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY("kotlin.daemon.socket.connect.interval"),
     KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY("kotlin.environment.keepalive"),
     COMPILE_DAEMON_CUSTOM_RUN_FILES_PATH_FOR_TESTS("kotlin.daemon.custom.run.files.path.for.tests"),
+    COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS("kotlin.incremental.classpath.snapshot.enabled"),
     KOTLIN_COLORS_ENABLED_PROPERTY("kotlin.colors.enabled"),
 
     KOTLIN_STAT_ENABLED_PROPERTY("kotlin.plugin.stat.enabled"),
@@ -48,7 +49,7 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess
     USER_HOME("user.home", alwaysDirectAccess = true),
     JAVA_VERSION("java.specification.version", alwaysDirectAccess = true),
     JAVA_HOME("java.home", alwaysDirectAccess = true),
-    JAVA_CLASS_PATH("java.class.path", alwaysDirectAccess = true),
+    JAVA_CLASS_PATH("java.class.path", alwaysDirectAccess = true)
     ;
 
     private fun <T> getProperFunction(custom: T?, default: T): T {

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt

@@ -312,6 +312,8 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
     if (inheritAdditionalProperties) {
         CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.property}=\"$it\"") }
         CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.property}") }
+        //Temporary solution to test abi snapshot
+        CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.property}") }
     }
 
     if (opts.jvmParams.none { it.matches(jvmAssertArgsRegex) }) {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt

@@ -0,0 +1,165 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.build.report.BuildReporter
+import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
+import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmNameResolver
+import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
+import org.jetbrains.kotlin.metadata.jvm.serialization.JvmStringTable
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.protobuf.MessageLite
+import java.io.*
+
+interface AbiSnapshot {
+    val protos: MutableMap<FqName, ProtoData>
+}
+
+
+
+class AbiSnapshotImpl(override val protos: MutableMap<FqName, ProtoData>) : AbiSnapshot {
+
+    companion object {
+        fun ObjectInputStream.readStringArray(): Array<String> {
+            val size = readInt()
+            val stringArray = arrayOfNulls<String>(size)
+            repeat(size) {
+                stringArray[it] = readUTF()
+            }
+
+            return stringArray.requireNoNulls()
+        }
+
+
+        fun ObjectInputStream.readAbiSnapshot(): AbiSnapshotImpl {
+            // Format:
+            // numRecords: Int
+            // record {
+            //   fqName
+            //   isClassProtoData
+            //   *for packageClassData - packageFqName
+            //   protodata via JvmProtoBufUtil
+            //   string array with size
+            // }
+            val size = readInt()
+            val mutableMap = hashMapOf<FqName, ProtoData>()
+            repeat(size) {
+                val fqNameString = readUTF()
+                val isClassProtoData = readBoolean()
+                if (isClassProtoData) {
+                    val fqName = FqName(fqNameString)
+                    val bytes = readStringArray()
+                    val strings = readStringArray()
+                    val (nameResolver, classProto) = JvmProtoBufUtil.readClassDataFrom(bytes, strings)
+                    mutableMap[fqName] = ClassProtoData(classProto, nameResolver)
+                } else {
+                    val fqName = FqName(fqNameString)
+                    val packageFqName = FqName(readUTF())
+                    val bytes = readStringArray()
+                    val strings = readStringArray()
+                    val (nameResolver, proto) = JvmProtoBufUtil.readPackageDataFrom(bytes, strings)
+                    mutableMap[fqName] = PackagePartProtoData(proto, nameResolver, packageFqName)
+                }
+            }
+            return AbiSnapshotImpl(mutableMap)
+        }
+
+        fun ObjectOutputStream.writeStringArray(stringArray: Array<String>) {
+            writeInt(stringArray.size)
+            stringArray.forEach { writeUTF(it) }
+        }
+
+        fun ObjectOutputStream.writeAbiSnapshot(abiSnapshot: AbiSnapshot) {
+            //TODO(valtman) temp solution while packageProto is not fully support
+            writeInt(abiSnapshot.protos.size)
+            for (entry in abiSnapshot.protos) {
+                writeUTF(entry.key.asString())
+                val protoData = entry.value
+                when (protoData) {
+                    is ClassProtoData -> {
+                        writeBoolean(true) //TODO(valtman) until PackageProto doesn't work
+                        val nameResolver = protoData.nameResolver
+                        when (nameResolver) {
+                            is NameResolverImpl -> {
+                                writeMessageWithNameResolverImpl(protoData.proto, nameResolver)
+                            }
+                            is JvmNameResolver -> {
+                                writeMessageWithJvmNameResolver(protoData.proto, nameResolver)
+                            }
+                            else -> throw IllegalStateException("Can't store name resolver for class proto: ${nameResolver.javaClass}")
+                        }
+                    }
+                    is PackagePartProtoData -> {
+                        writeBoolean(false)
+                        writeUTF(protoData.packageFqName.asString())
+
+                        val nameResolver = protoData.nameResolver
+                        when (nameResolver) {
+                            is JvmNameResolver -> {
+                                writeMessageWithJvmNameResolver(protoData.proto, nameResolver)
+                            }
+                            is NameResolverImpl -> {
+                                writeMessageWithNameResolverImpl(protoData.proto, nameResolver)
+                            }
+                            else -> throw IllegalStateException("Can't store name resolver for package proto: ${nameResolver.javaClass}")
+                        }
+
+                    }
+                }
+            }
+        }
+
+        private fun ObjectOutputStream.writeMessageWithNameResolverImpl(
+            message: MessageLite,
+            nameResolver: NameResolverImpl
+        ) {
+            val stringTable = JvmStringTable()
+            repeat(nameResolver.strings.getStringCount()) {
+                stringTable.getStringIndex(nameResolver.getString(it))
+            }
+            repeat(nameResolver.qualifiedNames.qualifiedNameCount) {
+                stringTable.getQualifiedClassNameIndex(
+                    nameResolver.getQualifiedClassName(it),
+                    nameResolver.isLocalClassName(it)
+                )
+            }
+            val writeData = JvmProtoBufUtil.writeData(message, stringTable)
+            writeStringArray(writeData)
+            val size = nameResolver.strings.getStringCount()
+            writeInt(size)
+            repeat(size) {
+                val string = nameResolver.getString(it)
+                writeUTF(string)
+            }
+        }
+
+        private fun ObjectOutputStream.writeMessageWithJvmNameResolver(
+            message: MessageLite,
+            nameResolver: JvmNameResolver
+        ) {
+            val writeData = JvmProtoBufUtil.writeData(message, JvmStringTable(nameResolver))
+            writeStringArray(writeData)
+            writeStringArray(nameResolver.strings)
+        }
+
+        fun write(buildInfo: AbiSnapshot, file: File) {
+            ObjectOutputStream(FileOutputStream(file)).use {
+                it.writeAbiSnapshot(buildInfo)
+            }
+        }
+
+        fun read(file: File, reporter: BuildReporter): AbiSnapshot? {
+            if (!file.exists()) {
+                reporter.report { "jar snapshot $file is found for jar" }
+                return null
+            }
+
+            ObjectInputStream(FileInputStream(file)).use {
+                return it.readAbiSnapshot()
+            }
+        }
+    }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshotDiffService.kt

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.incremental.ChangesCollector.Companion.getNonPrivateMemberNames
+import org.jetbrains.kotlin.metadata.ProtoBuf.Visibility.PRIVATE
+import org.jetbrains.kotlin.metadata.deserialization.Flags
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME
+
+//TODO(valtman) Should be in gradle daemon.
+class AbiSnapshotDiffService() {
+
+    companion object {
+        //Store list of changed lookups
+        private val diffCache: MutableMap<Pair<AbiSnapshot, AbiSnapshot>, DirtyData> = mutableMapOf()
+
+        //TODO(valtman) move out from Kotlin daemon
+        fun compareJarsInternal(
+            oldSnapshot: AbiSnapshot, newSnapshot: AbiSnapshot,
+            caches: IncrementalCacheCommon
+        ) = diffCache.computeIfAbsent(Pair(oldSnapshot, newSnapshot)) { (snapshot, actual) -> doCompute(snapshot, actual, caches, emptyList()) }
+
+        fun inScope(fqName: FqName, scopes: Collection<String>) = scopes.any { scope -> fqName.toString().startsWith(scope) }
+
+        fun doCompute(snapshot: AbiSnapshot, actual: AbiSnapshot, caches: IncrementalCacheCommon, scopes: Collection<String>): DirtyData {
+
+            val dirtyFqNames = mutableListOf<FqName>()
+            val dirtyLookupSymbols = mutableListOf<LookupSymbol>()
+
+            for ((fqName, protoData) in snapshot.protos) {
+                if (!inScope(fqName, scopes)) continue
+                val newProtoData = actual.protos[fqName]
+                if (newProtoData == null) {
+                    val (fqNames, symbols) = addProtoInfo(protoData, fqName)
+                    dirtyFqNames.addAll(fqNames)
+                    dirtyLookupSymbols.addAll(symbols)
+                } else {
+                    if (protoData is ClassProtoData && newProtoData is ClassProtoData) {
+                        ProtoCompareGenerated(
+                            protoData.nameResolver, newProtoData.nameResolver,
+                            protoData.proto.typeTable, newProtoData.proto.typeTable
+                        )
+                        val diff = DifferenceCalculatorForClass(protoData, newProtoData).difference()
+
+                        if (diff.isClassAffected) {
+                            //TODO(valtman) get cache to mark dirty all subtypes if subclass affected
+//                            val fqNames = if (!diff.areSubclassesAffected) listOf(fqName) else withSubtypes(fqName, caches)
+                            dirtyFqNames.add(fqName)
+                            assert(!fqName.isRoot) { "$fqName is root" }
+
+                            val scope = fqName.parent().asString()
+                            val name = fqName.shortName().identifier
+                            dirtyLookupSymbols.add(LookupSymbol(name, scope))
+                        }
+                        for (member in diff.changedMembersNames) {
+                            //TODO(valtman) mark dirty symbols for subclasses
+                            val subtypeFqNames = withSubtypes(fqName, listOf(caches))
+                            dirtyFqNames.addAll(subtypeFqNames)
+
+                            for (subtypeFqName in subtypeFqNames) {
+                                dirtyLookupSymbols.add(LookupSymbol(member, subtypeFqName.asString()))
+                                dirtyLookupSymbols.add(LookupSymbol(SAM_LOOKUP_NAME.asString(), subtypeFqName.asString()))
+                            }
+                        }
+
+                    } else if (protoData is PackagePartProtoData && newProtoData is PackagePartProtoData) {
+                        val diff = DifferenceCalculatorForPackageFacade(protoData, newProtoData).difference()
+                        for (member in diff.changedMembersNames) {
+                            dirtyLookupSymbols.add(LookupSymbol(member, fqName.asString()))
+                        }
+                    } else {
+                        //TODO(valtman) is it a valid case
+                        throw IllegalStateException("package proto and class proto have the same fqName: $fqName")
+                    }
+                }
+            }
+
+//                fqNames.addAll(snapshot.protos.keys.removeAll(actual.protos.keys))
+            DirtyData(dirtyLookupSymbols, dirtyFqNames)
+            // .removeAll(actual.protos.keys)
+            val oldFqNames = snapshot.protos.keys
+            dirtyFqNames.addAll(actual.protos.keys.filter { !oldFqNames.contains(it) })
+            return DirtyData(dirtyLookupSymbols, dirtyFqNames)
+
+        }
+
+        //TODO(valtman) change to return type
+        private fun addProtoInfo(
+            protoData: ProtoData,
+            fqName: FqName,
+        ) : Pair<List<FqName>, List<LookupSymbol>>{
+            val fqNames = ArrayList<FqName>()
+            val symbols = ArrayList<LookupSymbol>()
+            when (protoData) {
+                is ClassProtoData -> {
+                    fqNames.add(fqName)
+                    symbols.addAll(protoData.getNonPrivateMemberNames().map { LookupSymbol(it, fqName.asString()) })
+                }
+                is PackagePartProtoData -> {
+                    symbols.addAll(
+                        protoData.proto.functionOrBuilderList.filterNot { Flags.VISIBILITY.get(it.flags) == PRIVATE }
+                            .map { LookupSymbol(protoData.nameResolver.getString(it.name), fqName.asString()) }.toSet()
+                    )
+
+                }
+            }
+            return Pair(fqNames, symbols)
+        }
+
+    }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt

@@ -95,6 +95,23 @@ data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
         }
 
         private fun ObjectInputStream.readDirtyData(): DirtyData {
+            val lookupSymbols = readLookups()
+            val dirtyClassesFqNames = readFqNames()
+
+            return DirtyData(lookupSymbols, dirtyClassesFqNames)
+        }
+
+        fun ObjectInputStream.readFqNames(): ArrayList<FqName> {
+            val dirtyClassesSize = readInt()
+            val dirtyClassesFqNames = ArrayList<FqName>(dirtyClassesSize)
+            repeat(dirtyClassesSize) {
+                val fqNameString = readUTF()
+                dirtyClassesFqNames.add(FqName(fqNameString))
+            }
+            return dirtyClassesFqNames
+        }
+
+        fun ObjectInputStream.readLookups(): ArrayList<LookupSymbol> {
             val lookupSymbolSize = readInt()
             val lookupSymbols = ArrayList<LookupSymbol>(lookupSymbolSize)
             repeat(lookupSymbolSize) {
@@ -102,30 +119,27 @@ data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
                 val scope = readUTF()
                 lookupSymbols.add(LookupSymbol(name = name, scope = scope))
             }
-
-            val dirtyClassesSize = readInt()
-            val dirtyClassesFqNames = ArrayList<FqName>(dirtyClassesSize)
-            repeat(dirtyClassesSize) {
-                val fqNameString = readUTF()
-                dirtyClassesFqNames.add(FqName(fqNameString))
-            }
-
-            return DirtyData(lookupSymbols, dirtyClassesFqNames)
+            return lookupSymbols
         }
 
         private fun ObjectOutputStream.writeDirtyData(dirtyData: DirtyData) {
-            val lookupSymbols = dirtyData.dirtyLookupSymbols
+            writeLookups(dirtyData.dirtyLookupSymbols)
+            writeFqNames(dirtyData.dirtyClassesFqNames)
+        }
+
+        fun ObjectOutputStream.writeFqNames(dirtyClassesFqNames: Collection<FqName> ) {
+            writeInt(dirtyClassesFqNames.size)
+            for (fqName in dirtyClassesFqNames) {
+                writeUTF(fqName.asString())
+            }
+        }
+
+        fun ObjectOutputStream.writeLookups(lookupSymbols: Collection<LookupSymbol>) {
             writeInt(lookupSymbols.size)
             for ((name, scope) in lookupSymbols) {
                 writeUTF(name)
                 writeUTF(scope)
             }
-
-            val dirtyClassesFqNames = dirtyData.dirtyClassesFqNames
-            writeInt(dirtyClassesFqNames.size)
-            for (fqName in dirtyClassesFqNames) {
-                writeUTF(fqName.asString())
-            }
         }
 
         internal const val MAX_DIFFS_ENTRIES: Int = 10

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt

@@ -16,14 +16,37 @@
 
 package org.jetbrains.kotlin.incremental
 
+import org.jetbrains.kotlin.incremental.AbiSnapshotImpl.Companion.readAbiSnapshot
+import org.jetbrains.kotlin.incremental.AbiSnapshotImpl.Companion.writeAbiSnapshot
 import java.io.*
 
-data class BuildInfo(val startTS: Long) : Serializable {
+data class BuildInfo(val startTS: Long, val dependencyToAbiSnapshot: Map<String, AbiSnapshot> = mapOf()) : Serializable {
     companion object {
+        private fun ObjectInputStream.readBuildInfo() : BuildInfo {
+            val ts = readLong()
+            val size = readInt()
+            val abiSnapshots = HashMap<String, AbiSnapshot>(size)
+            repeat(size) {
+                val identifier = readUTF()
+                val snapshot = readAbiSnapshot()
+                abiSnapshots.put(identifier, snapshot)
+            }
+            return BuildInfo(ts, abiSnapshots)
+        }
+
+        private fun ObjectOutputStream.writeBuildInfo(buildInfo: BuildInfo) {
+            writeLong(buildInfo.startTS)
+            writeInt(buildInfo.dependencyToAbiSnapshot.size)
+            for((identifier, abiSnapshot) in buildInfo.dependencyToAbiSnapshot) {
+                writeUTF(identifier)
+                writeAbiSnapshot(abiSnapshot)
+            }
+        }
+
         fun read(file: File): BuildInfo? =
             try {
                 ObjectInputStream(FileInputStream(file)).use {
-                    it.readObject() as BuildInfo
+                    it.readBuildInfo()
                 }
             } catch (e: Exception) {
                 null
@@ -31,7 +54,7 @@ data class BuildInfo(val startTS: Long) : Serializable {
 
         fun write(buildInfo: BuildInfo, file: File) {
             ObjectOutputStream(FileOutputStream(file)).use {
-                it.writeObject(buildInfo)
+                it.writeBuildInfo(buildInfo)
             }
         }
     }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -87,7 +87,7 @@ class IncrementalJvmCachesManager(
 
 class IncrementalJsCachesManager(
     cachesRootDir: File,
-    rootProjectDir: File,
+    rootProjectDir: File?,
     reporter: ICReporter,
     serializerProtocol: SerializerExtensionProtocol
 ) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, rootProjectDir, reporter) {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -22,9 +22,11 @@ import org.jetbrains.kotlin.build.report.BuildReporter
 import org.jetbrains.kotlin.build.report.metrics.BuildTime
 import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
 import org.jetbrains.kotlin.build.report.metrics.measure
+import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.compilerRunner.MessageCollectorToOutputItemsCollectorAdapter
 import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.compilerRunner.SimpleOutputItem
@@ -39,6 +41,7 @@ import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import java.io.File
 import java.io.IOException
 import java.util.*
+import kotlin.collections.HashMap
 
 abstract class IncrementalCompilerRunner<
         Args : CommonCompilerArguments,
@@ -56,8 +59,12 @@ abstract class IncrementalCompilerRunner<
     protected val cacheDirectory = File(workingDir, cacheDirName)
     private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
     protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
+    protected val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
     protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
 
+    //TODO(valtman) temporal measure to ensure quick disable, should be deleted after successful release
+    protected val withSnapshot: Boolean = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value.toBooleanLenient() ?: false
+
     protected abstract fun isICEnabled(): Boolean
     protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager
     protected abstract fun destinationDir(args: Args): File
@@ -84,6 +91,19 @@ abstract class IncrementalCompilerRunner<
         assert(isICEnabled()) { "Incremental compilation is not enabled" }
         var caches = createCacheManager(args, projectDir)
 
+        if (withSnapshot) {
+            reporter.report { "Incremental compilation with ABI snapshot enabled" }
+        }
+        //TODO if abi-snapshot is corrupted unable to rebuild. Should roll back to withSnapshot = false?
+        val classpathAbiSnapshot =
+            if (withSnapshot) {
+                reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) {
+                    setupJarDependencies(args, withSnapshot, reporter)
+                }
+            } else {
+                emptyMap()
+            }
+
         fun rebuild(reason: BuildAttribute): ExitCode {
             reporter.report { "Non-incremental compilation will be performed: $reason" }
             caches.close(false)
@@ -96,7 +116,8 @@ abstract class IncrementalCompilerRunner<
                 caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
             }
             val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
-            return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector)
+            return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withSnapshot,
+                                        classpathAbiSnapshot = classpathAbiSnapshot)
         }
 
         // If compilation has crashed or we failed to close caches we have to clear them
@@ -114,11 +135,36 @@ abstract class IncrementalCompilerRunner<
                 else -> providedChangedFiles
             }
 
-            val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector)
+
+            val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot)
 
             val exitCode = when (compilationMode) {
                 is CompilationMode.Incremental -> {
-                    compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector)
+                    if (withSnapshot) {
+                        val abiSnapshot = AbiSnapshotImpl.read(abiSnapshotFile, reporter)
+                        if (abiSnapshot != null) {
+                            compileIncrementally(
+                                args,
+                                caches,
+                                allSourceFiles,
+                                compilationMode,
+                                messageCollector,
+                                withSnapshot,
+                                abiSnapshot,
+                                classpathAbiSnapshot
+                            )
+                        } else {
+                            rebuild(BuildAttribute.NO_ABI_SNAPSHOT)
+                        }
+                    } else {
+                        compileIncrementally(
+                            args,
+                            caches,
+                            allSourceFiles,
+                            compilationMode,
+                            messageCollector,
+                            withSnapshot)
+                    }
                 }
                 is CompilationMode.Rebuild -> {
                     rebuild(compilationMode.reason)
@@ -173,28 +219,33 @@ abstract class IncrementalCompilerRunner<
         caches: CacheManager,
         changedFiles: ChangedFiles,
         args: Args,
-        messageCollector: MessageCollector
+        messageCollector: MessageCollector,
+        dependenciesAbiSnapshots: Map<String, AbiSnapshot>
     ): CompilationMode =
         when (changedFiles) {
-            is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector)
+            is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector, dependenciesAbiSnapshots)
             is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
             is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)")
         }
 
     private fun calculateSourcesToCompile(
-        caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector
+        caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector,
+        abiSnapshots: Map<String, AbiSnapshot>
     ): CompilationMode =
         reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
-            calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector)
+            calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, abiSnapshots)
         }
 
     protected abstract fun calculateSourcesToCompileImpl(
         caches: CacheManager,
         changedFiles: ChangedFiles.Known,
         args: Args,
-        messageCollector: MessageCollector
+        messageCollector: MessageCollector,
+        classpathAbiSnapshots: Map<String, AbiSnapshot>
     ): CompilationMode
 
+    protected open fun setupJarDependencies(args: Args, withSnapshot: Boolean, reporter: BuildReporter): Map<String, AbiSnapshot> = mapOf()
+
     protected fun initDirtyFiles(dirtyFiles: DirtyFilesContainer, changedFiles: ChangedFiles.Known) {
         dirtyFiles.add(changedFiles.modified, "was modified since last time")
         dirtyFiles.add(changedFiles.removed, "was removed since last time")
@@ -251,7 +302,10 @@ abstract class IncrementalCompilerRunner<
         caches: CacheManager,
         allKotlinSources: List<File>,
         compilationMode: CompilationMode,
-        originalMessageCollector: MessageCollector
+        originalMessageCollector: MessageCollector,
+        withSnapshot: Boolean,
+        abiSnapshot: AbiSnapshot = AbiSnapshotImpl(mutableMapOf()),
+        classpathAbiSnapshot: Map<String, AbiSnapshot> = HashMap()
     ): ExitCode {
         preBuildHook(args, compilationMode)
 
@@ -268,7 +322,7 @@ abstract class IncrementalCompilerRunner<
             }
         }
 
-        val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
+        val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), classpathAbiSnapshot)
         val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
         val buildDirtyFqNames = HashSet<FqName>()
         val allDirtySources = HashSet<File>()
@@ -283,6 +337,7 @@ abstract class IncrementalCompilerRunner<
 
             val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
             val expectActualTracker = ExpectActualTrackerImpl()
+            //TODO(valtman) sourceToCompile calculate based on abiSnapshot
             val (sourcesToCompile, removedKotlinSources) = dirtySources.partition(File::exists)
 
             allDirtySources.addAll(dirtySources)
@@ -327,9 +382,15 @@ abstract class IncrementalCompilerRunner<
                 caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
                 caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
                 caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
-                updateCaches(services, caches, generatedFiles, changesCollector)
+            updateCaches(services, caches, generatedFiles, changesCollector)
+
+}
+            if (compilationMode is CompilationMode.Rebuild) {
+                if (withSnapshot) {
+                    abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
+                }
+                break
             }
-            if (compilationMode is CompilationMode.Rebuild) break
 
             val (dirtyLookupSymbols, dirtyClassFqNames, forceRecompile) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
             val compiledInThisIterationSet = sourcesToCompile.toHashSet()
@@ -353,10 +414,25 @@ abstract class IncrementalCompilerRunner<
 
             buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
             buildDirtyFqNames.addAll(dirtyClassFqNames)
+
+            //update
+            if (withSnapshot) {
+                //TODO(valtman) check method/ kts class remove
+                changesCollector.protoDataRemoved().forEach { abiSnapshot.protos.remove(it) }
+                abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
+            }
         }
 
         if (exitCode == ExitCode.OK) {
-            BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
+            reporter.measure(BuildTime.STORE_BUILD_INFO) {
+                BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
+
+                //write abi snapshot
+                if (withSnapshot) {
+                    //TODO(valtman) check method/class remove
+                    AbiSnapshotImpl.write(abiSnapshot, abiSnapshotFile)
+                }
+            }
         }
         if (exitCode == ExitCode.OK && compilationMode is CompilationMode.Incremental) {
             buildDirtyLookupSymbols.addAll(additionalDirtyLookupSymbols())
@@ -407,12 +483,14 @@ abstract class IncrementalCompilerRunner<
             BuildDifference(currentBuildInfo.startTS, false, emptyDirtyData)
         }
 
+        //TODO(valtman) old history build should be restored in case of build fail
         BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
     }
 
     companion object {
         const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
         const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
+        const val ABI_SNAPSHOT_FILE_NAME = "abi-snapshot.bin"
     }
 
     private object EmptyCompilationCanceledStatus : CompilationCanceledStatus {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -75,7 +75,7 @@ inline fun <R> withJsIC(fn: () -> R): R {
 }
 
 class IncrementalJsCompilerRunner(
-    private val workingDir: File,
+    workingDir: File,
     reporter: BuildReporter,
     buildHistoryFile: File,
     private val modulesApiHistory: ModulesApiHistory,
@@ -91,7 +91,7 @@ class IncrementalJsCompilerRunner(
 
     override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
         val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
-        return IncrementalJsCachesManager(cacheDirectory, workingDir, reporter, serializerProtocol)
+        return IncrementalJsCachesManager(cacheDirectory, projectDir, reporter, serializerProtocol)
     }
 
     override fun destinationDir(args: K2JSCompilerArguments): File =
@@ -101,7 +101,8 @@ class IncrementalJsCompilerRunner(
         caches: IncrementalJsCachesManager,
         changedFiles: ChangedFiles.Known,
         args: K2JSCompilerArguments,
-        messageCollector: MessageCollector
+        messageCollector: MessageCollector,
+        classpathAbiSnapshots: Map<String, AbiSnapshot> //Ignore for now
     ): CompilationMode {
         val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
             ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
@@ -110,7 +111,12 @@ class IncrementalJsCompilerRunner(
         initDirtyFiles(dirtyFiles, changedFiles)
 
         val libs = (args.libraries ?: "").split(File.pathSeparator).map { File(it) }
-        val classpathChanges = getClasspathChanges(libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
+        //TODO(valtman) check for JS
+        val classpathChanges = getClasspathChanges(
+            libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter,
+            mapOf(), false, caches.platformCache,
+            caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct()
+        )
 
         @Suppress("UNUSED_VARIABLE") // for sealed when
         val unused = when (classpathChanges) {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -45,6 +45,7 @@ import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
 import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
 import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
+import org.jetbrains.kotlin.incremental.util.Either
 import org.jetbrains.kotlin.load.java.JavaClassesTracker
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
@@ -81,6 +82,7 @@ fun makeIncrementally(
             modulesApiHistory = EmptyModulesApiHistory,
             kotlinSourceFilesExtensions = kotlinExtensions
         )
+        //TODO set properly
         compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null)
     }
 }
@@ -125,7 +127,6 @@ class IncrementalJvmCompilerRunner(
     override fun isICEnabled(): Boolean =
         IncrementalCompilation.isEnabledForJvm()
 
-    //TODO
     override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
         IncrementalJvmCachesManager(cacheDirectory, projectDir, File(args.destination), reporter)
 
@@ -163,20 +164,43 @@ class IncrementalJvmCompilerRunner(
         caches: IncrementalJvmCachesManager,
         changedFiles: ChangedFiles.Known,
         args: K2JVMCompilerArguments,
-        messageCollector: MessageCollector
+        messageCollector: MessageCollector,
+        classpathAbiSnapshots: Map<String, AbiSnapshot>
     ): CompilationMode {
         return try {
-            calculateSourcesToCompileImpl(caches, changedFiles, args)
+            calculateSourcesToCompileImpl(caches, changedFiles, args, classpathAbiSnapshots)
         } finally {
             psiFileProvider.messageCollector.flush(messageCollector)
             psiFileProvider.messageCollector.clear()
         }
     }
 
+    //TODO can't use the same way as for build-history files because abi-snapshot for all dependencies should be stored into last-build
+    // and not only changed one
+    // (but possibly we dont need to read it all and may be it is possible to update only those who was changed)
+    override fun setupJarDependencies(args: K2JVMCompilerArguments, withSnapshot: Boolean, reporter: BuildReporter): Map<String, AbiSnapshot> {
+        //fill abiSnapshots
+        if (!withSnapshot) return emptyMap()
+        val abiSnapshots = HashMap<String, AbiSnapshot>()
+        args.classpathAsList
+            .filter { it.extension.equals("jar", ignoreCase = true) }
+            .forEach {
+                modulesApiHistory.abiSnapshot(it).let { result ->
+                    if (result is Either.Success<Set<File>>) {
+                        result.value.forEach { file ->
+                            AbiSnapshotImpl.read(file, reporter)?.also { abiSnapshot -> abiSnapshots[it.absolutePath] = abiSnapshot }
+                        }
+                    }
+                }
+            }
+        return abiSnapshots
+    }
+
     private fun calculateSourcesToCompileImpl(
         caches: IncrementalJvmCachesManager,
         changedFiles: ChangedFiles.Known,
-        args: K2JVMCompilerArguments
+        args: K2JVMCompilerArguments,
+        abiSnapshots: Map<String, AbiSnapshot> = HashMap()
     ): CompilationMode {
         val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
         initDirtyFiles(dirtyFiles, changedFiles)
@@ -185,7 +209,9 @@ class IncrementalJvmCompilerRunner(
         reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
 
         val classpathChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
-            getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
+            val scopes = caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct()
+            getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter,
+                                abiSnapshots, withSnapshot, caches.platformCache, scopes)
         }
 
         @Suppress("UNUSED_VARIABLE") // for sealed when

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt

@@ -19,7 +19,11 @@ internal fun getClasspathChanges(
     changedFiles: ChangedFiles.Known,
     lastBuildInfo: BuildInfo,
     modulesApiHistory: ModulesApiHistory,
-    reporter: BuildReporter
+    reporter: BuildReporter,
+    abiSnapshots: Map<String, AbiSnapshot>,
+    withSnapshot: Boolean,
+    caches: IncrementalCacheCommon,
+    scopes: Collection<String>
 ): ChangesEither {
     val classpathSet = HashSet<File>()
     for (file in classpath) {
@@ -40,52 +44,79 @@ internal fun getClasspathChanges(
 
     if (modifiedClasspath.isEmpty()) return ChangesEither.Known()
 
-    val lastBuildTS = lastBuildInfo.startTS
+    if (withSnapshot) {
+        fun analyzeJarFiles(): ChangesEither {
+            val symbols = HashSet<LookupSymbol>()
+            val fqNames = HashSet<FqName>()
 
-    val symbols = HashSet<LookupSymbol>()
-    val fqNames = HashSet<FqName>()
+            for ((module, abiSnapshot) in abiSnapshots) {
+                val actualAbiSnapshot = lastBuildInfo.dependencyToAbiSnapshot[module]
+                if (actualAbiSnapshot == null) {
 
-    val historyFilesEither =
-        reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) {
-            modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
-        }
-    val historyFiles = when (historyFilesEither) {
-        is Either.Success<Set<File>> -> historyFilesEither.value
-        is Either.Error -> {
-            reporter.report { "Could not find history files: ${historyFilesEither.reason}" }
-            return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND)
-        }
-    }
-
-    fun analyzeHistoryFiles(): ChangesEither {
-        for (historyFile in historyFiles) {
-            val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
-                ?: return run {
-                    reporter.report { "Could not read diffs from $historyFile" }
-                    ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ)
+                    reporter.report { "Some jar are removed from classpath $module" }
+                    return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_REMOVED_ENTRY)
                 }
-            val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
-            if (knownBuilds.isEmpty()) {
-                reporter.report { "No previously known builds for $historyFile" }
-                return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS)
+                val diffData = AbiSnapshotDiffService.doCompute(abiSnapshot, actualAbiSnapshot, caches, scopes)
+                symbols.addAll(diffData.dirtyLookupSymbols)
+                fqNames.addAll(diffData.dirtyClassesFqNames)
+
+            }
+            return ChangesEither.Known(symbols, fqNames)
+        }
+        return reporter.measure(BuildTime.IC_ANALYZE_JAR_FILES) {
+            analyzeJarFiles()
+        }
+    } else {
+        val lastBuildTS = lastBuildInfo.startTS
+
+        val symbols = HashSet<LookupSymbol>()
+        val fqNames = HashSet<FqName>()
+
+        val historyFilesEither =
+            reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) {
+                modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
             }
 
-            for (buildDiff in newBuilds) {
-                if (!buildDiff.isIncremental) {
-                    reporter.report { "Non-incremental build from dependency $historyFile" }
-                    return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP)
-
-                }
-                val dirtyData = buildDiff.dirtyData
-                symbols.addAll(dirtyData.dirtyLookupSymbols)
-                fqNames.addAll(dirtyData.dirtyClassesFqNames)
+        val historyFiles = when (historyFilesEither) {
+            is Either.Success<Set<File>> -> historyFilesEither.value
+            is Either.Error -> {
+                reporter.report { "Could not find history files: ${historyFilesEither.reason}" }
+                return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND)
             }
         }
 
-        return ChangesEither.Known(symbols, fqNames)
-    }
+        fun analyzeHistoryFiles(): ChangesEither {
+            for (historyFile in historyFiles) {
+                val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
+                    ?: return run {
+                        reporter.report { "Could not read diffs from $historyFile" }
+                        ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ)
+                    }
 
-    return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) {
-        analyzeHistoryFiles()
+                val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
+                if (knownBuilds.isEmpty()) {
+                    reporter.report { "No previously known builds for $historyFile" }
+                    return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS)
+                }
+
+
+                for (buildDiff in newBuilds) {
+                    if (!buildDiff.isIncremental) {
+                        reporter.report { "Non-incremental build from dependency $historyFile" }
+                        return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP)
+
+                    }
+                    val dirtyData = buildDiff.dirtyData
+                    symbols.addAll(dirtyData.dirtyLookupSymbols)
+                    fqNames.addAll(dirtyData.dirtyClassesFqNames)
+                }
+            }
+
+            return ChangesEither.Known(symbols, fqNames)
+        }
+
+        return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) {
+            analyzeHistoryFiles()
+        }
     }
 }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.incremental.multiproject
 
+import org.jetbrains.kotlin.incremental.IncrementalModuleEntry
 import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
 import org.jetbrains.kotlin.incremental.util.Either
 import java.io.File
@@ -14,11 +15,14 @@ import java.util.zip.ZipFile
 
 interface ModulesApiHistory {
     fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>>
+    fun abiSnapshot(jar: File): Either<Set<File>>
 }
 
 object EmptyModulesApiHistory : ModulesApiHistory {
     override fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>> =
         Either.Error("Multi-module IC is not configured")
+
+    override fun abiSnapshot(jar: File): Either<Set<File>> = Either.Error("Not supported")
 }
 
 abstract class ModulesApiHistoryBase(protected val modulesInfo: IncrementalModuleInfo) : ModulesApiHistory {
@@ -130,6 +134,14 @@ class ModulesApiHistoryJvm(modulesInfo: IncrementalModuleInfo) : ModulesApiHisto
 
         return Either.Success(result)
     }
+
+    override fun abiSnapshot(jar: File): Either<Set<File>> {
+        val abiSnapshot = modulesInfo.jarToModule[jar]?.abiSnapshot ?: modulesInfo.jarToAbiSnapshot[jar]
+        return if (abiSnapshot != null)
+            Either.Success(setOf(abiSnapshot))
+        else
+            Either.Error("Failed to find abi snapshot for file ${jar.absolutePath}")
+    }
 }
 
 class ModulesApiHistoryJs(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) {
@@ -141,6 +153,11 @@ class ModulesApiHistoryJs(modulesInfo: IncrementalModuleInfo) : ModulesApiHistor
             else -> Either.Error("No module is found for jar $jar")
         }
     }
+
+    override fun abiSnapshot(jar: File): Either<Set<File>> {
+        return modulesInfo.jarToModule[jar]?.abiSnapshot?.let { Either.Success(setOf(it)) } ?: Either.Error("Failed to find snapshot for file ${jar.absolutePath}")
+
+    }
 }
 
 class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) {
@@ -158,7 +175,15 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH
         if (!isInProjectBuildDir(jar)) return Either.Error("Non-project jar is modified $jar")
 
         val jarPath = Paths.get(jar.absolutePath)
-        return getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath))
+        return getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath), IncrementalModuleEntry::buildHistoryFile)
+    }
+
+    override fun abiSnapshot(jar: File): Either<Set<File>> {
+        val jarPath = Paths.get(jar.absolutePath)
+        return when (val result = getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath), IncrementalModuleEntry::abiSnapshot)) {
+            is Either.Success -> Either.Success(result.value)
+            is Either.Error -> Either.Error(result.reason)
+        }
     }
 
     override fun getBuildHistoryForDir(file: File): Either<Set<File>> {
@@ -175,7 +200,7 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH
             }
         }
 
-        return getHistoryForModuleNames(file.toPath(), moduleNames)
+        return getHistoryForModuleNames(file.toPath(), moduleNames, IncrementalModuleEntry::buildHistoryFile)
     }
 
     private fun getPossibleModuleNamesFromJar(path: Path): Collection<String> {
@@ -205,13 +230,13 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH
         return path.listFiles().filter { it.name.endsWith(".kotlin_module", ignoreCase = true) }.map { it.nameWithoutExtension }
     }
 
-    private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable<String>): Either<Set<File>> {
+    private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable<String>, fileLocation: (IncrementalModuleEntry) -> File): Either<Set<File>> {
         val possibleModules =
             moduleNames.flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() }
         val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(path) }
         if (modules.isEmpty()) return Either.Error("Unknown module for $path (candidates: ${possibleModules.joinToString()})")
 
-        val result = modules.mapTo(HashSet()) { it.buildHistoryFile }
+        val result = modules.mapTo(HashSet()) { fileLocation(it) }
         return Either.Success(result)
     }
 }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt

@@ -30,9 +30,10 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest<Args : CommonCom
     private val nameToModules = mutableMapOf<String, MutableSet<IncrementalModuleEntry>>()
     private val jarToClassListFile = mutableMapOf<File, File>()
     private val jarToModule = mutableMapOf<File, IncrementalModuleEntry>()
+    private val jarToAbiSnapshot = mutableMapOf<File, File>()
 
     protected val incrementalModuleInfo: IncrementalModuleInfo by lazy {
-        IncrementalModuleInfo(workingDir, workingDir, dirToModule, nameToModules, jarToClassListFile, jarToModule)
+        IncrementalModuleInfo(workingDir, workingDir, dirToModule, nameToModules, jarToClassListFile, jarToModule, jarToAbiSnapshot)
     }
 
     protected abstract val modulesApiHistory: ApiHistory
@@ -97,8 +98,9 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest<Args : CommonCom
         val moduleBuildDir = File(outDir, moduleName)
         val moduleCacheDir = File(cacheDir, moduleName)
         val moduleBuildHistoryFile = buildHistoryFile(moduleCacheDir)
+        val abiSnapshotFile = abiSnapshotFile(moduleCacheDir)
 
-        val moduleEntry = IncrementalModuleEntry(workingDir.absolutePath, moduleName, outDir, moduleBuildHistoryFile)
+        val moduleEntry = IncrementalModuleEntry(workingDir.absolutePath, moduleName, outDir, moduleBuildHistoryFile, abiSnapshotFile)
 
         dirToModule[moduleBuildDir] = moduleEntry
         nameToModules.getOrPut(moduleName) { mutableSetOf() }.add(moduleEntry)

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt

@@ -177,6 +177,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
         @JvmStatic
         protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin")
 
+        @JvmStatic
+        protected fun abiSnapshotFile(cacheDir: File): File = File(cacheDir, IncrementalCompilerRunner.ABI_SNAPSHOT_FILE_NAME)
+
         private const val ARGUMENTS_FILE_NAME = "args.txt"
 
         private fun parseAdditionalArgs(testDir: File): List<String> {

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt

@@ -26,9 +26,11 @@ class ModulesApiHistoryAndroidTest {
     private lateinit var appRoot: File
     private lateinit var appKotlinDestination: File
     private lateinit var appHistory: File
+    private lateinit var appAbiSnapshot: File
     private lateinit var libRoot: File
     private lateinit var libKotlinDestination: File
     private lateinit var libHistory: File
+    private lateinit var libAbiSnapshot: File
 
     private lateinit var androidHistory: ModulesApiHistoryAndroid
 
@@ -38,8 +40,9 @@ class ModulesApiHistoryAndroidTest {
 
         appRoot = projectRoot.resolve("app")
         appHistory = appRoot.resolve("build/tmp/kotlin/app_history.bin")
+        appAbiSnapshot = appRoot.resolve("build/tmp/kotlin/app_abi_snapshot.bin")
         appKotlinDestination = appRoot.resolve("build/tmp/kotlin-classes").apply { mkdirs() }
-        val appEntry = IncrementalModuleEntry(":app", "app", appRoot.resolve("build"), appHistory)
+        val appEntry = IncrementalModuleEntry(":app", "app", appRoot.resolve("build"), appHistory, appAbiSnapshot)
         appRoot.resolve("build/intermediates/classes/meta-inf/").apply {
             mkdirs()
             resolve("app.kotlin_module").createNewFile()
@@ -47,8 +50,9 @@ class ModulesApiHistoryAndroidTest {
 
         libRoot = projectRoot.resolve("lib")
         libHistory = libRoot.resolve("lib/build/tmp/kotlin/lib_history.bin")
+        libAbiSnapshot = libRoot.resolve("lib/build/tmp/kotlin/lib_abi_snapshot.bin")
         libKotlinDestination = libRoot.resolve("build/tmp/kotlin-classes").apply { mkdirs() }
-        val libEntry = IncrementalModuleEntry(":lib", "lib", libRoot.resolve("build"), libHistory)
+        val libEntry = IncrementalModuleEntry(":lib", "lib", libRoot.resolve("build"), libHistory, libAbiSnapshot)
         libRoot.resolve("build/intermediates/classes/meta-inf/").apply {
             mkdirs()
             resolve("lib.kotlin_module").createNewFile()
@@ -60,7 +64,8 @@ class ModulesApiHistoryAndroidTest {
             dirToModule = mapOf(appKotlinDestination to appEntry, libKotlinDestination to libEntry),
             nameToModules = mapOf("app" to setOf(appEntry), "lib" to setOf(libEntry)),
             jarToClassListFile = mapOf(),
-            jarToModule = mapOf()
+            jarToModule = mapOf(),
+            jarToAbiSnapshot = mapOf()
         )
 
         androidHistory = ModulesApiHistoryAndroid(info)

core/metadata/src/org/jetbrains/kotlin/metadata/deserialization/NameResolverImpl.kt

@@ -10,8 +10,8 @@ import org.jetbrains.kotlin.metadata.ProtoBuf.QualifiedNameTable.QualifiedName
 import java.util.*
 
 class NameResolverImpl(
-    private val strings: ProtoBuf.StringTable,
-    private val qualifiedNames: ProtoBuf.QualifiedNameTable
+    val strings: ProtoBuf.StringTable,
+    val qualifiedNames: ProtoBuf.QualifiedNameTable
 ) : NameResolver {
     override fun getString(index: Int): String = strings.getString(index)
 

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt

@@ -251,6 +251,7 @@ abstract class BaseGradleIT {
         val useFir: Boolean = false,
         val customEnvironmentVariables: Map<String, String> = mapOf(),
         val dryRun: Boolean = false,
+        val abiSnapshot: Boolean = false,
     )
 
     enum class ConfigurationCacheProblems {
@@ -953,6 +954,9 @@ Finished executing task ':$taskName'|
             if (options.dryRun) {
                 add("--dry-run")
             }
+            if (options.abiSnapshot) {
+                add("-Dkotlin.incremental.classpath.snapshot.enabled=true")
+            }
 
             add("-Dorg.gradle.unsafe.configuration-cache=${options.configurationCache}")
             add("-Dorg.gradle.unsafe.configuration-cache-problems=${options.configurationCacheProblems.name.toLowerCase()}")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationBaseIT.kt

@@ -27,6 +27,14 @@ abstract class IncrementalCompilationBaseIT : BaseGradleIT() {
     protected fun doTest(
         modifyProject: (Project) -> Unit,
         expectedAffectedFileNames: Collection<String>,
+    ) {
+        doTest(defaultBuildOptions(), modifyProject, expectedAffectedFileNames)
+    }
+
+    protected fun doTest(
+        options: BuildOptions,
+        modifyProject: (Project) -> Unit,
+        expectedAffectedFileNames: Collection<String>,
     ) {
         val project = defaultProject()
         project.build("build") {
@@ -35,7 +43,7 @@ abstract class IncrementalCompilationBaseIT : BaseGradleIT() {
 
         modifyProject(project)
 
-        project.build("build") {
+        project.build("build", options = options) {
             assertSuccessful()
             val expectedAffectedFiles = project.projectDir.getFilesByNames(*expectedAffectedFileNames.toTypedArray())
             val expectedAffectedFileRelativePaths = project.relativize(expectedAffectedFiles)

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt

@@ -189,6 +189,30 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation
         )
     }
 
+    @Test
+    fun testAddNewMethodToLib() {
+        doTest(
+            options = defaultBuildOptions().copy(abiSnapshot = true),
+            { project ->
+                val aKt = project.projectDir.getFileByName("A.kt")
+                aKt.writeText(
+                    """
+package bar
+
+open class A {
+    fun a() {}
+    fun newA() {}
+}
+"""
+                )
+            },
+            //TODO for abi-snapshot "BB.kt" should not be recompiled
+            expectedAffectedFileNames = listOf(
+                "A.kt", "B.kt", "AA.kt", "BB.kt", "AAA.kt"
+            )
+        )
+    }
+
     @Test
     fun testLibClassBecameFinal() {
         // TODO: fix fir IC and remove
@@ -229,6 +253,8 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation
         project.projectFile("BarDummy.kt").modify {
             it.replace("class BarDummy", "open class BarDummy")
         }
+
+        //don't need to recompile app classes because lib's proto stays the same
         project.build("build") {
             assertSuccessful()
             val affectedSources = project.projectDir.allKotlinFiles()
@@ -244,9 +270,67 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation
         }
     }
 
+    @Test
+    fun testCleanBuildLibForAbiSnapshot() {
+        val options = defaultBuildOptions().copy(abiSnapshot = true)
+        val project = defaultProject()
+
+        project.build("build", options = options) {
+            assertSuccessful()
+        }
+
+        project.build(":lib:clean", options = options) {
+            assertSuccessful()
+        }
+
+        // Change file so Gradle won't skip :app:compile
+        project.projectFile("BarDummy.kt").modify {
+            it.replace("class BarDummy", "open class BarDummy")
+        }
+
+        //don't need to recompile app classes because lib's proto stays the same
+        project.build("build", options = options) {
+            val affectedSources = project.projectDir.allKotlinFiles()
+            val relativePaths = project.relativize(affectedSources)
+            assertCompiledKotlinSources(relativePaths)
+        }
+
+        val aaKt = project.projectFile("AA.kt")
+        aaKt.modify { "$it " }
+        project.build("build", options = options) {
+            assertSuccessful()
+            assertCompiledKotlinSources(project.relativize(aaKt))
+        }
+    }
+
     protected abstract val additionalLibDependencies: String
     protected abstract val compileKotlinTaskName: String
 
+    @Test
+    fun testAddMethodToLibForAbiSnapshot() {
+        val project = defaultProject()
+        val options = defaultBuildOptions().copy(abiSnapshot = true)
+
+        project.build("build", options = options) {
+            assertSuccessful()
+        }
+
+        // Change file so Gradle won't skip :app:compile
+        project.projectFile("BarDummy.kt").modify {
+            it.replace("class BarDummy", "open class BarDummy")
+        }
+
+        val barDummyClassFile = project.projectFile("BarDummy.kt")
+        barDummyClassFile.modify { "$it { fun m() = 42}" }
+
+        project.build("build", options = options) {
+            assertSuccessful()
+            val relativePaths = project.relativize(barDummyClassFile)
+            assertCompiledKotlinSources(relativePaths)
+        }
+
+    }
+
     @Test
     fun testAddDependencyToLib() {
         val project = defaultProject()

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.gradle.logging.kotlinDebug
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
 import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget
 import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
 import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
 import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider
@@ -212,6 +213,7 @@ internal open class GradleCompilerRunner(
             val nameToModules = HashMap<String, HashSet<IncrementalModuleEntry>>()
             val jarToClassListFile = HashMap<File, File>()
             val jarToModule = HashMap<File, IncrementalModuleEntry>()
+            val jarToAbiSnapshot = HashMap<File, File>()
 
             val multiplatformProjectTasks = mutableMapOf<Project, MutableSet<String>>()
 
@@ -229,7 +231,8 @@ internal open class GradleCompilerRunner(
                         project.path,
                         task.moduleName.get(),
                         project.buildDir,
-                        task.buildHistoryFile.get().asFile
+                        task.buildHistoryFile.get().asFile,
+                        task.abiSnapshotFile.get().asFile
                     )
                     dirToModule[task.destinationDir] = module
                     task.javaOutputDir.orNull?.asFile?.let { dirToModule[it] = module }
@@ -240,6 +243,15 @@ internal open class GradleCompilerRunner(
                             jarToModule[it] = module
                         }
                     }
+//                    for (target in task.targets) {
+//                        if (target is KotlinWithJavaTarget<*>) {
+//                            val jar = project.tasks.getByName(target.artifactsTaskName) as Jar
+//                            jarToClassListFile[jar.archivePathCompatible.canonicalFile] = target.defaultArtifactClassesListFile.get()
+//                            //configure abiSnapshot mapping for jars
+//                            jarToAbiSnapshot[jar.archivePathCompatible.canonicalFile] =
+//                                target.buildDir.get().file(task.abiSnapshotRelativePath).asFile
+//                        }
+//                    }
                 } else if (task is InspectClassesForMultiModuleIC) {
                     jarToClassListFile[File(task.archivePath.get())] = task.classesListFile
                 }
@@ -260,10 +272,19 @@ internal open class GradleCompilerRunner(
                             project.path,
                             kotlinTask.moduleName.get(),
                             project.buildDir,
-                            kotlinTask.buildHistoryFile.get().asFile
+                            kotlinTask.buildHistoryFile.get().asFile,
+                            kotlinTask.abiSnapshotFile.get().asFile
                         )
                         val jarTask = project.tasks.findByName(target.artifactsTaskName) as? AbstractArchiveTask ?: continue
                         jarToModule[jarTask.archivePathCompatible.canonicalFile] = module
+                        if (target is KotlinWithJavaTarget<*>) {
+                            val jar = project.tasks.getByName(target.artifactsTaskName) as Jar
+                            jarToClassListFile[jar.archivePathCompatible.canonicalFile] = target.defaultArtifactClassesListFile.get()
+                            //configure abiSnapshot mapping for jars
+                            jarToAbiSnapshot[jar.archivePathCompatible.canonicalFile] =
+                                target.buildDir.get().file(kotlinTask.abiSnapshotRelativePath).get().asFile
+                        }
+
                     }
                 }
             }
@@ -274,7 +295,8 @@ internal open class GradleCompilerRunner(
                 dirToModule = dirToModule,
                 nameToModules = nameToModules,
                 jarToClassListFile = jarToClassListFile,
-                jarToModule = jarToModule
+                jarToModule = jarToModule,
+                jarToAbiSnapshot = jarToAbiSnapshot
             ).also {
                 cachedGradle = WeakReference(gradle)
                 cachedModulesInfo = it

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin
+
+import org.gradle.api.services.BuildService
+import org.gradle.api.services.BuildServiceParameters
+import org.jetbrains.kotlin.build.report.BuildReporter
+import org.jetbrains.kotlin.incremental.*
+
+abstract class AbiSnapshotDiffService() : BuildService<AbiSnapshotDiffService.Parameters> {
+    abstract class Parameters : BuildServiceParameters {
+        abstract val caches: IncrementalCachesManager<*>
+        abstract val reporter: BuildReporter
+        abstract val sourceFilesExtensions: List<String>
+    }
+
+    val caches: IncrementalCachesManager<*> = parameters.caches
+    val reporter: BuildReporter = parameters.reporter
+    val sourceFilesExtensions: List<String> = parameters.sourceFilesExtensions
+
+    companion object {
+//        //Store list of changed lookups
+//        val diffCache: MutableMap<Pair<AbiSnapshot, AbiSnapshot>, DirtyFilesContainer> = mutableMapOf()
+//        @TestOnly
+//        fun compareJarsInternal(caches: IncrementalCachesManager<*>, reporter: ICReporter, sourceFilesExtensions: List<String>,
+//                                snapshot: AbiSnapshot, newJar: AbiSnapshot) =
+//            diffCache.computeIfAbsent(Pair(snapshot, newJar)) { (snapshotJar, actualJar) ->
+//                DirtyFilesContainer(caches, reporter, sourceFilesExtensions)
+//                    .also {
+//                        it.addByDirtyClasses(snapshotJar.fqNames.minus(actualJar.fqNames))
+//                        it.addByDirtyClasses(actualJar.fqNames.minus(snapshotJar.fqNames))
+//                        it.addByDirtySymbols(snapshotJar.symbols.minus(actualJar.symbols))
+//                        it.addByDirtySymbols(actualJar.symbols.minus(snapshotJar.symbols))
+//                    }
+//            }
+    }
+
+//    @Synchronized
+//    fun compareJarsInternal(snapshot: AbiSnapshot, newJar: AbiSnapshot): DirtyFilesContainer {
+//        return diffCache.computeIfAbsent(Pair(snapshot, newJar)) { (snapshotJar, actualJar) ->
+//            DirtyFilesContainer(caches, reporter, sourceFilesExtensions)
+//                .also {
+//                    it.addByDirtyClasses(snapshotJar.fqNames.minus(actualJar.fqNames))
+//                    it.addByDirtyClasses(actualJar.fqNames.minus(snapshotJar.fqNames))
+//                    it.addByDirtySymbols(snapshotJar.symbols.minus(actualJar.symbols))
+//                    it.addByDirtySymbols(actualJar.symbols.minus(snapshotJar.symbols))
+//                }
+//
+//        }
+//    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/jvm/KotlinWithJavaTarget.kt

@@ -9,6 +9,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp
 import org.gradle.api.NamedDomainObjectContainer
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Dependency
+import org.gradle.api.file.Directory
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.provider.Provider
 import org.gradle.jvm.tasks.Jar
@@ -52,6 +53,8 @@ open class KotlinWithJavaTarget<KotlinOptionsType : KotlinCommonOptions>(
             val jarTask = project.tasks.getByName(artifactsTaskName) as Jar
             it.file("${sanitizeFileName(jarTask.archiveFileName.get())}-classes.txt").asFile
         }
+
+    internal val buildDir: Provider<Directory> = layout.buildDirectory.dir(KOTLIN_BUILD_DIR_NAME)
 }
 
 private fun sanitizeFileName(candidate: String): String = candidate.filter { it.isLetterOrDigit() }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -54,6 +54,7 @@ import org.jetbrains.kotlin.gradle.utils.*
 import org.jetbrains.kotlin.gradle.utils.isParentOf
 import org.jetbrains.kotlin.gradle.utils.pathsAsStringRelativeTo
 import org.jetbrains.kotlin.incremental.ChangedFiles
+import org.jetbrains.kotlin.incremental.IncrementalCompilerRunner
 import org.jetbrains.kotlin.library.impl.isKotlinLibrary
 import org.jetbrains.kotlin.utils.JsLibraryUtils
 import java.io.File
@@ -274,6 +275,16 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
     @get:Input
     internal val moduleName: Property<String> = objects.property(String::class.java)
 
+    @get:Internal
+    val abiSnapshotFile
+        get() = taskBuildDirectory.file(IncrementalCompilerRunner.ABI_SNAPSHOT_FILE_NAME)
+
+    @get:Input
+    val abiSnapshotRelativePath: Property<String> = objects.property(String::class.java).value(
+        //TODO update to support any jar changes
+        "$name/${IncrementalCompilerRunner.ABI_SNAPSHOT_FILE_NAME}"
+    )
+
     @get:Internal
     internal val friendSourceSets = objects.listProperty(String::class.java)
 
@@ -300,34 +311,36 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> : AbstractKotl
 
     @TaskAction
     fun execute(inputs: IncrementalTaskInputs) {
-        systemPropertiesService.get().startIntercept()
-        CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value = "true"
+        metrics.measure(BuildTime.GRADLE_TASK_ACTION) {
+            systemPropertiesService.get().startIntercept()
+            CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value = "true"
 
-        // If task throws exception, but its outputs are changed during execution,
-        // then Gradle forces next build to be non-incremental (see Gradle's DefaultTaskArtifactStateRepository#persistNewOutputs)
-        // To prevent this, we backup outputs before incremental build and restore when exception is thrown
-        val outputsBackup: TaskOutputsBackup? =
-            if (isIncrementalCompilationEnabled() && inputs.isIncremental)
-                metrics.measure(BuildTime.BACKUP_OUTPUT) {
-                    TaskOutputsBackup(allOutputFiles())
-                }
-            else null
+            // If task throws exception, but its outputs are changed during execution,
+            // then Gradle forces next build to be non-incremental (see Gradle's DefaultTaskArtifactStateRepository#persistNewOutputs)
+            // To prevent this, we backup outputs before incremental build and restore when exception is thrown
+            val outputsBackup: TaskOutputsBackup? =
+                if (isIncrementalCompilationEnabled() && inputs.isIncremental)
+                    metrics.measure(BuildTime.BACKUP_OUTPUT) {
+                        TaskOutputsBackup(allOutputFiles())
+                    }
+                else null
 
-        if (!isIncrementalCompilationEnabled()) {
-            clearLocalState("IC is disabled")
-        } else if (!inputs.isIncremental) {
-            clearLocalState("Task cannot run incrementally")
-        }
-
-        try {
-            executeImpl(inputs)
-        } catch (t: Throwable) {
-            if (outputsBackup != null) {
-                metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
-                    outputsBackup.restoreOutputs()
-                }
+            if (!isIncrementalCompilationEnabled()) {
+                clearLocalState("IC is disabled")
+            } else if (!inputs.isIncremental) {
+                clearLocalState("Task cannot run incrementally")
+            }
+
+            try {
+                executeImpl(inputs)
+            } catch (t: Throwable) {
+                if (outputsBackup != null) {
+                    metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
+                        outputsBackup.restoreOutputs()
+                    }
+                }
+                throw t
             }
-            throw t
         }
     }