From f6b428f2719c3cb9debe2364aadc25f7f544681f Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Tue, 20 Oct 2020 13:14:11 +0300 Subject: [PATCH] KT-34862 use abi snapshot instead of build history files Use jar snapshot instead build history file to avoid using time stamps and support remote gradle cache --- .../build/report/metrics/BuildAttribute.kt | 1 + .../kotlin/build/report/metrics/BuildTime.kt | 5 + .../kotlin/incremental/ChangesCollector.kt | 82 ++++++--- .../kotlin/incremental/IncrementalJvmCache.kt | 4 +- .../incremental/IncrementalModuleInfo.kt | 7 +- .../kotlin/incremental/LookupStorage.kt | 2 +- .../jetbrains/kotlin/incremental/buildUtil.kt | 17 +- .../kotlin/incremental/storage/LookupMap.kt | 2 +- .../incremental/storage/externalizers.kt | 38 +++- .../kotlin/incremental/storage/values.kt | 24 ++- .../jetbrains/kotlin/cli/common/Properties.kt | 3 +- .../kotlin/daemon/common/DaemonParams.kt | 2 + .../kotlin/incremental/AbiSnapshot.kt | 165 ++++++++++++++++++ .../incremental/AbiSnapshotDiffService.kt | 115 ++++++++++++ .../kotlin/incremental/BuildDiffsStorage.kt | 46 +++-- .../jetbrains/kotlin/incremental/BuildInfo.kt | 29 ++- .../incremental/IncrementalCachesManager.kt | 2 +- .../incremental/IncrementalCompilerRunner.kt | 104 +++++++++-- .../IncrementalJsCompilerRunner.kt | 14 +- .../IncrementalJvmCompilerRunner.kt | 36 +++- .../incremental/changesDetectionUtils.kt | 109 +++++++----- .../multiproject/ModulesApiHistory.kt | 33 +++- ...ncrementaMultiModulelCompilerRunnerTest.kt | 6 +- ...stractIncrementalCompilerRunnerTestBase.kt | 3 + .../ModulesApiHistoryAndroidTest.kt | 11 +- .../deserialization/NameResolverImpl.kt | 4 +- .../jetbrains/kotlin/gradle/BaseGradleIT.kt | 4 + .../gradle/IncrementalCompilationBaseIT.kt | 10 +- .../IncrementalCompilationMultiProjectIT.kt | 84 +++++++++ .../GradleKotlinCompilerRunner.kt | 28 ++- .../gradle/plugin/AbiSnapshotDiffService.kt | 54 ++++++ .../targets/jvm/KotlinWithJavaTarget.kt | 3 + .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 63 ++++--- 33 files changed, 942 insertions(+), 168 deletions(-) create mode 100644 compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt create mode 100644 compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshotDiffService.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt index 31793ed66d4..1b48a7563c7 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt @@ -17,6 +17,7 @@ enum class BuildAttributeKind : Serializable { enum class BuildAttribute(val kind: BuildAttributeKind) : Serializable { NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON), + NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON), CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON), UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON), JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON), diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index cd7b22996c7..4e37017bb2b 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -9,6 +9,7 @@ import java.io.Serializable @Suppress("Reformat") enum class BuildTime(val parent: BuildTime? = null) : Serializable { + GRADLE_TASK_ACTION, GRADLE_TASK, CLEAR_OUTPUT(GRADLE_TASK), BACKUP_OUTPUT(GRADLE_TASK), @@ -20,6 +21,10 @@ enum class BuildTime(val parent: BuildTime? = null) : Serializable { NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILER), NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILER), INCREMENTAL_COMPILATION(RUN_COMPILER), + STORE_BUILD_INFO(INCREMENTAL_COMPILATION), + JAR_SNAPSHOT(INCREMENTAL_COMPILATION), + SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT), + IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT), IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION), IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET), IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES), diff --git a/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt b/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt index 90499179685..1f8f103e81d 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt @@ -30,6 +30,28 @@ class ChangesCollector { private val changedMembers = hashMapOf>() private val areSubclassesAffected = hashMapOf() + //TODO for test only: ProtoData or ProtoBuf + private val storage = hashMapOf() + private val removed = ArrayList() + + //TODO change to immutable map + fun protoDataChanges(): Map = storage + fun protoDataRemoved(): List = removed + + companion object { + fun T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List) = + members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet() + + fun ClassProtoData.getNonPrivateMemberNames(): Set { + return proto.getNonPrivateNames( + nameResolver, + ProtoBuf.Class::getConstructorList, + ProtoBuf.Class::getFunctionList, + ProtoBuf.Class::getPropertyList + ) + proto.enumEntryList.map { nameResolver.getString(it.name) } + } + } + fun changes(): List { val changes = arrayListOf() @@ -57,7 +79,7 @@ class ChangesCollector { } private fun MutableMap>.getSet(key: T) = - getOrPut(key) { HashSet() } + getOrPut(key) { HashSet() } private fun collectChangedMember(scope: FqName, name: String) { changedMembers.getSet(scope).add(name) @@ -79,11 +101,35 @@ class ChangesCollector { } } - fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?, collectAllMembersForNewClass: Boolean = false) { + fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?, collectAllMembersForNewClass: Boolean = false, packageProtoKey: String? = null) { if (oldData == null && newData == null) { throw IllegalStateException("Old and new value are null") } + if (newData != null) { + when (newData) { + is ClassProtoData -> { + val fqName = newData.nameResolver.getClassId(newData.proto.fqName).asSingleFqName() + storage[fqName] = newData + } + is PackagePartProtoData -> { + //TODO fqName is not unique. It's package and can be present in both java and kotlin + val fqName = newData.packageFqName + storage[packageProtoKey?.let { FqName(it) } ?: fqName] = newData + } + } + } else { + when (oldData) { + is ClassProtoData -> { + removed.add(oldData.nameResolver.getClassId(oldData.proto.fqName).asSingleFqName()) + } + is PackagePartProtoData -> { + //TODO fqName is not unique. It's package and can be present in both java and kotlin + removed.add(packageProtoKey?.let { FqName(it) } ?: oldData.packageFqName) + } + } + } + if (oldData == null) { newData!!.collectAll(isRemoved = false, isAdded = true, collectAllMembersForNewClass = collectAllMembersForNewClass) return @@ -125,8 +171,8 @@ class ChangesCollector { } } - private fun T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List): Set = - members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet() + fun T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List) = + members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet() //TODO remember all sealed parent classes private fun ProtoData.collectAll(isRemoved: Boolean, isAdded: Boolean, collectAllMembersForNewClass: Boolean = false) = @@ -137,16 +183,15 @@ class ChangesCollector { private fun PackagePartProtoData.collectAllFromPackage(isRemoved: Boolean) { val memberNames = - proto.getNonPrivateNames( - nameResolver, - ProtoBuf.Package::getFunctionList, - ProtoBuf.Package::getPropertyList - ) + proto.getNonPrivateNames( + nameResolver, + ProtoBuf.Package::getFunctionList, + ProtoBuf.Package::getPropertyList + ) if (isRemoved) { collectRemovedMembers(packageFqName, memberNames) - } - else { + } else { collectChangedMembers(packageFqName, memberNames) } } @@ -161,8 +206,7 @@ class ChangesCollector { val collectMember = if (isRemoved) this@ChangesCollector::collectRemovedMember else this@ChangesCollector::collectChangedMember collectMember(classFqName.parent(), classFqName.shortName().asString()) memberNames.forEach { collectMember(classFqName, it) } - } - else { + } else { if (!isRemoved && collectAllMembersForNewClass) { val memberNames = getNonPrivateMemberNames() memberNames.forEach { this@ChangesCollector.collectChangedMember(classFqName, it) } @@ -189,15 +233,6 @@ class ChangesCollector { addChangedParents(fqName, changedParentsFqNames) } - private fun ClassProtoData.getNonPrivateMemberNames(): Set { - return proto.getNonPrivateNames( - nameResolver, - ProtoBuf.Class::getConstructorList, - ProtoBuf.Class::getFunctionList, - ProtoBuf.Class::getPropertyList - ) + proto.enumEntryList.map { nameResolver.getString(it.name) } - } - fun collectMemberIfValueWasChanged(scope: FqName, name: String, oldValue: Any?, newValue: Any?) { if (oldValue == null && newValue == null) { throw IllegalStateException("Old and new value are null for $scope#$name") @@ -205,8 +240,7 @@ class ChangesCollector { if (oldValue != null && newValue == null) { collectRemovedMember(scope, name) - } - else if (oldValue != newValue) { + } else if (oldValue != newValue) { collectChangedMember(scope, name) } } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt index 9437b03279a..2bab98fe8a7 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt @@ -185,7 +185,7 @@ open class IncrementalJvmCache( sourceToClassesMap.add(source, jvmClassName) val (proto, nameResolver) = serializedJavaClass.toProtoData() addToClassStorage(proto, nameResolver, source) - +// collector.addJavaProto(ClassProtoData(proto, nameResolver)) dirtyOutputClassesMap.notDirty(jvmClassName) } @@ -308,7 +308,7 @@ open class IncrementalJvmCache( storage[key] = newData val packageFqName = kotlinClass.className.packageFqName - changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName)) + changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName), packageProtoKey = key) } operator fun contains(className: JvmClassName): Boolean = diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalModuleInfo.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalModuleInfo.kt index 30ee30d35e1..7941a5396a1 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalModuleInfo.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalModuleInfo.kt @@ -12,7 +12,8 @@ data class IncrementalModuleEntry( private val projectPath: String, val name: String, val buildDir: File, - val buildHistoryFile: File + val buildHistoryFile: File, + val abiSnapshot: File ) : Serializable { companion object { private const val serialVersionUID = 0L @@ -26,7 +27,9 @@ class IncrementalModuleInfo( val nameToModules: Map>, val jarToClassListFile: Map, // only for js and mpp - val jarToModule: Map + val jarToModule: Map, + //for JVM only + val jarToAbiSnapshot: Map ) : Serializable { companion object { private const val serialVersionUID = 1L diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index a54d678debc..6287871aeb5 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -41,7 +41,7 @@ open class LookupStorage( private val countersFile = "counters".storageFile private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter)) private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter)) - private val lookupMap = registerMap(LookupMap("lookups".storageFile)) + val lookupMap = registerMap(LookupMap("lookups".storageFile)) @Volatile private var size: Int = 0 diff --git a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt index cf215b95ab5..b9f905d6225 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt @@ -255,22 +255,23 @@ fun withSubtypes( typeFqName: FqName, caches: Iterable ): Set { - val types = LinkedHashSet(listOf(typeFqName)) - val subtypes = hashSetOf() + val typesToProccess = LinkedHashSet(listOf(typeFqName)) + val proccessedTypes = hashSetOf() - while (types.isNotEmpty()) { - val iterator = types.iterator() + + while (typesToProccess.isNotEmpty()) { + val iterator = typesToProccess.iterator() val unprocessedType = iterator.next() iterator.remove() caches.asSequence() .flatMap { it.getSubtypesOf(unprocessedType) } - .filter { it !in subtypes } - .forEach { types.add(it) } + .filter { it !in proccessedTypes } + .forEach { typesToProccess.add(it) } - subtypes.add(unprocessedType) + proccessedTypes.add(unprocessedType) } - return subtypes + return proccessedTypes } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt index 1747b123fca..5b606336506 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt @@ -18,7 +18,7 @@ package org.jetbrains.kotlin.incremental.storage import java.io.File -internal class LookupMap(storage: File) : BasicMap>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) { +class LookupMap(storage: File) : BasicMap>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) { override fun dumpKey(key: LookupSymbolKey): String = key.toString() override fun dumpValue(value: Collection): String = value.toString() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt index 795de067365..378549fce61 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt @@ -21,23 +21,49 @@ import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.IOUtil import com.intellij.util.io.KeyDescriptor +import org.jetbrains.kotlin.cli.common.CompilerSystemProperties +import org.jetbrains.kotlin.cli.common.toBooleanLenient import java.io.DataInput import java.io.DataInputStream import java.io.DataOutput import java.io.File import java.util.* +/** + * Storage versioning: + * 0 - only name and value hashes are saved + * 1 - name and scope are saved + */ object LookupSymbolKeyDescriptor : KeyDescriptor { override fun read(input: DataInput): LookupSymbolKey { - val first = input.readInt() - val second = input.readInt() - - return LookupSymbolKey(first, second) + val version = input.readByte() + return when (version.toInt()) { + 0 -> { + val name = input.readUTF() + val scope = input.readUTF() + LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope) + } + 1 -> { + val first = input.readInt() + val second = input.readInt() + LookupSymbolKey(first, second, "", "") + } + else -> throw RuntimeException("Unknown version of LookupSymbolKeyDescriptor=${version}") + } } + private val storeFullFqName = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value.toBooleanLenient() ?: false + override fun save(output: DataOutput, value: LookupSymbolKey) { - output.writeInt(value.nameHash) - output.writeInt(value.scopeHash) + if (storeFullFqName) { + output.writeByte(0) + output.writeUTF(value.name) + output.writeUTF(value.scope) + } else { + output.writeByte(1) + output.writeInt(value.nameHash) + output.writeInt(value.scopeHash) + } } override fun getHashCode(value: LookupSymbolKey): Int = value.hashCode() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/values.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/values.kt index 32a51179e3c..9fb871dee08 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/values.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/values.kt @@ -16,8 +16,8 @@ package org.jetbrains.kotlin.incremental.storage -data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int) : Comparable { - constructor(name: String, scope: String) : this(name.hashCode(), scope.hashCode()) +data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int, val name:String, val scope:String) : Comparable { + constructor(name: String, scope: String) : this(name.hashCode(), scope.hashCode(), name, scope) override fun compareTo(other: LookupSymbolKey): Int { val nameCmp = nameHash.compareTo(other.nameHash) @@ -26,6 +26,26 @@ data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int) : Comparable) diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt index 3dece2fbed8..50a2532bc24 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/Properties.kt @@ -35,6 +35,7 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY("kotlin.daemon.socket.connect.interval"), KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY("kotlin.environment.keepalive"), COMPILE_DAEMON_CUSTOM_RUN_FILES_PATH_FOR_TESTS("kotlin.daemon.custom.run.files.path.for.tests"), + COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS("kotlin.incremental.classpath.snapshot.enabled"), KOTLIN_COLORS_ENABLED_PROPERTY("kotlin.colors.enabled"), KOTLIN_STAT_ENABLED_PROPERTY("kotlin.plugin.stat.enabled"), @@ -48,7 +49,7 @@ enum class CompilerSystemProperties(val property: String, val alwaysDirectAccess USER_HOME("user.home", alwaysDirectAccess = true), JAVA_VERSION("java.specification.version", alwaysDirectAccess = true), JAVA_HOME("java.home", alwaysDirectAccess = true), - JAVA_CLASS_PATH("java.class.path", alwaysDirectAccess = true), + JAVA_CLASS_PATH("java.class.path", alwaysDirectAccess = true) ; private fun getProperFunction(custom: T?, default: T): T { diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt index a2bf98aa99a..b5f1881dee6 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt @@ -312,6 +312,8 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions, if (inheritAdditionalProperties) { CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_DAEMON_LOG_PATH_PROPERTY.property}=\"$it\"") } CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.property}") } + //Temporary solution to test abi snapshot + CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value?.let { opts.jvmParams.add("D${CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.property}") } } if (opts.jvmParams.none { it.matches(jvmAssertArgsRegex) }) { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt new file mode 100644 index 00000000000..ea6d2919e75 --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshot.kt @@ -0,0 +1,165 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl +import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmNameResolver +import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil +import org.jetbrains.kotlin.metadata.jvm.serialization.JvmStringTable +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.protobuf.MessageLite +import java.io.* + +interface AbiSnapshot { + val protos: MutableMap +} + + + +class AbiSnapshotImpl(override val protos: MutableMap) : AbiSnapshot { + + companion object { + fun ObjectInputStream.readStringArray(): Array { + val size = readInt() + val stringArray = arrayOfNulls(size) + repeat(size) { + stringArray[it] = readUTF() + } + + return stringArray.requireNoNulls() + } + + + fun ObjectInputStream.readAbiSnapshot(): AbiSnapshotImpl { + // Format: + // numRecords: Int + // record { + // fqName + // isClassProtoData + // *for packageClassData - packageFqName + // protodata via JvmProtoBufUtil + // string array with size + // } + val size = readInt() + val mutableMap = hashMapOf() + repeat(size) { + val fqNameString = readUTF() + val isClassProtoData = readBoolean() + if (isClassProtoData) { + val fqName = FqName(fqNameString) + val bytes = readStringArray() + val strings = readStringArray() + val (nameResolver, classProto) = JvmProtoBufUtil.readClassDataFrom(bytes, strings) + mutableMap[fqName] = ClassProtoData(classProto, nameResolver) + } else { + val fqName = FqName(fqNameString) + val packageFqName = FqName(readUTF()) + val bytes = readStringArray() + val strings = readStringArray() + val (nameResolver, proto) = JvmProtoBufUtil.readPackageDataFrom(bytes, strings) + mutableMap[fqName] = PackagePartProtoData(proto, nameResolver, packageFqName) + } + } + return AbiSnapshotImpl(mutableMap) + } + + fun ObjectOutputStream.writeStringArray(stringArray: Array) { + writeInt(stringArray.size) + stringArray.forEach { writeUTF(it) } + } + + fun ObjectOutputStream.writeAbiSnapshot(abiSnapshot: AbiSnapshot) { + //TODO(valtman) temp solution while packageProto is not fully support + writeInt(abiSnapshot.protos.size) + for (entry in abiSnapshot.protos) { + writeUTF(entry.key.asString()) + val protoData = entry.value + when (protoData) { + is ClassProtoData -> { + writeBoolean(true) //TODO(valtman) until PackageProto doesn't work + val nameResolver = protoData.nameResolver + when (nameResolver) { + is NameResolverImpl -> { + writeMessageWithNameResolverImpl(protoData.proto, nameResolver) + } + is JvmNameResolver -> { + writeMessageWithJvmNameResolver(protoData.proto, nameResolver) + } + else -> throw IllegalStateException("Can't store name resolver for class proto: ${nameResolver.javaClass}") + } + } + is PackagePartProtoData -> { + writeBoolean(false) + writeUTF(protoData.packageFqName.asString()) + + val nameResolver = protoData.nameResolver + when (nameResolver) { + is JvmNameResolver -> { + writeMessageWithJvmNameResolver(protoData.proto, nameResolver) + } + is NameResolverImpl -> { + writeMessageWithNameResolverImpl(protoData.proto, nameResolver) + } + else -> throw IllegalStateException("Can't store name resolver for package proto: ${nameResolver.javaClass}") + } + + } + } + } + } + + private fun ObjectOutputStream.writeMessageWithNameResolverImpl( + message: MessageLite, + nameResolver: NameResolverImpl + ) { + val stringTable = JvmStringTable() + repeat(nameResolver.strings.getStringCount()) { + stringTable.getStringIndex(nameResolver.getString(it)) + } + repeat(nameResolver.qualifiedNames.qualifiedNameCount) { + stringTable.getQualifiedClassNameIndex( + nameResolver.getQualifiedClassName(it), + nameResolver.isLocalClassName(it) + ) + } + val writeData = JvmProtoBufUtil.writeData(message, stringTable) + writeStringArray(writeData) + val size = nameResolver.strings.getStringCount() + writeInt(size) + repeat(size) { + val string = nameResolver.getString(it) + writeUTF(string) + } + } + + private fun ObjectOutputStream.writeMessageWithJvmNameResolver( + message: MessageLite, + nameResolver: JvmNameResolver + ) { + val writeData = JvmProtoBufUtil.writeData(message, JvmStringTable(nameResolver)) + writeStringArray(writeData) + writeStringArray(nameResolver.strings) + } + + fun write(buildInfo: AbiSnapshot, file: File) { + ObjectOutputStream(FileOutputStream(file)).use { + it.writeAbiSnapshot(buildInfo) + } + } + + fun read(file: File, reporter: BuildReporter): AbiSnapshot? { + if (!file.exists()) { + reporter.report { "jar snapshot $file is found for jar" } + return null + } + + ObjectInputStream(FileInputStream(file)).use { + return it.readAbiSnapshot() + } + } + } +} \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshotDiffService.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshotDiffService.kt new file mode 100644 index 00000000000..43fdca1f403 --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/AbiSnapshotDiffService.kt @@ -0,0 +1,115 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.incremental.ChangesCollector.Companion.getNonPrivateMemberNames +import org.jetbrains.kotlin.metadata.ProtoBuf.Visibility.PRIVATE +import org.jetbrains.kotlin.metadata.deserialization.Flags +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME + +//TODO(valtman) Should be in gradle daemon. +class AbiSnapshotDiffService() { + + companion object { + //Store list of changed lookups + private val diffCache: MutableMap, DirtyData> = mutableMapOf() + + //TODO(valtman) move out from Kotlin daemon + fun compareJarsInternal( + oldSnapshot: AbiSnapshot, newSnapshot: AbiSnapshot, + caches: IncrementalCacheCommon + ) = diffCache.computeIfAbsent(Pair(oldSnapshot, newSnapshot)) { (snapshot, actual) -> doCompute(snapshot, actual, caches, emptyList()) } + + fun inScope(fqName: FqName, scopes: Collection) = scopes.any { scope -> fqName.toString().startsWith(scope) } + + fun doCompute(snapshot: AbiSnapshot, actual: AbiSnapshot, caches: IncrementalCacheCommon, scopes: Collection): DirtyData { + + val dirtyFqNames = mutableListOf() + val dirtyLookupSymbols = mutableListOf() + + for ((fqName, protoData) in snapshot.protos) { + if (!inScope(fqName, scopes)) continue + val newProtoData = actual.protos[fqName] + if (newProtoData == null) { + val (fqNames, symbols) = addProtoInfo(protoData, fqName) + dirtyFqNames.addAll(fqNames) + dirtyLookupSymbols.addAll(symbols) + } else { + if (protoData is ClassProtoData && newProtoData is ClassProtoData) { + ProtoCompareGenerated( + protoData.nameResolver, newProtoData.nameResolver, + protoData.proto.typeTable, newProtoData.proto.typeTable + ) + val diff = DifferenceCalculatorForClass(protoData, newProtoData).difference() + + if (diff.isClassAffected) { + //TODO(valtman) get cache to mark dirty all subtypes if subclass affected +// val fqNames = if (!diff.areSubclassesAffected) listOf(fqName) else withSubtypes(fqName, caches) + dirtyFqNames.add(fqName) + assert(!fqName.isRoot) { "$fqName is root" } + + val scope = fqName.parent().asString() + val name = fqName.shortName().identifier + dirtyLookupSymbols.add(LookupSymbol(name, scope)) + } + for (member in diff.changedMembersNames) { + //TODO(valtman) mark dirty symbols for subclasses + val subtypeFqNames = withSubtypes(fqName, listOf(caches)) + dirtyFqNames.addAll(subtypeFqNames) + + for (subtypeFqName in subtypeFqNames) { + dirtyLookupSymbols.add(LookupSymbol(member, subtypeFqName.asString())) + dirtyLookupSymbols.add(LookupSymbol(SAM_LOOKUP_NAME.asString(), subtypeFqName.asString())) + } + } + + } else if (protoData is PackagePartProtoData && newProtoData is PackagePartProtoData) { + val diff = DifferenceCalculatorForPackageFacade(protoData, newProtoData).difference() + for (member in diff.changedMembersNames) { + dirtyLookupSymbols.add(LookupSymbol(member, fqName.asString())) + } + } else { + //TODO(valtman) is it a valid case + throw IllegalStateException("package proto and class proto have the same fqName: $fqName") + } + } + } + +// fqNames.addAll(snapshot.protos.keys.removeAll(actual.protos.keys)) + DirtyData(dirtyLookupSymbols, dirtyFqNames) + // .removeAll(actual.protos.keys) + val oldFqNames = snapshot.protos.keys + dirtyFqNames.addAll(actual.protos.keys.filter { !oldFqNames.contains(it) }) + return DirtyData(dirtyLookupSymbols, dirtyFqNames) + + } + + //TODO(valtman) change to return type + private fun addProtoInfo( + protoData: ProtoData, + fqName: FqName, + ) : Pair, List>{ + val fqNames = ArrayList() + val symbols = ArrayList() + when (protoData) { + is ClassProtoData -> { + fqNames.add(fqName) + symbols.addAll(protoData.getNonPrivateMemberNames().map { LookupSymbol(it, fqName.asString()) }) + } + is PackagePartProtoData -> { + symbols.addAll( + protoData.proto.functionOrBuilderList.filterNot { Flags.VISIBILITY.get(it.flags) == PRIVATE } + .map { LookupSymbol(protoData.nameResolver.getString(it.name), fqName.asString()) }.toSet() + ) + + } + } + return Pair(fqNames, symbols) + } + + } +} \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt index 0e45f818432..1bba7480770 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildDiffsStorage.kt @@ -95,6 +95,23 @@ data class BuildDiffsStorage(val buildDiffs: List) { } private fun ObjectInputStream.readDirtyData(): DirtyData { + val lookupSymbols = readLookups() + val dirtyClassesFqNames = readFqNames() + + return DirtyData(lookupSymbols, dirtyClassesFqNames) + } + + fun ObjectInputStream.readFqNames(): ArrayList { + val dirtyClassesSize = readInt() + val dirtyClassesFqNames = ArrayList(dirtyClassesSize) + repeat(dirtyClassesSize) { + val fqNameString = readUTF() + dirtyClassesFqNames.add(FqName(fqNameString)) + } + return dirtyClassesFqNames + } + + fun ObjectInputStream.readLookups(): ArrayList { val lookupSymbolSize = readInt() val lookupSymbols = ArrayList(lookupSymbolSize) repeat(lookupSymbolSize) { @@ -102,30 +119,27 @@ data class BuildDiffsStorage(val buildDiffs: List) { val scope = readUTF() lookupSymbols.add(LookupSymbol(name = name, scope = scope)) } - - val dirtyClassesSize = readInt() - val dirtyClassesFqNames = ArrayList(dirtyClassesSize) - repeat(dirtyClassesSize) { - val fqNameString = readUTF() - dirtyClassesFqNames.add(FqName(fqNameString)) - } - - return DirtyData(lookupSymbols, dirtyClassesFqNames) + return lookupSymbols } private fun ObjectOutputStream.writeDirtyData(dirtyData: DirtyData) { - val lookupSymbols = dirtyData.dirtyLookupSymbols + writeLookups(dirtyData.dirtyLookupSymbols) + writeFqNames(dirtyData.dirtyClassesFqNames) + } + + fun ObjectOutputStream.writeFqNames(dirtyClassesFqNames: Collection ) { + writeInt(dirtyClassesFqNames.size) + for (fqName in dirtyClassesFqNames) { + writeUTF(fqName.asString()) + } + } + + fun ObjectOutputStream.writeLookups(lookupSymbols: Collection) { writeInt(lookupSymbols.size) for ((name, scope) in lookupSymbols) { writeUTF(name) writeUTF(scope) } - - val dirtyClassesFqNames = dirtyData.dirtyClassesFqNames - writeInt(dirtyClassesFqNames.size) - for (fqName in dirtyClassesFqNames) { - writeUTF(fqName.asString()) - } } internal const val MAX_DIFFS_ENTRIES: Int = 10 diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt index 62f85156091..856759ad0bd 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt @@ -16,14 +16,37 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.incremental.AbiSnapshotImpl.Companion.readAbiSnapshot +import org.jetbrains.kotlin.incremental.AbiSnapshotImpl.Companion.writeAbiSnapshot import java.io.* -data class BuildInfo(val startTS: Long) : Serializable { +data class BuildInfo(val startTS: Long, val dependencyToAbiSnapshot: Map = mapOf()) : Serializable { companion object { + private fun ObjectInputStream.readBuildInfo() : BuildInfo { + val ts = readLong() + val size = readInt() + val abiSnapshots = HashMap(size) + repeat(size) { + val identifier = readUTF() + val snapshot = readAbiSnapshot() + abiSnapshots.put(identifier, snapshot) + } + return BuildInfo(ts, abiSnapshots) + } + + private fun ObjectOutputStream.writeBuildInfo(buildInfo: BuildInfo) { + writeLong(buildInfo.startTS) + writeInt(buildInfo.dependencyToAbiSnapshot.size) + for((identifier, abiSnapshot) in buildInfo.dependencyToAbiSnapshot) { + writeUTF(identifier) + writeAbiSnapshot(abiSnapshot) + } + } + fun read(file: File): BuildInfo? = try { ObjectInputStream(FileInputStream(file)).use { - it.readObject() as BuildInfo + it.readBuildInfo() } } catch (e: Exception) { null @@ -31,7 +54,7 @@ data class BuildInfo(val startTS: Long) : Serializable { fun write(buildInfo: BuildInfo, file: File) { ObjectOutputStream(FileOutputStream(file)).use { - it.writeObject(buildInfo) + it.writeBuildInfo(buildInfo) } } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 27dd6adbbcf..73f5f8c56f9 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -87,7 +87,7 @@ class IncrementalJvmCachesManager( class IncrementalJsCachesManager( cachesRootDir: File, - rootProjectDir: File, + rootProjectDir: File?, reporter: ICReporter, serializerProtocol: SerializerExtensionProtocol ) : IncrementalCachesManager(cachesRootDir, rootProjectDir, reporter) { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 31588994616..3779a91a5e7 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -22,9 +22,11 @@ import org.jetbrains.kotlin.build.report.BuildReporter import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.build.report.metrics.measure +import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.compilerRunner.MessageCollectorToOutputItemsCollectorAdapter import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl import org.jetbrains.kotlin.compilerRunner.SimpleOutputItem @@ -39,6 +41,7 @@ import org.jetbrains.kotlin.progress.CompilationCanceledStatus import java.io.File import java.io.IOException import java.util.* +import kotlin.collections.HashMap abstract class IncrementalCompilerRunner< Args : CommonCompilerArguments, @@ -56,8 +59,12 @@ abstract class IncrementalCompilerRunner< protected val cacheDirectory = File(workingDir, cacheDirName) private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME) + protected val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) protected open val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS + //TODO(valtman) temporal measure to ensure quick disable, should be deleted after successful release + protected val withSnapshot: Boolean = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value.toBooleanLenient() ?: false + protected abstract fun isICEnabled(): Boolean protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager protected abstract fun destinationDir(args: Args): File @@ -84,6 +91,19 @@ abstract class IncrementalCompilerRunner< assert(isICEnabled()) { "Incremental compilation is not enabled" } var caches = createCacheManager(args, projectDir) + if (withSnapshot) { + reporter.report { "Incremental compilation with ABI snapshot enabled" } + } + //TODO if abi-snapshot is corrupted unable to rebuild. Should roll back to withSnapshot = false? + val classpathAbiSnapshot = + if (withSnapshot) { + reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) { + setupJarDependencies(args, withSnapshot, reporter) + } + } else { + emptyMap() + } + fun rebuild(reason: BuildAttribute): ExitCode { reporter.report { "Non-incremental compilation will be performed: $reason" } caches.close(false) @@ -96,7 +116,8 @@ abstract class IncrementalCompilerRunner< caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) } val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } - return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector) + return compileIncrementally(args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withSnapshot, + classpathAbiSnapshot = classpathAbiSnapshot) } // If compilation has crashed or we failed to close caches we have to clear them @@ -114,11 +135,36 @@ abstract class IncrementalCompilerRunner< else -> providedChangedFiles } - val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector) + + val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot) val exitCode = when (compilationMode) { is CompilationMode.Incremental -> { - compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector) + if (withSnapshot) { + val abiSnapshot = AbiSnapshotImpl.read(abiSnapshotFile, reporter) + if (abiSnapshot != null) { + compileIncrementally( + args, + caches, + allSourceFiles, + compilationMode, + messageCollector, + withSnapshot, + abiSnapshot, + classpathAbiSnapshot + ) + } else { + rebuild(BuildAttribute.NO_ABI_SNAPSHOT) + } + } else { + compileIncrementally( + args, + caches, + allSourceFiles, + compilationMode, + messageCollector, + withSnapshot) + } } is CompilationMode.Rebuild -> { rebuild(compilationMode.reason) @@ -173,28 +219,33 @@ abstract class IncrementalCompilerRunner< caches: CacheManager, changedFiles: ChangedFiles, args: Args, - messageCollector: MessageCollector + messageCollector: MessageCollector, + dependenciesAbiSnapshots: Map ): CompilationMode = when (changedFiles) { - is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector) + is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector, dependenciesAbiSnapshots) is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)") } private fun calculateSourcesToCompile( - caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector + caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector, + abiSnapshots: Map ): CompilationMode = reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) { - calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector) + calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, abiSnapshots) } protected abstract fun calculateSourcesToCompileImpl( caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, - messageCollector: MessageCollector + messageCollector: MessageCollector, + classpathAbiSnapshots: Map ): CompilationMode + protected open fun setupJarDependencies(args: Args, withSnapshot: Boolean, reporter: BuildReporter): Map = mapOf() + protected fun initDirtyFiles(dirtyFiles: DirtyFilesContainer, changedFiles: ChangedFiles.Known) { dirtyFiles.add(changedFiles.modified, "was modified since last time") dirtyFiles.add(changedFiles.removed, "was removed since last time") @@ -251,7 +302,10 @@ abstract class IncrementalCompilerRunner< caches: CacheManager, allKotlinSources: List, compilationMode: CompilationMode, - originalMessageCollector: MessageCollector + originalMessageCollector: MessageCollector, + withSnapshot: Boolean, + abiSnapshot: AbiSnapshot = AbiSnapshotImpl(mutableMapOf()), + classpathAbiSnapshot: Map = HashMap() ): ExitCode { preBuildHook(args, compilationMode) @@ -268,7 +322,7 @@ abstract class IncrementalCompilerRunner< } } - val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis()) + val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), classpathAbiSnapshot) val buildDirtyLookupSymbols = HashSet() val buildDirtyFqNames = HashSet() val allDirtySources = HashSet() @@ -283,6 +337,7 @@ abstract class IncrementalCompilerRunner< val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) val expectActualTracker = ExpectActualTrackerImpl() + //TODO(valtman) sourceToCompile calculate based on abiSnapshot val (sourcesToCompile, removedKotlinSources) = dirtySources.partition(File::exists) allDirtySources.addAll(dirtySources) @@ -327,9 +382,15 @@ abstract class IncrementalCompilerRunner< caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker) caches.inputsCache.registerOutputForSourceFiles(generatedFiles) caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) - updateCaches(services, caches, generatedFiles, changesCollector) + updateCaches(services, caches, generatedFiles, changesCollector) + +} + if (compilationMode is CompilationMode.Rebuild) { + if (withSnapshot) { + abiSnapshot.protos.putAll(changesCollector.protoDataChanges()) + } + break } - if (compilationMode is CompilationMode.Rebuild) break val (dirtyLookupSymbols, dirtyClassFqNames, forceRecompile) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter) val compiledInThisIterationSet = sourcesToCompile.toHashSet() @@ -353,10 +414,25 @@ abstract class IncrementalCompilerRunner< buildDirtyLookupSymbols.addAll(dirtyLookupSymbols) buildDirtyFqNames.addAll(dirtyClassFqNames) + + //update + if (withSnapshot) { + //TODO(valtman) check method/ kts class remove + changesCollector.protoDataRemoved().forEach { abiSnapshot.protos.remove(it) } + abiSnapshot.protos.putAll(changesCollector.protoDataChanges()) + } } if (exitCode == ExitCode.OK) { - BuildInfo.write(currentBuildInfo, lastBuildInfoFile) + reporter.measure(BuildTime.STORE_BUILD_INFO) { + BuildInfo.write(currentBuildInfo, lastBuildInfoFile) + + //write abi snapshot + if (withSnapshot) { + //TODO(valtman) check method/class remove + AbiSnapshotImpl.write(abiSnapshot, abiSnapshotFile) + } + } } if (exitCode == ExitCode.OK && compilationMode is CompilationMode.Incremental) { buildDirtyLookupSymbols.addAll(additionalDirtyLookupSymbols()) @@ -407,12 +483,14 @@ abstract class IncrementalCompilerRunner< BuildDifference(currentBuildInfo.startTS, false, emptyDirtyData) } + //TODO(valtman) old history build should be restored in case of build fail BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter) } companion object { const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt" const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin" + const val ABI_SNAPSHOT_FILE_NAME = "abi-snapshot.bin" } private object EmptyCompilationCanceledStatus : CompilationCanceledStatus { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 4838ee93cda..fde92d34fe6 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -75,7 +75,7 @@ inline fun withJsIC(fn: () -> R): R { } class IncrementalJsCompilerRunner( - private val workingDir: File, + workingDir: File, reporter: BuildReporter, buildHistoryFile: File, private val modulesApiHistory: ModulesApiHistory, @@ -91,7 +91,7 @@ class IncrementalJsCompilerRunner( override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager { val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol - return IncrementalJsCachesManager(cacheDirectory, workingDir, reporter, serializerProtocol) + return IncrementalJsCachesManager(cacheDirectory, projectDir, reporter, serializerProtocol) } override fun destinationDir(args: K2JSCompilerArguments): File = @@ -101,7 +101,8 @@ class IncrementalJsCompilerRunner( caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments, - messageCollector: MessageCollector + messageCollector: MessageCollector, + classpathAbiSnapshots: Map //Ignore for now ): CompilationMode { val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) @@ -110,7 +111,12 @@ class IncrementalJsCompilerRunner( initDirtyFiles(dirtyFiles, changedFiles) val libs = (args.libraries ?: "").split(File.pathSeparator).map { File(it) } - val classpathChanges = getClasspathChanges(libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter) + //TODO(valtman) check for JS + val classpathChanges = getClasspathChanges( + libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter, + mapOf(), false, caches.platformCache, + caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct() + ) @Suppress("UNUSED_VARIABLE") // for sealed when val unused = when (classpathChanges) { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index fa06ba59f6c..cf10554526d 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -45,6 +45,7 @@ import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector +import org.jetbrains.kotlin.incremental.util.Either import org.jetbrains.kotlin.load.java.JavaClassesTracker import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents @@ -81,6 +82,7 @@ fun makeIncrementally( modulesApiHistory = EmptyModulesApiHistory, kotlinSourceFilesExtensions = kotlinExtensions ) + //TODO set properly compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null) } } @@ -125,7 +127,6 @@ class IncrementalJvmCompilerRunner( override fun isICEnabled(): Boolean = IncrementalCompilation.isEnabledForJvm() - //TODO override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager = IncrementalJvmCachesManager(cacheDirectory, projectDir, File(args.destination), reporter) @@ -163,20 +164,43 @@ class IncrementalJvmCompilerRunner( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments, - messageCollector: MessageCollector + messageCollector: MessageCollector, + classpathAbiSnapshots: Map ): CompilationMode { return try { - calculateSourcesToCompileImpl(caches, changedFiles, args) + calculateSourcesToCompileImpl(caches, changedFiles, args, classpathAbiSnapshots) } finally { psiFileProvider.messageCollector.flush(messageCollector) psiFileProvider.messageCollector.clear() } } + //TODO can't use the same way as for build-history files because abi-snapshot for all dependencies should be stored into last-build + // and not only changed one + // (but possibly we dont need to read it all and may be it is possible to update only those who was changed) + override fun setupJarDependencies(args: K2JVMCompilerArguments, withSnapshot: Boolean, reporter: BuildReporter): Map { + //fill abiSnapshots + if (!withSnapshot) return emptyMap() + val abiSnapshots = HashMap() + args.classpathAsList + .filter { it.extension.equals("jar", ignoreCase = true) } + .forEach { + modulesApiHistory.abiSnapshot(it).let { result -> + if (result is Either.Success>) { + result.value.forEach { file -> + AbiSnapshotImpl.read(file, reporter)?.also { abiSnapshot -> abiSnapshots[it.absolutePath] = abiSnapshot } + } + } + } + } + return abiSnapshots + } + private fun calculateSourcesToCompileImpl( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, - args: K2JVMCompilerArguments + args: K2JVMCompilerArguments, + abiSnapshots: Map = HashMap() ): CompilationMode { val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) initDirtyFiles(dirtyFiles, changedFiles) @@ -185,7 +209,9 @@ class IncrementalJvmCompilerRunner( reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" } val classpathChanges = reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { - getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter) + val scopes = caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct() + getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, + abiSnapshots, withSnapshot, caches.platformCache, scopes) } @Suppress("UNUSED_VARIABLE") // for sealed when diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt index 290712bf01b..b3a5b9b50a3 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/changesDetectionUtils.kt @@ -19,7 +19,11 @@ internal fun getClasspathChanges( changedFiles: ChangedFiles.Known, lastBuildInfo: BuildInfo, modulesApiHistory: ModulesApiHistory, - reporter: BuildReporter + reporter: BuildReporter, + abiSnapshots: Map, + withSnapshot: Boolean, + caches: IncrementalCacheCommon, + scopes: Collection ): ChangesEither { val classpathSet = HashSet() for (file in classpath) { @@ -40,52 +44,79 @@ internal fun getClasspathChanges( if (modifiedClasspath.isEmpty()) return ChangesEither.Known() - val lastBuildTS = lastBuildInfo.startTS + if (withSnapshot) { + fun analyzeJarFiles(): ChangesEither { + val symbols = HashSet() + val fqNames = HashSet() - val symbols = HashSet() - val fqNames = HashSet() + for ((module, abiSnapshot) in abiSnapshots) { + val actualAbiSnapshot = lastBuildInfo.dependencyToAbiSnapshot[module] + if (actualAbiSnapshot == null) { - val historyFilesEither = - reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) { - modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath) - } - val historyFiles = when (historyFilesEither) { - is Either.Success> -> historyFilesEither.value - is Either.Error -> { - reporter.report { "Could not find history files: ${historyFilesEither.reason}" } - return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND) - } - } - - fun analyzeHistoryFiles(): ChangesEither { - for (historyFile in historyFiles) { - val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter) - ?: return run { - reporter.report { "Could not read diffs from $historyFile" } - ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ) + reporter.report { "Some jar are removed from classpath $module" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_REMOVED_ENTRY) } - val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS } - if (knownBuilds.isEmpty()) { - reporter.report { "No previously known builds for $historyFile" } - return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS) + val diffData = AbiSnapshotDiffService.doCompute(abiSnapshot, actualAbiSnapshot, caches, scopes) + symbols.addAll(diffData.dirtyLookupSymbols) + fqNames.addAll(diffData.dirtyClassesFqNames) + + } + return ChangesEither.Known(symbols, fqNames) + } + return reporter.measure(BuildTime.IC_ANALYZE_JAR_FILES) { + analyzeJarFiles() + } + } else { + val lastBuildTS = lastBuildInfo.startTS + + val symbols = HashSet() + val fqNames = HashSet() + + val historyFilesEither = + reporter.measure(BuildTime.IC_FIND_HISTORY_FILES) { + modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath) } - for (buildDiff in newBuilds) { - if (!buildDiff.isIncremental) { - reporter.report { "Non-incremental build from dependency $historyFile" } - return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP) - - } - val dirtyData = buildDiff.dirtyData - symbols.addAll(dirtyData.dirtyLookupSymbols) - fqNames.addAll(dirtyData.dirtyClassesFqNames) + val historyFiles = when (historyFilesEither) { + is Either.Success> -> historyFilesEither.value + is Either.Error -> { + reporter.report { "Could not find history files: ${historyFilesEither.reason}" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_IS_NOT_FOUND) } } - return ChangesEither.Known(symbols, fqNames) - } + fun analyzeHistoryFiles(): ChangesEither { + for (historyFile in historyFiles) { + val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter) + ?: return run { + reporter.report { "Could not read diffs from $historyFile" } + ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_CANNOT_BE_READ) + } - return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) { - analyzeHistoryFiles() + val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS } + if (knownBuilds.isEmpty()) { + reporter.report { "No previously known builds for $historyFile" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_HISTORY_NO_KNOWN_BUILDS) + } + + + for (buildDiff in newBuilds) { + if (!buildDiff.isIncremental) { + reporter.report { "Non-incremental build from dependency $historyFile" } + return ChangesEither.Unknown(BuildAttribute.DEP_CHANGE_NON_INCREMENTAL_BUILD_IN_DEP) + + } + val dirtyData = buildDiff.dirtyData + symbols.addAll(dirtyData.dirtyLookupSymbols) + fqNames.addAll(dirtyData.dirtyClassesFqNames) + } + } + + return ChangesEither.Known(symbols, fqNames) + } + + return reporter.measure(BuildTime.IC_ANALYZE_HISTORY_FILES) { + analyzeHistoryFiles() + } } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt index ce7ac983cad..289886a22b8 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.incremental.multiproject +import org.jetbrains.kotlin.incremental.IncrementalModuleEntry import org.jetbrains.kotlin.incremental.IncrementalModuleInfo import org.jetbrains.kotlin.incremental.util.Either import java.io.File @@ -14,11 +15,14 @@ import java.util.zip.ZipFile interface ModulesApiHistory { fun historyFilesForChangedFiles(changedFiles: Set): Either> + fun abiSnapshot(jar: File): Either> } object EmptyModulesApiHistory : ModulesApiHistory { override fun historyFilesForChangedFiles(changedFiles: Set): Either> = Either.Error("Multi-module IC is not configured") + + override fun abiSnapshot(jar: File): Either> = Either.Error("Not supported") } abstract class ModulesApiHistoryBase(protected val modulesInfo: IncrementalModuleInfo) : ModulesApiHistory { @@ -130,6 +134,14 @@ class ModulesApiHistoryJvm(modulesInfo: IncrementalModuleInfo) : ModulesApiHisto return Either.Success(result) } + + override fun abiSnapshot(jar: File): Either> { + val abiSnapshot = modulesInfo.jarToModule[jar]?.abiSnapshot ?: modulesInfo.jarToAbiSnapshot[jar] + return if (abiSnapshot != null) + Either.Success(setOf(abiSnapshot)) + else + Either.Error("Failed to find abi snapshot for file ${jar.absolutePath}") + } } class ModulesApiHistoryJs(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) { @@ -141,6 +153,11 @@ class ModulesApiHistoryJs(modulesInfo: IncrementalModuleInfo) : ModulesApiHistor else -> Either.Error("No module is found for jar $jar") } } + + override fun abiSnapshot(jar: File): Either> { + return modulesInfo.jarToModule[jar]?.abiSnapshot?.let { Either.Success(setOf(it)) } ?: Either.Error("Failed to find snapshot for file ${jar.absolutePath}") + + } } class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) { @@ -158,7 +175,15 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH if (!isInProjectBuildDir(jar)) return Either.Error("Non-project jar is modified $jar") val jarPath = Paths.get(jar.absolutePath) - return getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath)) + return getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath), IncrementalModuleEntry::buildHistoryFile) + } + + override fun abiSnapshot(jar: File): Either> { + val jarPath = Paths.get(jar.absolutePath) + return when (val result = getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath), IncrementalModuleEntry::abiSnapshot)) { + is Either.Success -> Either.Success(result.value) + is Either.Error -> Either.Error(result.reason) + } } override fun getBuildHistoryForDir(file: File): Either> { @@ -175,7 +200,7 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH } } - return getHistoryForModuleNames(file.toPath(), moduleNames) + return getHistoryForModuleNames(file.toPath(), moduleNames, IncrementalModuleEntry::buildHistoryFile) } private fun getPossibleModuleNamesFromJar(path: Path): Collection { @@ -205,13 +230,13 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH return path.listFiles().filter { it.name.endsWith(".kotlin_module", ignoreCase = true) }.map { it.nameWithoutExtension } } - private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable): Either> { + private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable, fileLocation: (IncrementalModuleEntry) -> File): Either> { val possibleModules = moduleNames.flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() } val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(path) } if (modules.isEmpty()) return Either.Error("Unknown module for $path (candidates: ${possibleModules.joinToString()})") - val result = modules.mapTo(HashSet()) { it.buildHistoryFile } + val result = modules.mapTo(HashSet()) { fileLocation(it) } return Either.Success(result) } } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt index 2c25c1f3612..bb6125f6573 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt @@ -30,9 +30,10 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest>() private val jarToClassListFile = mutableMapOf() private val jarToModule = mutableMapOf() + private val jarToAbiSnapshot = mutableMapOf() protected val incrementalModuleInfo: IncrementalModuleInfo by lazy { - IncrementalModuleInfo(workingDir, workingDir, dirToModule, nameToModules, jarToClassListFile, jarToModule) + IncrementalModuleInfo(workingDir, workingDir, dirToModule, nameToModules, jarToClassListFile, jarToModule, jarToAbiSnapshot) } protected abstract val modulesApiHistory: ApiHistory @@ -97,8 +98,9 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest { diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt index c783c7de358..a17cefbb2be 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt @@ -26,9 +26,11 @@ class ModulesApiHistoryAndroidTest { private lateinit var appRoot: File private lateinit var appKotlinDestination: File private lateinit var appHistory: File + private lateinit var appAbiSnapshot: File private lateinit var libRoot: File private lateinit var libKotlinDestination: File private lateinit var libHistory: File + private lateinit var libAbiSnapshot: File private lateinit var androidHistory: ModulesApiHistoryAndroid @@ -38,8 +40,9 @@ class ModulesApiHistoryAndroidTest { appRoot = projectRoot.resolve("app") appHistory = appRoot.resolve("build/tmp/kotlin/app_history.bin") + appAbiSnapshot = appRoot.resolve("build/tmp/kotlin/app_abi_snapshot.bin") appKotlinDestination = appRoot.resolve("build/tmp/kotlin-classes").apply { mkdirs() } - val appEntry = IncrementalModuleEntry(":app", "app", appRoot.resolve("build"), appHistory) + val appEntry = IncrementalModuleEntry(":app", "app", appRoot.resolve("build"), appHistory, appAbiSnapshot) appRoot.resolve("build/intermediates/classes/meta-inf/").apply { mkdirs() resolve("app.kotlin_module").createNewFile() @@ -47,8 +50,9 @@ class ModulesApiHistoryAndroidTest { libRoot = projectRoot.resolve("lib") libHistory = libRoot.resolve("lib/build/tmp/kotlin/lib_history.bin") + libAbiSnapshot = libRoot.resolve("lib/build/tmp/kotlin/lib_abi_snapshot.bin") libKotlinDestination = libRoot.resolve("build/tmp/kotlin-classes").apply { mkdirs() } - val libEntry = IncrementalModuleEntry(":lib", "lib", libRoot.resolve("build"), libHistory) + val libEntry = IncrementalModuleEntry(":lib", "lib", libRoot.resolve("build"), libHistory, libAbiSnapshot) libRoot.resolve("build/intermediates/classes/meta-inf/").apply { mkdirs() resolve("lib.kotlin_module").createNewFile() @@ -60,7 +64,8 @@ class ModulesApiHistoryAndroidTest { dirToModule = mapOf(appKotlinDestination to appEntry, libKotlinDestination to libEntry), nameToModules = mapOf("app" to setOf(appEntry), "lib" to setOf(libEntry)), jarToClassListFile = mapOf(), - jarToModule = mapOf() + jarToModule = mapOf(), + jarToAbiSnapshot = mapOf() ) androidHistory = ModulesApiHistoryAndroid(info) diff --git a/core/metadata/src/org/jetbrains/kotlin/metadata/deserialization/NameResolverImpl.kt b/core/metadata/src/org/jetbrains/kotlin/metadata/deserialization/NameResolverImpl.kt index 44c5d19c812..cef4cd8ebde 100644 --- a/core/metadata/src/org/jetbrains/kotlin/metadata/deserialization/NameResolverImpl.kt +++ b/core/metadata/src/org/jetbrains/kotlin/metadata/deserialization/NameResolverImpl.kt @@ -10,8 +10,8 @@ import org.jetbrains.kotlin.metadata.ProtoBuf.QualifiedNameTable.QualifiedName import java.util.* class NameResolverImpl( - private val strings: ProtoBuf.StringTable, - private val qualifiedNames: ProtoBuf.QualifiedNameTable + val strings: ProtoBuf.StringTable, + val qualifiedNames: ProtoBuf.QualifiedNameTable ) : NameResolver { override fun getString(index: Int): String = strings.getString(index) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt index d92c914a711..cb41d297efa 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt @@ -251,6 +251,7 @@ abstract class BaseGradleIT { val useFir: Boolean = false, val customEnvironmentVariables: Map = mapOf(), val dryRun: Boolean = false, + val abiSnapshot: Boolean = false, ) enum class ConfigurationCacheProblems { @@ -953,6 +954,9 @@ Finished executing task ':$taskName'| if (options.dryRun) { add("--dry-run") } + if (options.abiSnapshot) { + add("-Dkotlin.incremental.classpath.snapshot.enabled=true") + } add("-Dorg.gradle.unsafe.configuration-cache=${options.configurationCache}") add("-Dorg.gradle.unsafe.configuration-cache-problems=${options.configurationCacheProblems.name.toLowerCase()}") diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationBaseIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationBaseIT.kt index 5860c897d8f..883c602d1fa 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationBaseIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationBaseIT.kt @@ -27,6 +27,14 @@ abstract class IncrementalCompilationBaseIT : BaseGradleIT() { protected fun doTest( modifyProject: (Project) -> Unit, expectedAffectedFileNames: Collection, + ) { + doTest(defaultBuildOptions(), modifyProject, expectedAffectedFileNames) + } + + protected fun doTest( + options: BuildOptions, + modifyProject: (Project) -> Unit, + expectedAffectedFileNames: Collection, ) { val project = defaultProject() project.build("build") { @@ -35,7 +43,7 @@ abstract class IncrementalCompilationBaseIT : BaseGradleIT() { modifyProject(project) - project.build("build") { + project.build("build", options = options) { assertSuccessful() val expectedAffectedFiles = project.projectDir.getFilesByNames(*expectedAffectedFileNames.toTypedArray()) val expectedAffectedFileRelativePaths = project.relativize(expectedAffectedFiles) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt index b3cfdbf9c6a..65cf347523c 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt @@ -189,6 +189,30 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation ) } + @Test + fun testAddNewMethodToLib() { + doTest( + options = defaultBuildOptions().copy(abiSnapshot = true), + { project -> + val aKt = project.projectDir.getFileByName("A.kt") + aKt.writeText( + """ +package bar + +open class A { + fun a() {} + fun newA() {} +} +""" + ) + }, + //TODO for abi-snapshot "BB.kt" should not be recompiled + expectedAffectedFileNames = listOf( + "A.kt", "B.kt", "AA.kt", "BB.kt", "AAA.kt" + ) + ) + } + @Test fun testLibClassBecameFinal() { // TODO: fix fir IC and remove @@ -229,6 +253,8 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation project.projectFile("BarDummy.kt").modify { it.replace("class BarDummy", "open class BarDummy") } + + //don't need to recompile app classes because lib's proto stays the same project.build("build") { assertSuccessful() val affectedSources = project.projectDir.allKotlinFiles() @@ -244,9 +270,67 @@ abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilation } } + @Test + fun testCleanBuildLibForAbiSnapshot() { + val options = defaultBuildOptions().copy(abiSnapshot = true) + val project = defaultProject() + + project.build("build", options = options) { + assertSuccessful() + } + + project.build(":lib:clean", options = options) { + assertSuccessful() + } + + // Change file so Gradle won't skip :app:compile + project.projectFile("BarDummy.kt").modify { + it.replace("class BarDummy", "open class BarDummy") + } + + //don't need to recompile app classes because lib's proto stays the same + project.build("build", options = options) { + val affectedSources = project.projectDir.allKotlinFiles() + val relativePaths = project.relativize(affectedSources) + assertCompiledKotlinSources(relativePaths) + } + + val aaKt = project.projectFile("AA.kt") + aaKt.modify { "$it " } + project.build("build", options = options) { + assertSuccessful() + assertCompiledKotlinSources(project.relativize(aaKt)) + } + } + protected abstract val additionalLibDependencies: String protected abstract val compileKotlinTaskName: String + @Test + fun testAddMethodToLibForAbiSnapshot() { + val project = defaultProject() + val options = defaultBuildOptions().copy(abiSnapshot = true) + + project.build("build", options = options) { + assertSuccessful() + } + + // Change file so Gradle won't skip :app:compile + project.projectFile("BarDummy.kt").modify { + it.replace("class BarDummy", "open class BarDummy") + } + + val barDummyClassFile = project.projectFile("BarDummy.kt") + barDummyClassFile.modify { "$it { fun m() = 42}" } + + project.build("build", options = options) { + assertSuccessful() + val relativePaths = project.relativize(barDummyClassFile) + assertCompiledKotlinSources(relativePaths) + } + + } + @Test fun testAddDependencyToLib() { val project = defaultProject() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt index 482f3562430..75dfe8710f3 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt @@ -20,6 +20,7 @@ import org.jetbrains.kotlin.gradle.logging.kotlinDebug import org.jetbrains.kotlin.gradle.logging.kotlinInfo import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider @@ -212,6 +213,7 @@ internal open class GradleCompilerRunner( val nameToModules = HashMap>() val jarToClassListFile = HashMap() val jarToModule = HashMap() + val jarToAbiSnapshot = HashMap() val multiplatformProjectTasks = mutableMapOf>() @@ -229,7 +231,8 @@ internal open class GradleCompilerRunner( project.path, task.moduleName.get(), project.buildDir, - task.buildHistoryFile.get().asFile + task.buildHistoryFile.get().asFile, + task.abiSnapshotFile.get().asFile ) dirToModule[task.destinationDir] = module task.javaOutputDir.orNull?.asFile?.let { dirToModule[it] = module } @@ -240,6 +243,15 @@ internal open class GradleCompilerRunner( jarToModule[it] = module } } +// for (target in task.targets) { +// if (target is KotlinWithJavaTarget<*>) { +// val jar = project.tasks.getByName(target.artifactsTaskName) as Jar +// jarToClassListFile[jar.archivePathCompatible.canonicalFile] = target.defaultArtifactClassesListFile.get() +// //configure abiSnapshot mapping for jars +// jarToAbiSnapshot[jar.archivePathCompatible.canonicalFile] = +// target.buildDir.get().file(task.abiSnapshotRelativePath).asFile +// } +// } } else if (task is InspectClassesForMultiModuleIC) { jarToClassListFile[File(task.archivePath.get())] = task.classesListFile } @@ -260,10 +272,19 @@ internal open class GradleCompilerRunner( project.path, kotlinTask.moduleName.get(), project.buildDir, - kotlinTask.buildHistoryFile.get().asFile + kotlinTask.buildHistoryFile.get().asFile, + kotlinTask.abiSnapshotFile.get().asFile ) val jarTask = project.tasks.findByName(target.artifactsTaskName) as? AbstractArchiveTask ?: continue jarToModule[jarTask.archivePathCompatible.canonicalFile] = module + if (target is KotlinWithJavaTarget<*>) { + val jar = project.tasks.getByName(target.artifactsTaskName) as Jar + jarToClassListFile[jar.archivePathCompatible.canonicalFile] = target.defaultArtifactClassesListFile.get() + //configure abiSnapshot mapping for jars + jarToAbiSnapshot[jar.archivePathCompatible.canonicalFile] = + target.buildDir.get().file(kotlinTask.abiSnapshotRelativePath).get().asFile + } + } } } @@ -274,7 +295,8 @@ internal open class GradleCompilerRunner( dirToModule = dirToModule, nameToModules = nameToModules, jarToClassListFile = jarToClassListFile, - jarToModule = jarToModule + jarToModule = jarToModule, + jarToAbiSnapshot = jarToAbiSnapshot ).also { cachedGradle = WeakReference(gradle) cachedModulesInfo = it diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt new file mode 100644 index 00000000000..4a258319df9 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/AbiSnapshotDiffService.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin + +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.incremental.* + +abstract class AbiSnapshotDiffService() : BuildService { + abstract class Parameters : BuildServiceParameters { + abstract val caches: IncrementalCachesManager<*> + abstract val reporter: BuildReporter + abstract val sourceFilesExtensions: List + } + + val caches: IncrementalCachesManager<*> = parameters.caches + val reporter: BuildReporter = parameters.reporter + val sourceFilesExtensions: List = parameters.sourceFilesExtensions + + companion object { +// //Store list of changed lookups +// val diffCache: MutableMap, DirtyFilesContainer> = mutableMapOf() +// @TestOnly +// fun compareJarsInternal(caches: IncrementalCachesManager<*>, reporter: ICReporter, sourceFilesExtensions: List, +// snapshot: AbiSnapshot, newJar: AbiSnapshot) = +// diffCache.computeIfAbsent(Pair(snapshot, newJar)) { (snapshotJar, actualJar) -> +// DirtyFilesContainer(caches, reporter, sourceFilesExtensions) +// .also { +// it.addByDirtyClasses(snapshotJar.fqNames.minus(actualJar.fqNames)) +// it.addByDirtyClasses(actualJar.fqNames.minus(snapshotJar.fqNames)) +// it.addByDirtySymbols(snapshotJar.symbols.minus(actualJar.symbols)) +// it.addByDirtySymbols(actualJar.symbols.minus(snapshotJar.symbols)) +// } +// } + } + +// @Synchronized +// fun compareJarsInternal(snapshot: AbiSnapshot, newJar: AbiSnapshot): DirtyFilesContainer { +// return diffCache.computeIfAbsent(Pair(snapshot, newJar)) { (snapshotJar, actualJar) -> +// DirtyFilesContainer(caches, reporter, sourceFilesExtensions) +// .also { +// it.addByDirtyClasses(snapshotJar.fqNames.minus(actualJar.fqNames)) +// it.addByDirtyClasses(actualJar.fqNames.minus(snapshotJar.fqNames)) +// it.addByDirtySymbols(snapshotJar.symbols.minus(actualJar.symbols)) +// it.addByDirtySymbols(actualJar.symbols.minus(snapshotJar.symbols)) +// } +// +// } +// } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/jvm/KotlinWithJavaTarget.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/jvm/KotlinWithJavaTarget.kt index 82e3a9956d5..d57e72e4877 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/jvm/KotlinWithJavaTarget.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/jvm/KotlinWithJavaTarget.kt @@ -9,6 +9,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp import org.gradle.api.NamedDomainObjectContainer import org.gradle.api.Project import org.gradle.api.artifacts.Dependency +import org.gradle.api.file.Directory import org.gradle.api.plugins.JavaPlugin import org.gradle.api.provider.Provider import org.gradle.jvm.tasks.Jar @@ -52,6 +53,8 @@ open class KotlinWithJavaTarget( val jarTask = project.tasks.getByName(artifactsTaskName) as Jar it.file("${sanitizeFileName(jarTask.archiveFileName.get())}-classes.txt").asFile } + + internal val buildDir: Provider = layout.buildDirectory.dir(KOTLIN_BUILD_DIR_NAME) } private fun sanitizeFileName(candidate: String): String = candidate.filter { it.isLetterOrDigit() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index eae6125bb64..8d8c503c8cc 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -54,6 +54,7 @@ import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.isParentOf import org.jetbrains.kotlin.gradle.utils.pathsAsStringRelativeTo import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.incremental.IncrementalCompilerRunner import org.jetbrains.kotlin.library.impl.isKotlinLibrary import org.jetbrains.kotlin.utils.JsLibraryUtils import java.io.File @@ -274,6 +275,16 @@ abstract class AbstractKotlinCompile : AbstractKotl @get:Input internal val moduleName: Property = objects.property(String::class.java) + @get:Internal + val abiSnapshotFile + get() = taskBuildDirectory.file(IncrementalCompilerRunner.ABI_SNAPSHOT_FILE_NAME) + + @get:Input + val abiSnapshotRelativePath: Property = objects.property(String::class.java).value( + //TODO update to support any jar changes + "$name/${IncrementalCompilerRunner.ABI_SNAPSHOT_FILE_NAME}" + ) + @get:Internal internal val friendSourceSets = objects.listProperty(String::class.java) @@ -300,34 +311,36 @@ abstract class AbstractKotlinCompile : AbstractKotl @TaskAction fun execute(inputs: IncrementalTaskInputs) { - systemPropertiesService.get().startIntercept() - CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value = "true" + metrics.measure(BuildTime.GRADLE_TASK_ACTION) { + systemPropertiesService.get().startIntercept() + CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value = "true" - // If task throws exception, but its outputs are changed during execution, - // then Gradle forces next build to be non-incremental (see Gradle's DefaultTaskArtifactStateRepository#persistNewOutputs) - // To prevent this, we backup outputs before incremental build and restore when exception is thrown - val outputsBackup: TaskOutputsBackup? = - if (isIncrementalCompilationEnabled() && inputs.isIncremental) - metrics.measure(BuildTime.BACKUP_OUTPUT) { - TaskOutputsBackup(allOutputFiles()) - } - else null + // If task throws exception, but its outputs are changed during execution, + // then Gradle forces next build to be non-incremental (see Gradle's DefaultTaskArtifactStateRepository#persistNewOutputs) + // To prevent this, we backup outputs before incremental build and restore when exception is thrown + val outputsBackup: TaskOutputsBackup? = + if (isIncrementalCompilationEnabled() && inputs.isIncremental) + metrics.measure(BuildTime.BACKUP_OUTPUT) { + TaskOutputsBackup(allOutputFiles()) + } + else null - if (!isIncrementalCompilationEnabled()) { - clearLocalState("IC is disabled") - } else if (!inputs.isIncremental) { - clearLocalState("Task cannot run incrementally") - } - - try { - executeImpl(inputs) - } catch (t: Throwable) { - if (outputsBackup != null) { - metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) { - outputsBackup.restoreOutputs() - } + if (!isIncrementalCompilationEnabled()) { + clearLocalState("IC is disabled") + } else if (!inputs.isIncremental) { + clearLocalState("Task cannot run incrementally") + } + + try { + executeImpl(inputs) + } catch (t: Throwable) { + if (outputsBackup != null) { + metrics.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) { + outputsBackup.restoreOutputs() + } + } + throw t } - throw t } }