azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KT-34862 use abi snapshot instead of build history files

Browse Source 
Use jar snapshot instead build history file to avoid using time stamps and support remote gradle cache
This commit is contained in:
nataliya.valtman
2020-10-20 13:14:11 +03:00
parent e5fbd0e2d8
commit f6b428f271
33 changed files with 942 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt
+1
View File
@@ -17,6 +17,7 @@ enum class BuildAttributeKind : Serializable {
enum class BuildAttribute(val kind: BuildAttributeKind) : Serializable {
NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON),
NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON),
CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON),
UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON),
JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON),
build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt
+5
View File
@@ -9,6 +9,7 @@ import java.io.Serializable
@Suppress("Reformat")
enum class BuildTime(val parent: BuildTime? = null) : Serializable {
GRADLE_TASK_ACTION,
GRADLE_TASK,
CLEAR_OUTPUT(GRADLE_TASK),
BACKUP_OUTPUT(GRADLE_TASK),
@@ -20,6 +21,10 @@ enum class BuildTime(val parent: BuildTime? = null) : Serializable {
NON_INCREMENTAL_COMPILATION_OUT_OF_PROCESS(RUN_COMPILER),
NON_INCREMENTAL_COMPILATION_DAEMON(RUN_COMPILER),
INCREMENTAL_COMPILATION(RUN_COMPILER),
STORE_BUILD_INFO(INCREMENTAL_COMPILATION),
JAR_SNAPSHOT(INCREMENTAL_COMPILATION),
SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT),
IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT),
IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION),
IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET),
IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES),
build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt
+58 -24
View File
@@ -30,6 +30,28 @@ class ChangesCollector {
private val changedMembers = hashMapOf<FqName, MutableSet<String>>()
private val areSubclassesAffected = hashMapOf<FqName, Boolean>()
//TODO for test only: ProtoData or ProtoBuf
private val storage = hashMapOf<FqName, ProtoData>()
private val removed = ArrayList<FqName>()
//TODO change to immutable map
fun protoDataChanges(): Map<FqName, ProtoData> = storage
fun protoDataRemoved(): List<FqName> = removed
companion object {
fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>) =
members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
fun ClassProtoData.getNonPrivateMemberNames(): Set<String> {
return proto.getNonPrivateNames(
nameResolver,
ProtoBuf.Class::getConstructorList,
ProtoBuf.Class::getFunctionList,
ProtoBuf.Class::getPropertyList
) + proto.enumEntryList.map { nameResolver.getString(it.name) }
}
}
fun changes(): List<ChangeInfo> {
val changes = arrayListOf<ChangeInfo>()
@@ -57,7 +79,7 @@ class ChangesCollector {
}
private fun <T, R> MutableMap<T, MutableSet<R>>.getSet(key: T) =
getOrPut(key) { HashSet() }
getOrPut(key) { HashSet() }
private fun collectChangedMember(scope: FqName, name: String) {
changedMembers.getSet(scope).add(name)
@@ -79,11 +101,35 @@ class ChangesCollector {
}
}
fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?, collectAllMembersForNewClass: Boolean = false) {
fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?, collectAllMembersForNewClass: Boolean = false, packageProtoKey: String? = null) {
if (oldData == null && newData == null) {
throw IllegalStateException("Old and new value are null")
}
if (newData != null) {
when (newData) {
is ClassProtoData -> {
val fqName = newData.nameResolver.getClassId(newData.proto.fqName).asSingleFqName()
storage[fqName] = newData
}
is PackagePartProtoData -> {
//TODO fqName is not unique. It's package and can be present in both java and kotlin
val fqName = newData.packageFqName
storage[packageProtoKey?.let { FqName(it) } ?: fqName] = newData
}
}
} else {
when (oldData) {
is ClassProtoData -> {
removed.add(oldData.nameResolver.getClassId(oldData.proto.fqName).asSingleFqName())
}
is PackagePartProtoData -> {
//TODO fqName is not unique. It's package and can be present in both java and kotlin
removed.add(packageProtoKey?.let { FqName(it) } ?: oldData.packageFqName)
}
}
}
if (oldData == null) {
newData!!.collectAll(isRemoved = false, isAdded = true, collectAllMembersForNewClass = collectAllMembersForNewClass)
return
@@ -125,8 +171,8 @@ class ChangesCollector {
}
}
private fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>): Set<String> =
members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>) =
members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
//TODO remember all sealed parent classes
private fun ProtoData.collectAll(isRemoved: Boolean, isAdded: Boolean, collectAllMembersForNewClass: Boolean = false) =
@@ -137,16 +183,15 @@ class ChangesCollector {
private fun PackagePartProtoData.collectAllFromPackage(isRemoved: Boolean) {
val memberNames =
proto.getNonPrivateNames(
nameResolver,
ProtoBuf.Package::getFunctionList,
ProtoBuf.Package::getPropertyList
)
proto.getNonPrivateNames(
nameResolver,
ProtoBuf.Package::getFunctionList,
ProtoBuf.Package::getPropertyList
)
if (isRemoved) {
collectRemovedMembers(packageFqName, memberNames)
}
else {
} else {
collectChangedMembers(packageFqName, memberNames)
}
}
@@ -161,8 +206,7 @@ class ChangesCollector {
val collectMember = if (isRemoved) this@ChangesCollector::collectRemovedMember else this@ChangesCollector::collectChangedMember
collectMember(classFqName.parent(), classFqName.shortName().asString())
memberNames.forEach { collectMember(classFqName, it) }
}
else {
} else {
if (!isRemoved && collectAllMembersForNewClass) {
val memberNames = getNonPrivateMemberNames()
memberNames.forEach { this@ChangesCollector.collectChangedMember(classFqName, it) }
@@ -189,15 +233,6 @@ class ChangesCollector {
addChangedParents(fqName, changedParentsFqNames)
}
private fun ClassProtoData.getNonPrivateMemberNames(): Set<String> {
return proto.getNonPrivateNames(
nameResolver,
ProtoBuf.Class::getConstructorList,
ProtoBuf.Class::getFunctionList,
ProtoBuf.Class::getPropertyList
) + proto.enumEntryList.map { nameResolver.getString(it.name) }
}
fun collectMemberIfValueWasChanged(scope: FqName, name: String, oldValue: Any?, newValue: Any?) {
if (oldValue == null && newValue == null) {
throw IllegalStateException("Old and new value are null for $scope#$name")
@@ -205,8 +240,7 @@ class ChangesCollector {
if (oldValue != null && newValue == null) {
collectRemovedMember(scope, name)
}
else if (oldValue != newValue) {
} else if (oldValue != newValue) {
collectChangedMember(scope, name)
}
}
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt
+2 -2
View File
@@ -185,7 +185,7 @@ open class IncrementalJvmCache(
sourceToClassesMap.add(source, jvmClassName)
val (proto, nameResolver) = serializedJavaClass.toProtoData()
addToClassStorage(proto, nameResolver, source)
// collector.addJavaProto(ClassProtoData(proto, nameResolver))
dirtyOutputClassesMap.notDirty(jvmClassName)
}
@@ -308,7 +308,7 @@ open class IncrementalJvmCache(
storage[key] = newData
val packageFqName = kotlinClass.className.packageFqName
changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName))
changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName), packageProtoKey = key)
}
operator fun contains(className: JvmClassName): Boolean =
build-common/src/org/jetbrains/kotlin/incremental/IncrementalModuleInfo.kt
+5 -2
View File
@@ -12,7 +12,8 @@ data class IncrementalModuleEntry(
private val projectPath: String,
val name: String,
val buildDir: File,
val buildHistoryFile: File
val buildHistoryFile: File,
val abiSnapshot: File
) : Serializable {
companion object {
private const val serialVersionUID = 0L
@@ -26,7 +27,9 @@ class IncrementalModuleInfo(
val nameToModules: Map<String, Set<IncrementalModuleEntry>>,
val jarToClassListFile: Map<File, File>,
// only for js and mpp
val jarToModule: Map<File, IncrementalModuleEntry>
val jarToModule: Map<File, IncrementalModuleEntry>,
//for JVM only
val jarToAbiSnapshot: Map<File, File>
) : Serializable {
companion object {
private const val serialVersionUID = 1L
build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+1 -1
View File
@@ -41,7 +41,7 @@ open class LookupStorage(
private val countersFile = "counters".storageFile
private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter))
private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter))
private val lookupMap = registerMap(LookupMap("lookups".storageFile))
val lookupMap = registerMap(LookupMap("lookups".storageFile))
@Volatile
private var size: Int = 0
build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt
+9 -8
View File
@@ -255,22 +255,23 @@ fun withSubtypes(
typeFqName: FqName,
caches: Iterable<IncrementalCacheCommon>
): Set<FqName> {
val types = LinkedHashSet(listOf(typeFqName))
val subtypes = hashSetOf<FqName>()
val typesToProccess = LinkedHashSet(listOf(typeFqName))
val proccessedTypes = hashSetOf<FqName>()
while (types.isNotEmpty()) {
val iterator = types.iterator()
while (typesToProccess.isNotEmpty()) {
val iterator = typesToProccess.iterator()
val unprocessedType = iterator.next()
iterator.remove()
caches.asSequence()
.flatMap { it.getSubtypesOf(unprocessedType) }
.filter { it !in subtypes }
.forEach { types.add(it) }
.filter { it !in proccessedTypes }
.forEach { typesToProccess.add(it) }
subtypes.add(unprocessedType)
proccessedTypes.add(unprocessedType)
}
return subtypes
return proccessedTypes
}
build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt
+1 -1
View File
@@ -18,7 +18,7 @@ package org.jetbrains.kotlin.incremental.storage
import java.io.File
internal class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) {
class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) {
override fun dumpKey(key: LookupSymbolKey): String = key.toString()
override fun dumpValue(value: Collection<Int>): String = value.toString()
build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt
+32 -6
View File
@@ -21,23 +21,49 @@ import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.EnumeratorStringDescriptor
import com.intellij.util.io.IOUtil
import com.intellij.util.io.KeyDescriptor
import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
import org.jetbrains.kotlin.cli.common.toBooleanLenient
import java.io.DataInput
import java.io.DataInputStream
import java.io.DataOutput
import java.io.File
import java.util.*
/**
* Storage versioning:
* 0 - only name and value hashes are saved
* 1 - name and scope are saved
*/
object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
override fun read(input: DataInput): LookupSymbolKey {
val first = input.readInt()
val second = input.readInt()
return LookupSymbolKey(first, second)
val version = input.readByte()
return when (version.toInt()) {
0 -> {
val name = input.readUTF()
val scope = input.readUTF()
LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope)
}
1 -> {
val first = input.readInt()
val second = input.readInt()
LookupSymbolKey(first, second, "", "")
}
else -> throw RuntimeException("Unknown version of LookupSymbolKeyDescriptor=${version}")
}
}
private val storeFullFqName = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SHAPSHOTS.value.toBooleanLenient() ?: false
override fun save(output: DataOutput, value: LookupSymbolKey) {
output.writeInt(value.nameHash)
output.writeInt(value.scopeHash)
if (storeFullFqName) {
output.writeByte(0)
output.writeUTF(value.name)
output.writeUTF(value.scope)
} else {
output.writeByte(1)
output.writeInt(value.nameHash)
output.writeInt(value.scopeHash)
}
}
override fun getHashCode(value: LookupSymbolKey): Int = value.hashCode()
build-common/src/org/jetbrains/kotlin/incremental/storage/values.kt
+22 -2
View File
@@ -16,8 +16,8 @@
package org.jetbrains.kotlin.incremental.storage
data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int) : Comparable<LookupSymbolKey> {
constructor(name: String, scope: String) : this(name.hashCode(), scope.hashCode())
data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int, val name:String, val scope:String) : Comparable<LookupSymbolKey> {
constructor(name: String, scope: String) : this(name.hashCode(), scope.hashCode(), name, scope)
override fun compareTo(other: LookupSymbolKey): Int {
val nameCmp = nameHash.compareTo(other.nameHash)
@@ -26,6 +26,26 @@ data class LookupSymbolKey(val nameHash: Int, val scopeHash: Int) : Comparable<L
return scopeHash.compareTo(other.scopeHash)
}
override fun hashCode(): Int {
var result = nameHash
result = 31 * result + scopeHash
return result
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as LookupSymbolKey
if (nameHash != other.nameHash) return false
if (scopeHash != other.scopeHash) return false
return true
}
}
data class ProtoMapValue(val isPackageFacade: Boolean, val bytes: ByteArray, val strings: Array<String>)