azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KT-45777: Shrink classpath snapshot incrementally

Browse Source 
Currently, we shrink classpath snapshots at 2 steps:
  - Classpath diffing: Shrink the current classpath snapshot against
    the previous lookup symbols
  - Classpath snapshot saving: Shrink the current classpath snapshot
    against the current lookup symbols

With this commit, the shrinking at the second step is now incremental.
The shrinking at the first step is still non-incremental.
This commit is contained in:
Hung Nguyen
2021-12-05 19:54:20 +00:00
committed by teamcity
parent 4f3debdec6
commit 586fa8af64
20 changed files with 366 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt
+9 -8
View File
@@ -29,7 +29,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"),
COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"),
LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"),
REMOVE_DUPLICATE_CLASSES(LOAD_CURRENT_CLASSPATH_SNAPSHOT, "Remove duplicate classes"),
SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"),
GET_LOOKUP_SYMBOLS(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Get lookup symbols"),
FIND_REFERENCED_CLASSES(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Find referenced classes"),
FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"),
LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load shrunk previous classpath snapshot"),
COMPUTE_CHANGED_AND_IMPACTED_SET(COMPUTE_CLASSPATH_CHANGES, "Compute changed and impacted set"),
COMPUTE_CLASS_CHANGES(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute class changes"),
@@ -47,14 +51,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"),
NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"),
IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"),
SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Save shrunk current classpath snapshot after compilation"),
LOAD_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load classpath snapshot"),
SHRINK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink classpath snapshot"),
GET_NON_DUPLICATE_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Get non-duplicate classes"),
GET_LOOKUP_SYMBOLS(SHRINK_CLASSPATH_SNAPSHOT, "Get lookup symbols"),
FIND_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find referenced classes"),
FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"),
SAVE_SHRUNK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot"),
SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Shrink and save current classpath snapshot after compilation"),
LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load shrunk previous classpath snapshot after compilation"),
LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load current classpath snapshot after compilation"),
SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot after compilation"),
SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot after compilation"),
COMPILER_PERFORMANCE(readableString = "Compiler time"),
COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"),
CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"),
build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt
+6 -3
View File
@@ -11,7 +11,7 @@ import java.io.Serializable
/**
* Changes to the classpath of the `KotlinCompile` task, or information to compute them later by the Kotlin incremental compiler (see
* [ClasspathSnapshotEnabled.ToBeComputedByIncrementalCompiler].
* [ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler].
*/
sealed class ClasspathChanges : Serializable {
@@ -19,9 +19,12 @@ sealed class ClasspathChanges : Serializable {
abstract val classpathSnapshotFiles: ClasspathSnapshotFiles
class Empty(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
sealed class IncrementalRun : ClasspathSnapshotEnabled() {
class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
class NoChanges(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : IncrementalRun()
class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : IncrementalRun()
}
class NotAvailableDueToMissingClasspathSnapshot(override val classpathSnapshotFiles: ClasspathSnapshotFiles) :
ClasspathSnapshotEnabled()
build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+77 -3
View File
@@ -33,7 +33,8 @@ import java.util.*
open class LookupStorage(
targetDataDir: File,
pathConverter: FileToPathConverter,
storeFullFqNames: Boolean = false
storeFullFqNames: Boolean = false,
private val trackChanges: Boolean = false
) : BasicMapsOwner(targetDataDir) {
val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")
@@ -45,7 +46,7 @@ open class LookupStorage(
private val countersFile = "counters".storageFile
private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter))
private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter))
val lookupMap = registerMap(LookupMap("lookups".storageFile, storeFullFqNames))
private val lookupMap = TrackedLookupMap(registerMap(LookupMap("lookups".storageFile, storeFullFqNames)), trackChanges)
@Volatile
private var size: Int = 0
@@ -56,7 +57,8 @@ open class LookupStorage(
if (countersFile.exists()) {
val lines = countersFile.readLines()
size = lines.firstOrNull()?.toIntOrNull() ?: throw IOException("$countersFile exists, but it is empty. " +
"Counters file is corrupted")
"Counters file is corrupted"
)
oldSize = size
}
} catch (e: IOException) {
@@ -66,6 +68,24 @@ open class LookupStorage(
}
}
/** Set of [LookupSymbol]s that have been added after the initialization of this [LookupStorage] instance. */
val addedLookupSymbols: Set<LookupSymbolKey>
get() = run {
check(trackChanges) { "trackChanges is not enabled" }
lookupMap.addedKeys!!
}
/** Set of [LookupSymbol]s that have been removed after the initialization of this [LookupStorage] instance. */
val removedLookupSymbols: Set<LookupSymbolKey>
get() = run {
check(trackChanges) { "trackChanges is not enabled" }
lookupMap.removedKeys!!
}
/** Returns all [LookupSymbol]s in this storage. Note that this call takes a bit of time to run. */
val lookupSymbols: Collection<LookupSymbolKey>
get() = lookupMap.keys
@Synchronized
fun get(lookupSymbol: LookupSymbol): Collection<String> {
val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
@@ -266,3 +286,57 @@ data class LookupSymbol(val name: String, val scope: String) : Comparable<Lookup
return name.compareTo(other.name)
}
}
/**
* Wrapper of a [LookupMap] which tracks changes to the map after the initialization of this [TrackedLookupMap] instance, (unless
* [trackChanges] is set to `false`).
*/
private class TrackedLookupMap(private val lookupMap: LookupMap, private val trackChanges: Boolean) {
// Note that there may be multiple operations on the same key, and the following sets contain the *latest* differences with the original
// set of keys in the map. For example, if a key is added then removed, or vice versa, it will not be present in either set.
val addedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null
val removedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null
val keys: Collection<LookupSymbolKey>
get() = lookupMap.keys
operator fun get(key: LookupSymbolKey): Collection<Int>? = lookupMap[key]
operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) {
recordSet(key)
lookupMap[key] = fileIds
}
fun append(key: LookupSymbolKey, fileIds: Collection<Int>) {
recordSet(key)
lookupMap.append(key, fileIds)
}
fun remove(key: LookupSymbolKey) {
recordRemove(key)
lookupMap.remove(key)
}
private fun recordSet(key: LookupSymbolKey) {
if (!trackChanges) return
if (lookupMap[key] == null) {
if (key in removedKeys!!) {
removedKeys.remove(key)
} else {
addedKeys!!.add(key)
}
}
}
private fun recordRemove(key: LookupSymbolKey) {
if (!trackChanges) return
if (lookupMap[key] != null) {
if (key in addedKeys!!) {
addedKeys.remove(key)
} else {
removedKeys!!.add(key)
}
}
}
}
build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt
+18 -20
View File
@@ -27,40 +27,38 @@ import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import java.io.*
/**
* Storage versioning:
* 0 - only name and value hashes are saved
* 1 - name and scope are saved
*/
class LookupSymbolKeyDescriptor(
/** If `true`, original values are saved; if `false`, only hashes are saved. */
private val storeFullFqNames: Boolean = false
) : KeyDescriptor<LookupSymbolKey> {
override fun read(input: DataInput): LookupSymbolKey {
val version = input.readByte()
return when (version.toInt()) {
0 -> {
val name = input.readUTF()
val scope = input.readUTF()
LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope)
}
1 -> {
val first = input.readInt()
val second = input.readInt()
LookupSymbolKey(first, second, "", "")
}
else -> throw IllegalArgumentException("Unknown version of LookupSymbolKeyDescriptor=${version}")
// Note: The value of the storeFullFqNames variable below may or may not be the same as LookupSymbolKeyDescriptor.storeFullFqNames.
// Byte value `0` means storeFullFqNames == true, see `save` function below.
val storeFullFqNames = when (val byteValue = input.readByte().toInt()) {
0 -> true
1 -> false
else -> error("Unexpected byte value for storeFullFqNames: $byteValue")
}
return if (storeFullFqNames) {
val name = input.readUTF()
val scope = input.readUTF()
LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope)
} else {
val nameHash = input.readInt()
val scopeHash = input.readInt()
LookupSymbolKey(nameHash, scopeHash, "", "")
}
}
override fun save(output: DataOutput, value: LookupSymbolKey) {
// Write a Byte value `0` to represent storeFullFqNames == true for historical reasons (if we switch this value to `1` or write a
// Boolean instead, it might impact some tests).
output.writeByte(if (storeFullFqNames) 0 else 1)
if (storeFullFqNames) {
output.writeByte(0)
output.writeUTF(value.name)
output.writeUTF(value.scope)
} else {
output.writeByte(1)
output.writeInt(value.nameHash)
output.writeInt(value.scopeHash)
}