KT-45777: Compute classpath changes for incremental Kotlin compile

Test: Updated unit tests + incremental compilation integration tests
This commit is contained in:
Hung Nguyen
2021-08-25 16:23:20 +01:00
committed by nataliya.valtman
parent e3d7b7a30e
commit a48bf63630
12 changed files with 473 additions and 420 deletions
@@ -5,10 +5,12 @@
package org.jetbrains.kotlin.incremental package org.jetbrains.kotlin.incremental
import com.intellij.util.io.DataExternalizer
import org.jetbrains.kotlin.incremental.storage.FqNameExternalizer
import org.jetbrains.kotlin.incremental.storage.LinkedHashSetExternalizer
import org.jetbrains.kotlin.incremental.storage.LookupSymbolExternalizer
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import java.io.ObjectInputStream import java.io.*
import java.io.ObjectOutputStream
import java.io.Serializable
/** /**
* Changes to the classpath of the `KotlinCompile` task, used to compute the source files that need to be recompiled during an incremental * Changes to the classpath of the `KotlinCompile` task, used to compute the source files that need to be recompiled during an incremental
@@ -18,59 +20,54 @@ sealed class ClasspathChanges : Serializable {
class Available() : ClasspathChanges() { class Available() : ClasspathChanges() {
lateinit var lookupSymbols: List<LookupSymbol>
private set
lateinit var fqNames: List<FqName>
private set
constructor(lookupSymbols: List<LookupSymbol>, fqNames: List<FqName>) : this() {
this.lookupSymbols = lookupSymbols
this.fqNames = fqNames
}
private fun writeObject(out: ObjectOutputStream) {
out.writeInt(lookupSymbols.size)
lookupSymbols.forEach {
out.writeUTF(it.name)
out.writeUTF(it.scope)
}
out.writeInt(fqNames.size)
fqNames.forEach {
out.writeUTF(it.asString())
}
}
private fun readObject(ois: ObjectInputStream) {
val lookupSymbolsSize = ois.readInt()
val lookupSymbols = ArrayList<LookupSymbol>(lookupSymbolsSize)
repeat(lookupSymbolsSize) {
val name = ois.readUTF()
val scope = ois.readUTF()
lookupSymbols.add(LookupSymbol(name, scope))
}
this.lookupSymbols = lookupSymbols
val fqNamesSize = ois.readInt()
val fqNames = ArrayList<FqName>(fqNamesSize)
repeat(fqNamesSize) {
val fqNameString = ois.readUTF()
fqNames.add(FqName(fqNameString))
}
this.fqNames = fqNames
}
companion object { companion object {
private const val serialVersionUID = 0L private const val serialVersionUID = 0L
} }
lateinit var lookupSymbols: LinkedHashSet<LookupSymbol>
private set
lateinit var fqNames: LinkedHashSet<FqName>
private set
constructor(lookupSymbols: LinkedHashSet<LookupSymbol>, fqNames: LinkedHashSet<FqName>) : this() {
this.lookupSymbols = lookupSymbols
this.fqNames = fqNames
}
private fun writeObject(output: ObjectOutputStream) {
// Can't close DataOutputStream below as it will also close the underlying ObjectOutputStream, which is still in use.
ClasspathChangesAvailableExternalizer.save(DataOutputStream(output), this)
}
private fun readObject(input: ObjectInputStream) {
// Can't close DataInputStream below as it will also close the underlying ObjectInputStream, which is still in use.
ClasspathChangesAvailableExternalizer.read(DataInputStream(input)).also {
lookupSymbols = it.lookupSymbols
fqNames = it.fqNames
}
}
} }
sealed class NotAvailable : ClasspathChanges() { sealed class NotAvailable : ClasspathChanges() {
object UnableToCompute : NotAvailable()
object ForNonIncrementalRun : NotAvailable() object ForNonIncrementalRun : NotAvailable()
object ClasspathSnapshotIsDisabled : NotAvailable() object ClasspathSnapshotIsDisabled : NotAvailable()
object ReservedForTestsOnly : NotAvailable() object ReservedForTestsOnly : NotAvailable()
object ForJSCompiler : NotAvailable() object ForJSCompiler : NotAvailable()
} }
} }
private object ClasspathChangesAvailableExternalizer : DataExternalizer<ClasspathChanges.Available> {
override fun save(output: DataOutput, classpathChanges: ClasspathChanges.Available) {
LinkedHashSetExternalizer(LookupSymbolExternalizer).save(output, classpathChanges.lookupSymbols)
LinkedHashSetExternalizer(FqNameExternalizer).save(output, classpathChanges.fqNames)
}
override fun read(input: DataInput): ClasspathChanges.Available {
return ClasspathChanges.Available(
lookupSymbols = LinkedHashSetExternalizer(LookupSymbolExternalizer).read(input),
fqNames = LinkedHashSetExternalizer(FqNameExternalizer).read(input)
)
}
}
@@ -23,24 +23,39 @@ import org.jetbrains.org.objectweb.asm.Opcodes
* - [NestedNonLocalClass] * - [NestedNonLocalClass]
* - [LocalClass] (https://docs.oracle.com/javase/tutorial/java/javaOO/localclasses.html) * - [LocalClass] (https://docs.oracle.com/javase/tutorial/java/javaOO/localclasses.html)
*/ */
sealed class JavaClassName( sealed class JavaClassName {
/** The full name of this class (e.g., "com/example/Foo$Bar"). */ /** The full name of this class (e.g., "com/example/Foo$Bar"). */
val name: String abstract val name: String
) {
/**
* Whether this class is an anonymous class.
*
* Note: Even though an anonymous class has no name in the source code, it always has a (not-null, not-empty) name in the compiled
* class (e.g., "com/example/Foo$1").
*/
abstract val isAnonymous: Boolean
/** Whether this class is a synthetic class. */
abstract val isSynthetic: Boolean
/** The package name of this class (e.g., "com/example"). */ /** The package name of this class (e.g., "com/example"). */
val packageName: String val packageName: String
get() = name.substringBeforeLast('/', "") get() = name.substringBeforeLast('/', missingDelimiterValue = "")
/** The part of the full name of this class after [packageName] (e.g., "Foo$Bar"). */
val relativeClassName: String
get() = name.substringAfterLast('/', missingDelimiterValue = name)
companion object { companion object {
/** Computes the [JavaClassName] of a compiled Java class given its contents. */
fun compute(classContents: ByteArray): JavaClassName { fun compute(classContents: ByteArray): JavaClassName {
val nameRef = Ref.create<String>() val nameRef = Ref.create<String>()
val isSyntheticRef = Ref.create<Boolean>()
val isTopLevelRef = Ref.create<Boolean>() val isTopLevelRef = Ref.create<Boolean>()
val outerNameRef = Ref.create<String>() val outerNameRef = Ref.create<String>()
val isAnonymousInnerClassRef = Ref.create<Boolean>() val isAnonymousRef = Ref.create<Boolean>()
val isSyntheticInnerClassRef = Ref.create<Boolean>()
ClassReader(classContents).accept(object : ClassVisitor(Opcodes.API_VERSION) { ClassReader(classContents).accept(object : ClassVisitor(Opcodes.API_VERSION) {
override fun visit( override fun visit(
@@ -48,120 +63,110 @@ sealed class JavaClassName(
signature: String?, superName: String?, interfaces: Array<String?>? signature: String?, superName: String?, interfaces: Array<String?>?
) { ) {
nameRef.set(name) nameRef.set(name)
isSyntheticRef.set((access and Opcodes.ACC_SYNTHETIC) != 0)
} }
override fun visitInnerClass(name: String, outerName: String?, innerName: String?, access: Int) { override fun visitInnerClass(name: String, outerName: String?, innerName: String?, access: Int) {
if (name == nameRef.get()!!) { if (name == nameRef.get()!!) {
isTopLevelRef.set(false) isTopLevelRef.set(false)
outerNameRef.set(outerName) outerNameRef.set(outerName)
isAnonymousInnerClassRef.set(innerName == null) isAnonymousRef.set(innerName == null)
isSyntheticInnerClassRef.set((access and Opcodes.ACC_SYNTHETIC) != 0)
} }
} }
}, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES) }, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES)
val name = nameRef.get()!! val name = nameRef.get()!!
val isSynthetic = isSyntheticRef.get()!!
val isTopLevel = isTopLevelRef.get() ?: true val isTopLevel = isTopLevelRef.get() ?: true
val outerName = outerNameRef.get() val outerName = outerNameRef.get()
val isAnonymous = isAnonymousRef.get()
return when { return when {
isTopLevel -> TopLevelClass(name) isTopLevel -> TopLevelClass(name, isSynthetic)
outerName != null -> NestedNonLocalClass( outerName != null -> NestedNonLocalClass(name, outerName, isAnonymous!!, isSynthetic)
name, else -> LocalClass(name, isAnonymous!!, isSynthetic)
outerName,
isAnonymousInnerClassRef.get()!!,
isSyntheticInnerClassRef.get()!!
)
else -> LocalClass(name)
} }
} }
} }
} }
/** See [JavaClassName]. */ /** See [JavaClassName]. */
class TopLevelClass(name: String) : JavaClassName(name) { class TopLevelClass(
override val name: String,
/** override val isSynthetic: Boolean
* The simple name of this class (e.g., the simple name of class "com/example/Foo" is "Foo", the simple name of class ) : JavaClassName() {
* "com/example/ClassWith$Sign" is "ClassWith$Sign"). override val isAnonymous: Boolean = false // A top-level class is never anonymous
*/
val simpleName: String
get() = name.substringAfterLast('/')
} }
/** See [JavaClassName]. */ /** See [JavaClassName]. */
sealed class NestedClass(name: String) : JavaClassName(name) sealed class NestedClass : JavaClassName()
/** See [JavaClassName]. */ /** See [JavaClassName]. */
class NestedNonLocalClass( class NestedNonLocalClass(
name: String, override val name: String,
/** /**
* The full name of the outer class of this class (e.g., the outer name of "com/example/OuterClass$NestedClass" is * The full name of the outer class of this class (e.g., the outer name of "com/example/OuterClass$NestedClass" is
* "com/example/OuterClass", the outer name of class "com/example/OuterClassWith$Sign$NestedClassWith$Sign" is * "com/example/OuterClass", the outer name of "com/example/OuterClassWith$Sign$NestedClassWith$Sign" is
* "com/example/OuterClassWith$Sign"). * "com/example/OuterClassWith$Sign").
* *
* The outer class can be of any type ([TopLevelClass], [NestedNonLocalClass], or [LocalClass]). * The outer class can be of any type ([TopLevelClass], [NestedNonLocalClass], or [LocalClass]).
*/ */
val outerName: String, val outerName: String,
/** override val isAnonymous: Boolean,
* Whether this class is an anonymous class. override val isSynthetic: Boolean
* ) : NestedClass() {
* Note: Even though an anonymous class has no name in the source code, it always has a (not-null, not-empty) name in the compiled
* class. Therefore, [simpleName] is not `null` and not empty even for anonymous classes.
* */
val isAnonymous: Boolean,
/** Whether this class is a synthetic class. */
val isSynthetic: Boolean
) : NestedClass(name) {
init {
check(name.startsWith("$outerName\$"))
}
/** /**
* The simple name of this class (e.g., the simple name of "com/example/OuterClass$NestedClass" is "NestedClass", the simple name of * The simple name of this class (e.g., the simple name of "com/example/OuterClass$NestedClass" is "NestedClass", the simple name of
* class "com/example/OuterClassWith$Sign$NestedClassWith$Sign" is "NestedClassWith$Sign"). * class "com/example/OuterClassWith$Sign$NestedClassWith$Sign" is "NestedClassWith$Sign").
* *
* Note: [simpleName] is not `null` and not empty even for anonymous classes (see [isAnonymous]). * Note: [simpleName] is not `null` and not empty even for anonymous classes (see [JavaClassName.isAnonymous]).
*/ */
val simpleName: String val simpleName: String
get() = name.substring("$outerName\$".length) get() = run {
check(name.startsWith("$outerName\$"))
name.substring("$outerName\$".length).also { check(it.isNotEmpty()) }
}
} }
/** See [JavaClassName]. */ /** See [JavaClassName]. */
class LocalClass(name: String) : NestedClass(name) class LocalClass(
override val name: String,
override val isAnonymous: Boolean,
override val isSynthetic: Boolean
) : NestedClass()
/** /**
* Computes [ClassId]s of the given Java classes. * Computes [ClassId]s of the given Java classes.
* *
* Note that creating a [ClassId] for a nested class will require accessing the outer class for 2 reasons: * Note that creating a [ClassId] for a nested class will require accessing the outer class for 2 reasons:
* - To disambiguate any '$' characters in the class name (e.g., "com/example/OuterClassWith$Sign$NestedClassWith$Sign"). * - To disambiguate any '$' characters in the (outer) class name (e.g., "com/example/OuterClassWith$Sign$NestedClassWith$Sign").
* - To determine whether a class is a local class (a nested class of a local class is also considered local, see [ClassId]'s kdoc). * - To determine whether a class is a local class (a nested class of a local class is also considered local, see [ClassId]'s kdoc).
* *
* Therefore, outer classes and nested classes must be passed together in one invocation of this method. * Therefore, outer classes and nested classes must be passed together in one invocation of this method.
*/ */
fun computeJavaClassIds(javaClassNames: List<JavaClassName>): List<ClassId> { fun computeJavaClassIds(classNames: List<JavaClassName>): List<ClassId> {
val nameToJavaClassName: Map<String, JavaClassName> = javaClassNames.associateBy { it.name } val classNameToClassId: MutableMap<JavaClassName, ClassId> = HashMap(classNames.size)
val nameToClassId: MutableMap<String, ClassId?> = nameToJavaClassName.mapValues { null }.toMutableMap() val nameToClassName: Map<String, JavaClassName> = classNames.associateBy { it.name }
fun getOrCreateClassId(className: String): ClassId { fun JavaClassName.getClassId(): ClassId {
val classInfo = nameToJavaClassName[className] ?: error("Class name not found: $className") classNameToClassId[this]?.let { return it }
val computedClassId = nameToClassId[className]
if (computedClassId != null) {
return computedClassId
}
val packageName = FqName(classInfo.packageName.replace('/', '.')) val packageName = FqName(packageName.replace('/', '.'))
val classId = when (classInfo) { val classId = when (this) {
is TopLevelClass -> { is TopLevelClass -> {
ClassId(packageName, FqName(classInfo.simpleName), /* local */ false) ClassId(packageName, FqName(relativeClassName), /* local */ false)
} }
is NestedNonLocalClass -> { is NestedNonLocalClass -> {
val outerClassId = getOrCreateClassId(classInfo.outerName) // JavaClassName.relativeClassName can contain '$' but not '.', whereas ClassId.relativeClassName can contain both '$' and
val relativeClassName = FqName(outerClassId.relativeClassName.asString() + "." + classInfo.simpleName) // '.' (e.g., "com/example/OuterClassWith$Sign$NestedClassWith$Sign" has JavaClassName.relativeClassName
// "OuterClassWith$Sign$NestedClassWith$Sign", but its ClassId.relativeClassName will be
// "OuterClassWith$Sign.NestedClassWith$Sign". To disambiguate '$' in the (outer) class name, we need to get the ClassId of
// the outer class first.
val outerClassId = nameToClassName[outerName]?.getClassId() ?: error("Class name not found: $outerName")
val relativeClassName = FqName(outerClassId.relativeClassName.asString() + "." + simpleName)
// For ClassId, a nested non-local class of a local class is also considered local (see ClassId's kdoc). // For ClassId, a nested non-local class of a local class is also considered local (see ClassId's kdoc).
val isLocal = outerClassId.isLocal val isLocal = outerClassId.isLocal
@@ -180,7 +185,7 @@ fun computeJavaClassIds(javaClassNames: List<JavaClassName>): List<ClassId> {
// } // }
// } // }
// } // }
// The above class will compile into class "com/example/Foo" and "com/example/Foo$1Bar" (or // The above source will compile into class "com/example/Foo" and "com/example/Foo$1Bar" (or
// "com/example/Foo$SomeOtherArbitraryUniqueName which need not contain the string "Bar"). // "com/example/Foo$SomeOtherArbitraryUniqueName which need not contain the string "Bar").
// //
// Given that class, the difference between the computed ClassId and the expected ClassId is as follows: // Given that class, the difference between the computed ClassId and the expected ClassId is as follows:
@@ -196,15 +201,14 @@ fun computeJavaClassIds(javaClassNames: List<JavaClassName>): List<ClassId> {
// //
// Alternatively, check if we can safely adjust the definition of ClassId for a local class and update any related code // Alternatively, check if we can safely adjust the definition of ClassId for a local class and update any related code
// accordingly. // accordingly.
val relativeClassName = FqName(classInfo.name.substringAfterLast('/')) ClassId(packageName, FqName(relativeClassName), /* local */ true)
ClassId(packageName, relativeClassName, /* local */ true)
} }
} }
return classId.also { return classId.also {
nameToClassId[className] = it classNameToClassId[this] = it
} }
} }
return javaClassNames.map { getOrCreateClassId(it.name) } return classNames.map { it.getClassId() }
} }
@@ -24,6 +24,9 @@ import com.intellij.util.io.IOUtil
import com.intellij.util.io.KeyDescriptor import com.intellij.util.io.KeyDescriptor
import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.cli.common.toBooleanLenient
import org.jetbrains.kotlin.incremental.LookupSymbol
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import java.io.DataInput import java.io.DataInput
import java.io.DataInputStream import java.io.DataInputStream
import java.io.DataOutput import java.io.DataOutput
@@ -71,6 +74,46 @@ object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2 override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2
} }
object LookupSymbolExternalizer : DataExternalizer<LookupSymbol> {
override fun save(output: DataOutput, lookupSymbol: LookupSymbol) {
output.writeString(lookupSymbol.name)
output.writeString(lookupSymbol.scope)
}
override fun read(input: DataInput): LookupSymbol {
return LookupSymbol(name = input.readString(), scope = input.readString())
}
}
object FqNameExternalizer : DataExternalizer<FqName> {
override fun save(output: DataOutput, fqName: FqName) {
output.writeString(fqName.asString())
}
override fun read(input: DataInput): FqName {
return FqName(input.readString())
}
}
object ClassIdExternalizer : DataExternalizer<ClassId> {
override fun save(output: DataOutput, classId: ClassId) {
FqNameExternalizer.save(output, classId.packageFqName)
FqNameExternalizer.save(output, classId.relativeClassName)
output.writeBoolean(classId.isLocal)
}
override fun read(input: DataInput): ClassId {
return ClassId(
/* packageFqName */ FqNameExternalizer.read(input),
/* relativeClassName */ FqNameExternalizer.read(input),
/* isLocal */ input.readBoolean()
)
}
}
object ProtoMapValueExternalizer : DataExternalizer<ProtoMapValue> { object ProtoMapValueExternalizer : DataExternalizer<ProtoMapValue> {
override fun save(output: DataOutput, value: ProtoMapValue) { override fun save(output: DataOutput, value: ProtoMapValue) {
output.writeBoolean(value.isPackageFacade) output.writeBoolean(value.isPackageFacade)
@@ -210,6 +253,17 @@ object PathStringDescriptor : EnumeratorStringDescriptor() {
} }
} }
/**
* [DataExternalizer] for a [Collection].
*
* If you need a [DataExternalizer] for a more specific instance of [Collection] (e.g., [List]), use [ListExternalizer] or create another
* instance of [GenericCollectionExternalizer].
*
* Note: The implementations of this class and [GenericCollectionExternalizer] are similar but not exactly the same: the latter reads and
* writes the size of the collection to avoid resizing the collection when reading. Therefore, if we make this class extend
* [GenericCollectionExternalizer] to share code, we will need to update some expected files in tests as the serialized data will change
* slightly.
*/
open class CollectionExternalizer<T>( open class CollectionExternalizer<T>(
private val elementExternalizer: DataExternalizer<T>, private val elementExternalizer: DataExternalizer<T>,
private val newCollection: () -> MutableCollection<T> private val newCollection: () -> MutableCollection<T>
@@ -269,27 +323,35 @@ object ByteArrayExternalizer : DataExternalizer<ByteArray> {
} }
} }
class ListExternalizer<T>( open class GenericCollectionExternalizer<T, C : Collection<T>>(
private val elementExternalizer: DataExternalizer<T> private val elementExternalizer: DataExternalizer<T>,
) : DataExternalizer<List<T>> { private val newCollection: (size: Int) -> MutableCollection<T>
) : DataExternalizer<C> {
override fun save(output: DataOutput, value: List<T>) { override fun save(output: DataOutput, collection: C) {
output.writeInt(value.size) output.writeInt(collection.size)
value.forEach { collection.forEach {
elementExternalizer.save(output, it) elementExternalizer.save(output, it)
} }
} }
override fun read(input: DataInput): List<T> { override fun read(input: DataInput): C {
val size = input.readInt() val size = input.readInt()
val list = ArrayList<T>(size) val collection = newCollection(size)
repeat(size) { repeat(size) {
list.add(elementExternalizer.read(input)) collection.add(elementExternalizer.read(input))
} }
return list @Suppress("UNCHECKED_CAST")
return collection as C
} }
} }
class ListExternalizer<T>(elementExternalizer: DataExternalizer<T>) :
GenericCollectionExternalizer<T, List<T>>(elementExternalizer, { size -> ArrayList(size) })
class LinkedHashSetExternalizer<T>(elementExternalizer: DataExternalizer<T>) :
GenericCollectionExternalizer<T, LinkedHashSet<T>>(elementExternalizer, { size -> LinkedHashSet(size) })
class LinkedHashMapExternalizer<K, V>( class LinkedHashMapExternalizer<K, V>(
private val keyExternalizer: DataExternalizer<K>, private val keyExternalizer: DataExternalizer<K>,
private val valueExternalizer: DataExternalizer<V> private val valueExternalizer: DataExternalizer<V>
@@ -47,7 +47,6 @@ import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
import org.jetbrains.kotlin.incremental.util.Either import org.jetbrains.kotlin.incremental.util.Either
import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.UnableToCompute
import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForJSCompiler import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForJSCompiler
import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ReservedForTestsOnly import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ReservedForTestsOnly
import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForNonIncrementalRun import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForNonIncrementalRun
@@ -96,14 +95,12 @@ fun makeIncrementally(
} }
object EmptyICReporter : ICReporterBase() { object EmptyICReporter : ICReporterBase() {
override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection<File>, exitCode: ExitCode) { override fun report(message: () -> String) {}
} override fun reportVerbose(message: () -> String) {}
override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection<File>, exitCode: ExitCode) {}
override fun report(message: () -> String) { override fun reportMarkDirtyClass(affectedFiles: Iterable<File>, classFqName: String) {}
} override fun reportMarkDirtyMember(affectedFiles: Iterable<File>, scope: String, name: String) {}
override fun reportMarkDirty(affectedFiles: Iterable<File>, reason: String) {}
override fun reportVerbose(message: () -> String) {
}
} }
inline fun <R> withIC(enabled: Boolean = true, fn: () -> R): R { inline fun <R> withIC(enabled: Boolean = true, fn: () -> R): R {
@@ -218,10 +215,10 @@ class IncrementalJvmCompilerRunner(
reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" } reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
val classpathChanges = when (classpathChanges) { val classpathChanges = when (classpathChanges) {
// Note: classpathChanges is deserialized so they are no longer singleton objects and need to be compared using `is` (not `==`). // Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`)
is ClasspathChanges.Available -> ChangesEither.Known(classpathChanges.lookupSymbols, classpathChanges.fqNames) is ClasspathChanges.Available -> ChangesEither.Known(classpathChanges.lookupSymbols, classpathChanges.fqNames)
is ClasspathChanges.NotAvailable -> when (classpathChanges) { is ClasspathChanges.NotAvailable -> when (classpathChanges) {
is UnableToCompute, is ClasspathSnapshotIsDisabled, is ReservedForTestsOnly -> { is ClasspathSnapshotIsDisabled, is ReservedForTestsOnly -> {
reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
val scopes = caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct() val scopes = caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct()
getClasspathChanges( getClasspathChanges(
@@ -148,7 +148,24 @@ class IncrementalCompilationFirJvmMultiProjectIT : IncrementalCompilationJvmMult
} }
class IncrementalCompilationClasspathSnapshotJvmMultiProjectIT : IncrementalCompilationJvmMultiProjectIT() { class IncrementalCompilationClasspathSnapshotJvmMultiProjectIT : IncrementalCompilationJvmMultiProjectIT() {
override fun defaultBuildOptions() = super.defaultBuildOptions().copy(useClasspathSnapshot = true) override fun defaultBuildOptions() = super.defaultBuildOptions().copy(useClasspathSnapshot = true)
override fun testAddDependencyInLib_expectedFiles(project: Project): Iterable<File> {
// With classpath snapshot, no files are recompiled
return emptyList()
}
override fun testAbiChangeInLib_afterLibClean_expectedFiles(project: Project): Iterable<File> {
// With classpath snapshot, app compilation is incremental
return File(project.projectDir, "app").getFilesByNames("AA.kt", "AAA.kt", "BB.kt", "fooUseA.kt") +
File(project.projectDir, "lib").allKotlinFiles()
}
override fun testCompileLibWithGroovy_expectedFiles(project: Project): Iterable<File> {
// With classpath snapshot, no files in app are recompiled
return listOf(File(project.projectDir, "lib").getFileByName("A.kt"))
}
} }
abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilationBaseIT() { abstract class BaseIncrementalCompilationMultiProjectIT : IncrementalCompilationBaseIT() {
@@ -20,7 +20,26 @@ open class IncrementalJavaChangeDefaultIT : IncrementalCompilationJavaChangesBas
} }
class IncrementalJavaChangeClasspathSnapshotIT : IncrementalJavaChangeDefaultIT() { class IncrementalJavaChangeClasspathSnapshotIT : IncrementalJavaChangeDefaultIT() {
override fun defaultBuildOptions() = super.defaultBuildOptions().copy(useClasspathSnapshot = true) override fun defaultBuildOptions() = super.defaultBuildOptions().copy(useClasspathSnapshot = true)
@Test
override fun testAbiChangeInLib_changeMethodSignature() {
// With classpath snapshot, fewer Kotlin files are recompiled
doTest(
javaClass, changeSignature,
expectedCompiledFileNames = listOf("JavaClassChild.kt", "useJavaClass.kt")
)
}
@Test
override fun testNonAbiChangeInLib_changeMethodBody() {
// With classpath snapshot, no Kotlin files are recompiled
doTest(
javaClass, changeBody,
expectedCompiledFileNames = emptyList()
)
}
} }
class IncrementalJavaChangePreciseIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = true) { class IncrementalJavaChangePreciseIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = true) {
@@ -70,12 +89,12 @@ abstract class IncrementalCompilationJavaChangesBase(val usePreciseJavaTracking:
override fun defaultBuildOptions() = super.defaultBuildOptions().copy(usePreciseJavaTracking = usePreciseJavaTracking) override fun defaultBuildOptions() = super.defaultBuildOptions().copy(usePreciseJavaTracking = usePreciseJavaTracking)
protected val trackedJavaClass = "TrackedJavaClass.java" protected val trackedJavaClass = "TrackedJavaClass.java"
private val javaClass = "JavaClass.java" protected val javaClass = "JavaClass.java"
protected val changeBody: (String) -> String = { it.replace("Hello, World!", "Hello, World!!!!") } protected val changeBody: (String) -> String = { it.replace("Hello, World!", "Hello, World!!!!") }
protected val changeSignature: (String) -> String = { it.replace("String getString", "Object getString") } protected val changeSignature: (String) -> String = { it.replace("String getString", "Object getString") }
@Test @Test
fun testAbiChangeInLib_changeMethodSignature() { open fun testAbiChangeInLib_changeMethodSignature() {
doTest( doTest(
javaClass, changeSignature, javaClass, changeSignature,
expectedCompiledFileNames = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt") expectedCompiledFileNames = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
@@ -83,7 +102,7 @@ abstract class IncrementalCompilationJavaChangesBase(val usePreciseJavaTracking:
} }
@Test @Test
fun testNonAbiChangeInLib_changeMethodBody() { open fun testNonAbiChangeInLib_changeMethodBody() {
doTest( doTest(
javaClass, changeBody, javaClass, changeBody,
expectedCompiledFileNames = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt") expectedCompiledFileNames = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
@@ -5,184 +5,94 @@
package org.jetbrains.kotlin.gradle.incremental package org.jetbrains.kotlin.gradle.incremental
import com.google.common.annotations.VisibleForTesting
import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.io.FileUtil
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.*
import java.io.File
import org.jetbrains.kotlin.gradle.incremental.ChangesCollectorResult.Success
import org.jetbrains.kotlin.gradle.incremental.ChangesCollectorResult.Failure
import org.jetbrains.kotlin.incremental.storage.FileToCanonicalPathConverter import org.jetbrains.kotlin.incremental.storage.FileToCanonicalPathConverter
import java.util.* import java.util.*
import kotlin.collections.LinkedHashMap
/** Computes [ClasspathChanges] between two [ClasspathSnapshot]s .*/ /** Computes [ClasspathChanges] between two [ClasspathSnapshot]s .*/
object ClasspathChangesComputer { object ClasspathChangesComputer {
fun getChanges(current: ClasspathSnapshot, previous: ClasspathSnapshot): ClasspathChanges { fun compute(currentClasspathSnapshot: ClasspathSnapshot, previousClasspathSnapshot: ClasspathSnapshot): ClasspathChanges {
val changesCollector = ChangesCollector() val currentClassSnapshots = currentClasspathSnapshot.getClassSnapshots()
return when (collectClasspathChanges(current, previous, changesCollector)) { val previousClassSnapshots = previousClasspathSnapshot.getClassSnapshots()
Success -> {
val (lookupSymbols, fqNames, _) = changesCollector.getDirtyData(emptyList(), NoOpBuildReporter)
ClasspathChanges.Available(lookupSymbols.toList(), fqNames.toList())
}
is Failure -> ClasspathChanges.NotAvailable.UnableToCompute
}
}
private fun collectClasspathChanges(
current: ClasspathSnapshot,
previous: ClasspathSnapshot,
changesCollector: ChangesCollector
): ChangesCollectorResult {
if (current.classpathEntrySnapshots.size != previous.classpathEntrySnapshots.size) {
return Failure.AddedRemovedClasspathEntries
}
for (index in current.classpathEntrySnapshots.indices) {
val result = collectClasspathEntryChanges(
current.classpathEntrySnapshots[index],
previous.classpathEntrySnapshots[index],
changesCollector
)
if (result is Failure) {
return result
}
}
return Success
}
private fun collectClasspathEntryChanges(
current: ClasspathEntrySnapshot,
previous: ClasspathEntrySnapshot,
changesCollector: ChangesCollector
): ChangesCollectorResult {
if (current.classSnapshots.size != previous.classSnapshots.size) {
return Failure.AddedRemovedClasses
}
for (key in current.classSnapshots.keys) {
val currentSnapshot = current.classSnapshots[key]!!
val previousSnapshot = previous.classSnapshots[key] ?: return Failure.AddedRemovedClasses
val result = collectClassChanges(currentSnapshot, previousSnapshot, changesCollector)
if (result !is Success) {
return result
}
}
return Success
}
private fun collectClassChanges(
current: ClassSnapshot,
previous: ClassSnapshot,
@Suppress("UNUSED_PARAMETER") changesCollector: ChangesCollector
): ChangesCollectorResult {
if (current is JavaClassSnapshot && previous is JavaClassSnapshot &&
(current !is RegularJavaClassSnapshot || previous !is RegularJavaClassSnapshot)
) {
return Failure.NotYetImplemented
}
// TODO: Store results in changesCollector and return SUCCESS here
computeClassChanges(current, previous)
return Failure.NotYetImplemented
}
@VisibleForTesting
internal fun computeClassChanges(current: ClassSnapshot, previous: ClassSnapshot): DirtyData {
// TODO Create IncrementalJvmCache early once and reuse it here
val workingDir = val workingDir =
FileUtil.createTempDirectory(this::class.java.simpleName, "_WorkingDir_${UUID.randomUUID()}", /* deleteOnExit */ true) FileUtil.createTempDirectory(this::class.java.simpleName, "_WorkingDir_${UUID.randomUUID()}", /* deleteOnExit */ true)
val incrementalJvmCache = IncrementalJvmCache(workingDir, /* targetOutputDir */ null, FileToCanonicalPathConverter) val incrementalJvmCache = IncrementalJvmCache(workingDir, /* targetOutputDir */ null, FileToCanonicalPathConverter)
val changesCollector = ChangesCollector()
when { // Store previous class snapshots in incrementalJvmCache, the returned ChangesCollector result is not used.
current is KotlinClassSnapshot && previous is KotlinClassSnapshot -> val unusedChangesCollector = ChangesCollector()
collectKotlinClassChanges(current, previous, incrementalJvmCache, changesCollector) for (previousSnapshot in previousClassSnapshots) {
current is JavaClassSnapshot && previous is JavaClassSnapshot -> when (previousSnapshot) {
collectJavaClassChanges(current, previous, incrementalJvmCache, changesCollector) is KotlinClassSnapshot -> incrementalJvmCache.saveClassToCache(
else -> { kotlinClassInfo = previousSnapshot.classInfo,
// TODO: Handle current is KotlinClassSnapshot && previous is JavaClassSnapshot, and vice versa sourceFiles = null,
error("Incompatible types: ${current.javaClass.name} vs. ${previous.javaClass.name}") changesCollector = unusedChangesCollector
)
is RegularJavaClassSnapshot -> incrementalJvmCache.saveJavaClassProto(
source = null,
serializedJavaClass = previousSnapshot.serializedJavaClass,
collector = unusedChangesCollector
)
is EmptyJavaClassSnapshot -> {
// Nothing to process
}
} }
} }
// Call the following method even though there are no removed classes, just in case the method updates the state of
// incrementalJvmCache.
incrementalJvmCache.clearCacheForRemovedClasses(unusedChangesCollector)
// Compute changes between the current class snapshots and the previously stored snapshots, and save the result in changesCollector.
val changesCollector = ChangesCollector()
for (currentSnapshot in currentClassSnapshots) {
when (currentSnapshot) {
is KotlinClassSnapshot -> incrementalJvmCache.saveClassToCache(
kotlinClassInfo = currentSnapshot.classInfo,
sourceFiles = null,
changesCollector = changesCollector
)
is RegularJavaClassSnapshot -> incrementalJvmCache.saveJavaClassProto(
source = null,
serializedJavaClass = currentSnapshot.serializedJavaClass,
collector = changesCollector
)
is EmptyJavaClassSnapshot -> {
// Nothing to process
}
}
}
incrementalJvmCache.clearCacheForRemovedClasses(changesCollector)
val dirtyData = changesCollector.getDirtyData(listOf(incrementalJvmCache), EmptyICReporter)
workingDir.deleteRecursively() workingDir.deleteRecursively()
return changesCollector.getDirtyData(listOf(incrementalJvmCache), NoOpBuildReporter.NoOpICReporter)
}
private fun collectKotlinClassChanges( return ClasspathChanges.Available(
current: KotlinClassSnapshot, lookupSymbols = LinkedHashSet(dirtyData.dirtyLookupSymbols),
previous: KotlinClassSnapshot, fqNames = LinkedHashSet(dirtyData.dirtyClassesFqNames)
incrementalJvmCache: IncrementalJvmCache,
changesCollector: ChangesCollector
) {
// Store previous snapshot in incrementalJvmCache, the returned ChangesCollector result is not used.
incrementalJvmCache.saveClassToCache(
kotlinClassInfo = previous.classInfo,
sourceFiles = null,
changesCollector = ChangesCollector()
) )
incrementalJvmCache.clearCacheForRemovedClasses(changesCollector)
// Compute changes between the current snapshot and the previously stored snapshot, and store the result in changesCollector.
incrementalJvmCache.saveClassToCache(
kotlinClassInfo = current.classInfo,
sourceFiles = null,
changesCollector = changesCollector
)
incrementalJvmCache.clearCacheForRemovedClasses(changesCollector)
} }
private fun collectJavaClassChanges( private fun ClasspathSnapshot.getClassSnapshots(): List<ClassSnapshot> {
current: JavaClassSnapshot, // If there are duplicate classes on the classpath, retain only the first one to match the compiler's behavior.
previous: JavaClassSnapshot, // We still need to consider whether to remove duplicate classes based on the class file name or the `ClassId`, as two different
incrementalJvmCache: IncrementalJvmCache, // `ClassId`s can have the same class file name (e.g., nested class `B$C` of top-level class `A` in `first.jar` and nested class `C`
changesCollector: ChangesCollector // of top-level class `A$B` in `second.jar` both have the same class file name `A$B$C`).
) { // - If we use class file name, only `A$B$C` in `first.jar` will be retained. This matches the compiler's behavior because when
// Store previous snapshot in incrementalJvmCache, the returned ChangesCollector result is not used. // resolving either of those classes, the compiler will only look for `A$B$C` in the first jar, even when the actual class is
val previousSnapshot = (previous as RegularJavaClassSnapshot).serializedJavaClass // TODO Handle unsafe cast // located in the second jar.
incrementalJvmCache.saveJavaClassProto(/* source */ null, previousSnapshot, ChangesCollector()) // - If we use `ClassId`, both `A$B$C` in `first.jar` and `A$B$C` in `second.jar` will be retained. That means that the snapshot
incrementalJvmCache.clearCacheForRemovedClasses(changesCollector) // of the class in `second.jar` will be considered when computing classpath changes, which is not expected as it doesn't match
// the compiler's behavior.
// Compute changes between the current snapshot and the previously stored snapshot, and store the result in changesCollector. // Therefore, we will remove duplicate classes based on the class file name, not the `ClassId`.
val currentSnapshot = (current as RegularJavaClassSnapshot).serializedJavaClass val classSnapshots = LinkedHashMap<String, ClassSnapshot>(classpathEntrySnapshots.sumOf { it.classSnapshots.size })
incrementalJvmCache.saveJavaClassProto(/* source */ null, currentSnapshot, changesCollector) for (classpathEntrySnapshot in classpathEntrySnapshots) {
incrementalJvmCache.clearCacheForRemovedClasses(changesCollector) for ((unixStyleRelativePath, classSnapshot) in classpathEntrySnapshot.classSnapshots) {
} classSnapshots.putIfAbsent(unixStyleRelativePath, classSnapshot)
} }
}
private sealed class ChangesCollectorResult { return classSnapshots.values.toList()
object Success : ChangesCollectorResult()
sealed class Failure : ChangesCollectorResult() {
// TODO: Handle these cases
object AddedRemovedClasspathEntries : Failure()
object AddedRemovedClasses : Failure()
object NotYetImplemented : Failure()
}
}
private object NoOpBuildReporter : BuildReporter(NoOpICReporter, NoOpBuildMetricsReporter) {
object NoOpICReporter : ICReporter {
override fun report(message: () -> String) {}
override fun reportVerbose(message: () -> String) {}
override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection<File>, exitCode: ExitCode) {}
override fun reportMarkDirtyClass(affectedFiles: Iterable<File>, classFqName: String) {}
override fun reportMarkDirtyMember(affectedFiles: Iterable<File>, scope: String, name: String) {}
override fun reportMarkDirty(affectedFiles: Iterable<File>, reason: String) {}
}
object NoOpBuildMetricsReporter : BuildMetricsReporter {
override fun startMeasure(time: BuildTime, startNs: Long) {}
override fun endMeasure(time: BuildTime, endNs: Long) {}
override fun addTimeMetric(metric: BuildTime, durationMs: Long) {}
override fun addMetric(metric: BuildPerformanceMetric, value: Long) {}
override fun addAttribute(attribute: BuildAttribute) {}
override fun getMetrics(): BuildMetrics = BuildMetrics()
override fun addMetrics(metrics: BuildMetrics?) {}
} }
} }
@@ -43,9 +43,7 @@ class KotlinClassSnapshot(val classInfo: KotlinClassInfo) : ClassSnapshot()
sealed class JavaClassSnapshot : ClassSnapshot() sealed class JavaClassSnapshot : ClassSnapshot()
/** [JavaClassSnapshot] of a typical Java class. */ /** [JavaClassSnapshot] of a typical Java class. */
class RegularJavaClassSnapshot( class RegularJavaClassSnapshot(val serializedJavaClass: SerializedJavaClass) : JavaClassSnapshot()
val serializedJavaClass: SerializedJavaClass
) : JavaClassSnapshot()
/** /**
* [JavaClassSnapshot] of a Java class where there is nothing to capture. * [JavaClassSnapshot] of a Java class where there is nothing to capture.
@@ -10,8 +10,6 @@ import org.jetbrains.kotlin.incremental.JavaClassProtoMapValueExternalizer
import org.jetbrains.kotlin.incremental.KotlinClassInfo import org.jetbrains.kotlin.incremental.KotlinClassInfo
import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import java.io.* import java.io.*
/** Utility to serialize a [ClasspathSnapshot]. */ /** Utility to serialize a [ClasspathSnapshot]. */
@@ -93,34 +91,6 @@ object KotlinClassInfoExternalizer : DataExternalizer<KotlinClassInfo> {
} }
} }
object ClassIdExternalizer : DataExternalizer<ClassId> {
override fun save(output: DataOutput, classId: ClassId) {
FqNameExternalizer.save(output, classId.packageFqName)
FqNameExternalizer.save(output, classId.relativeClassName)
output.writeBoolean(classId.isLocal)
}
override fun read(input: DataInput): ClassId {
return ClassId(
/* packageFqName */ FqNameExternalizer.read(input),
/* relativeClassName */ FqNameExternalizer.read(input),
/* isLocal */ input.readBoolean()
)
}
}
object FqNameExternalizer : DataExternalizer<FqName> {
override fun save(output: DataOutput, fqName: FqName) {
output.writeString(fqName.asString())
}
override fun read(input: DataInput): FqName {
return FqName(input.readString())
}
}
object JavaClassSnapshotExternalizer : DataExternalizer<JavaClassSnapshot> { object JavaClassSnapshotExternalizer : DataExternalizer<JavaClassSnapshot> {
override fun save(output: DataOutput, snapshot: JavaClassSnapshot) { override fun save(output: DataOutput, snapshot: JavaClassSnapshot) {
@@ -166,39 +136,27 @@ object EmptyJavaClassSnapshotExternalizer : DataExternalizer<EmptyJavaClassSnaps
interface DataSerializer<T> : DataExternalizer<T> { interface DataSerializer<T> : DataExternalizer<T> {
fun save(file: File, value: T) { fun save(file: File, value: T) {
return FileOutputStream(file).buffered().use { return DataOutputStream(FileOutputStream(file).buffered()).use {
it.writeValue(value) save(it, value)
} }
} }
fun load(file: File): T { fun load(file: File): T {
return FileInputStream(file).buffered().use { return DataInputStream(FileInputStream(file).buffered()).use {
it.readValue() read(it)
} }
} }
fun toByteArray(value: T): ByteArray { fun toByteArray(value: T): ByteArray {
val byteArrayOutputStream = ByteArrayOutputStream() val byteArrayOutputStream = ByteArrayOutputStream()
byteArrayOutputStream.buffered().use { DataOutputStream(byteArrayOutputStream.buffered()).use {
it.writeValue(value) save(it, value)
} }
return byteArrayOutputStream.toByteArray() return byteArrayOutputStream.toByteArray()
} }
fun fromByteArray(byteArray: ByteArray): T { fun fromByteArray(byteArray: ByteArray): T {
return ByteArrayInputStream(byteArray).buffered().use { return DataInputStream(ByteArrayInputStream(byteArray).buffered()).use {
it.readValue()
}
}
private fun OutputStream.writeValue(value: T) {
DataOutputStream(this).use {
save(it, value)
}
}
private fun InputStream.readValue(): T {
return DataInputStream(this).use {
read(it) read(it)
} }
} }
@@ -30,8 +30,7 @@ object ClasspathEntrySnapshotter {
val snapshots = ClassSnapshotter.snapshot(classes) val snapshots = ClassSnapshotter.snapshot(classes)
val relativePathsToSnapshotsMap = val relativePathsToSnapshotsMap = classes.map { it.classFile.unixStyleRelativePath }.zipToMap(snapshots)
classes.map { it.classFile.unixStyleRelativePath }.zip(snapshots).toMap(LinkedHashMap())
return ClasspathEntrySnapshot(relativePathsToSnapshotsMap) return ClasspathEntrySnapshot(relativePathsToSnapshotsMap)
} }
} }
@@ -48,17 +47,16 @@ object ClassSnapshotter {
*/ */
fun snapshot(classes: List<ClassFileWithContents>): List<ClassSnapshot> { fun snapshot(classes: List<ClassFileWithContents>): List<ClassSnapshot> {
// Snapshot Kotlin classes first // Snapshot Kotlin classes first
val kotlinClassSnapshots: Map<ClassFile, KotlinClassSnapshot?> = classes.associate { val kotlinClassSnapshots: Map<ClassFileWithContents, KotlinClassSnapshot?> = classes.associateWith {
it.classFile to trySnapshotKotlinClass(it) trySnapshotKotlinClass(it)
} }
// Snapshot Java classes in one invocation // Snapshot the remaining Java classes in one invocation
val javaClasses: List<ClassFileWithContents> = classes.filter { kotlinClassSnapshots[it.classFile] == null } val javaClasses: List<ClassFileWithContents> = classes.filter { kotlinClassSnapshots[it] == null }
val snapshots: List<JavaClassSnapshot> = snapshotJavaClasses(javaClasses) val snapshots: List<JavaClassSnapshot> = snapshotJavaClasses(javaClasses)
val javaClassSnapshots: Map<ClassFile, JavaClassSnapshot> = javaClasses.map { it.classFile }.zip(snapshots).toMap() val javaClassSnapshots: Map<ClassFileWithContents, JavaClassSnapshot> = javaClasses.zipToMap(snapshots)
// Return a snapshot for each class return classes.map { kotlinClassSnapshots[it] ?: javaClassSnapshots[it]!! }
return classes.map { kotlinClassSnapshots[it.classFile] ?: javaClassSnapshots[it.classFile]!! }
} }
/** Creates [KotlinClassSnapshot] of the given class, or returns `null` if the class is not a Kotlin class. */ /** Creates [KotlinClassSnapshot] of the given class, or returns `null` if the class is not a Kotlin class. */
@@ -75,44 +73,56 @@ object ClassSnapshotter {
* Therefore, outer classes and nested classes must be passed together in one invocation of this method. * Therefore, outer classes and nested classes must be passed together in one invocation of this method.
*/ */
private fun snapshotJavaClasses(classes: List<ClassFileWithContents>): List<JavaClassSnapshot> { private fun snapshotJavaClasses(classes: List<ClassFileWithContents>): List<JavaClassSnapshot> {
val classFiles = classes.map { it.classFile } val classNames: List<JavaClassName> = classes.map { JavaClassName.compute(it.contents) }
val classesContents = classes.map { it.contents } val classNameToClassFile: LinkedHashMap<JavaClassName, ClassFileWithContents> = classNames.zipToMap(classes)
val classNames = classesContents.map { JavaClassName.compute(it) }
val classIds = computeJavaClassIds(classNames)
// Snapshot special cases first // We divide classes into 2 categories:
// Map a class index to its snapshot, or `null` if it will be created later // - Special classes, which includes local, anonymous, or synthetic classes, and their nested classes. These classes can't be
val specialCaseSnapshots: Map<Int, JavaClassSnapshot?> = classFiles.indices.associateWith { index -> // referenced from other source files, so any changes in these classes will not cause recompilation of other source files.
val className = classNames[index] // Therefore, the snapshots of these classes are empty.
val classId = classIds[index] // - Regular classes: Any classes that do not belong to the above category.
if (classId.isLocal) { val specialClasses = getSpecialClasses(classNames).toSet()
// A local class can't be referenced from other source files, so any changes in a local class will not cause recompilation
// of other source files. Therefore, the snapshot of a local class is empty. // Snapshot special classes first
// In that regard, a nested class of a local class is also considered local (which matches the definition of val specialClassSnapshots: Map<JavaClassName, JavaClassSnapshot?> = classNames.associateWith {
// ClassId.isLocal, see ClassId's kdoc). Therefore, we checked `classId.isLocal`, which is a super set of `className is if (it in specialClasses) {
// LocalClass`.
EmptyJavaClassSnapshot EmptyJavaClassSnapshot
} else if (className is NestedNonLocalClass && (className.isAnonymous || className.isSynthetic)) { } else null
// An anonymous or synthetic class also can't be referenced from other source files, so its snapshot is also empty. }
EmptyJavaClassSnapshot
} else { // Snapshot the remaining regular classes in one invocation
null val regularClasses: List<JavaClassName> = classNames.filter { specialClassSnapshots[it] == null }
val regularClassIds: List<ClassId> = computeJavaClassIds(regularClasses)
val regularClassesContents: List<ByteArray> = regularClasses.map { classNameToClassFile[it]!!.contents }
val snapshots: List<RegularJavaClassSnapshot> = JavaClassDescriptorCreator.create(regularClassIds, regularClassesContents).map {
RegularJavaClassSnapshot(it.toSerializedJavaClass())
}
val regularClassSnapshots: LinkedHashMap<JavaClassName, JavaClassSnapshot> = regularClasses.zipToMap(snapshots)
return classNames.map { specialClassSnapshots[it] ?: regularClassSnapshots[it]!! }
}
/** Returns local, anonymous, or synthetic classes, and their nested classes. */
private fun getSpecialClasses(classNames: List<JavaClassName>): List<JavaClassName> {
val specialClasses: MutableMap<JavaClassName, Boolean> = HashMap(classNames.size)
val nameToClassName: Map<String, JavaClassName> = classNames.associateBy { it.name }
fun JavaClassName.isSpecial(): Boolean {
specialClasses[this]?.let { return it }
return if (isAnonymous || isSynthetic) {
true
} else when (this) {
is TopLevelClass -> false
is NestedNonLocalClass -> nameToClassName[outerName]?.isSpecial() ?: error("Class name not found: $outerName")
is LocalClass -> true
}.also {
specialClasses[this] = it
} }
} }
// Snapshot the remaining classes in one invocation return classNames.filter { it.isSpecial() }
val remainingClassesIndices: List<Int> = classFiles.indices.filter { specialCaseSnapshots[it] == null }
val remainingClassIds: List<ClassId> = remainingClassesIndices.map { classIds[it] }
val remainingClassesContents: List<ByteArray> = remainingClassesIndices.map { classes[it].contents }
val snapshots: List<JavaClassSnapshot> = JavaClassDescriptorCreator.create(remainingClassIds, remainingClassesContents).map {
RegularJavaClassSnapshot(it.toSerializedJavaClass())
}
val remainingSnapshots: Map<Int, JavaClassSnapshot> /* maps a class index to its snapshot */ =
remainingClassesIndices.zip(snapshots).toMap()
// Return a snapshot for each class
return classFiles.indices.map { specialCaseSnapshots[it] ?: remainingSnapshots[it]!! }
} }
} }
@@ -120,14 +130,15 @@ object ClassSnapshotter {
private object DirectoryOrJarContentsReader { private object DirectoryOrJarContentsReader {
/** /**
* Returns a map from Unix-style relative paths of entries to their contents. The paths are relative to the container (directory or * Returns a map from Unix-style relative paths of entries to their contents. The paths are relative to the given container (directory
* jar). * or jar).
* *
* The map entries need to satisfy the given filter. * The map entries need to satisfy the given filter.
* *
* The map entries are sorted based on their Unix-style relative paths (to ensure deterministic results across filesystems). * The map entries are sorted based on their Unix-style relative paths (to ensure deterministic results across filesystems).
* *
* Note: If a jar has duplicate entries, only one of them will be used (there is no guarantee which one will be used). * Note: If a jar has duplicate entries, only one of them will be used (there is no guarantee which one will be used, but the selection
* will be deterministic).
*/ */
fun read( fun read(
directoryOrJar: File, directoryOrJar: File,
@@ -172,3 +183,18 @@ private object DirectoryOrJarContentsReader {
return relativePathsToContents.sortedBy { it.first }.toMap(LinkedHashMap()) return relativePathsToContents.sortedBy { it.first }.toMap(LinkedHashMap())
} }
} }
/**
* Combines two lists of the same size into a map.
*
* This method is more efficient than calling `[Iterable.zip].toMap()` as it doesn't create short-lived intermediate [Pair]s as done by
* [Iterable.zip].
*/
private fun <K, V> List<K>.zipToMap(other: List<V>): LinkedHashMap<K, V> {
check(this.size == other.size)
val map = LinkedHashMap<K, V>(size)
indices.forEach { index ->
map[this[index]] = other[index]
}
return map
}
@@ -556,7 +556,9 @@ abstract class KotlinCompile @Inject constructor(
it.attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE) it.attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE)
}.files }.files
) )
task.classpathSnapshotProperties.classpathSnapshotDir.value(getClasspathSnapshotDir(task)).disallowChanges() val classpathSnapshotDir = getClasspathSnapshotDir(task)
task.classpathSnapshotProperties.classpathSnapshotDir.value(classpathSnapshotDir).disallowChanges()
task.classpathSnapshotProperties.classpathSnapshotDirFileCollection.from(classpathSnapshotDir)
} }
} }
} }
@@ -604,6 +606,14 @@ abstract class KotlinCompile @Inject constructor(
@get:OutputDirectory @get:OutputDirectory
@get:Optional // Set if useClasspathSnapshot == true @get:Optional // Set if useClasspathSnapshot == true
abstract val classpathSnapshotDir: DirectoryProperty abstract val classpathSnapshotDir: DirectoryProperty
/**
* [FileCollection] containing a single file which is [classpathSnapshotDir], used when a [FileCollection] is required instead of a
* [DirectoryProperty].
*/
// Set if useClasspathSnapshot == true
@get:Internal
abstract val classpathSnapshotDirFileCollection: ConfigurableFileCollection
} }
@get:Internal @get:Internal
@@ -682,7 +692,7 @@ abstract class KotlinCompile @Inject constructor(
val classpathChanges = when { val classpathChanges = when {
!classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathChanges.NotAvailable.ClasspathSnapshotIsDisabled !classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathChanges.NotAvailable.ClasspathSnapshotIsDisabled
else -> when (changedFiles) { else -> when (changedFiles) {
is ChangedFiles.Known -> getClasspathChanges() is ChangedFiles.Known -> getClasspathChanges(changedFiles)
is ChangedFiles.Unknown -> ClasspathChanges.NotAvailable.ForNonIncrementalRun is ChangedFiles.Unknown -> ClasspathChanges.NotAvailable.ForNonIncrementalRun
is ChangedFiles.Dependencies -> error("Unexpected type: ${changedFiles.javaClass.name}") is ChangedFiles.Dependencies -> error("Unexpected type: ${changedFiles.javaClass.name}")
} }
@@ -698,9 +708,16 @@ abstract class KotlinCompile @Inject constructor(
) )
} else null } else null
with(classpathSnapshotProperties) {
if (isIncrementalCompilationEnabled() && useClasspathSnapshot.get()) {
copyClasspathSnapshotFilesToDir(classpathSnapshot.files.toList(), classpathSnapshotDir.get().asFile)
}
}
val environment = GradleCompilerEnvironment( val environment = GradleCompilerEnvironment(
defaultCompilerClasspath, messageCollector, outputItemCollector, defaultCompilerClasspath, messageCollector, outputItemCollector,
outputFiles = allOutputFiles(), // The compiler runner should not manage (read, modify, or delete) classpathSnapshotDir
outputFiles = allOutputFiles().minus(classpathSnapshotProperties.classpathSnapshotDirFileCollection),
reportingSettings = reportingSettings, reportingSettings = reportingSettings,
incrementalCompilationEnvironment = icEnv, incrementalCompilationEnvironment = icEnv,
kotlinScriptExtensions = sourceFilesExtensions.get().toTypedArray() kotlinScriptExtensions = sourceFilesExtensions.get().toTypedArray()
@@ -714,12 +731,6 @@ abstract class KotlinCompile @Inject constructor(
environment, environment,
defaultKotlinJavaToolchain.get().providedJvm.get().javaHome defaultKotlinJavaToolchain.get().providedJvm.get().javaHome
) )
with(classpathSnapshotProperties) {
if (isIncrementalCompilationEnabled() && useClasspathSnapshot.get()) {
copyClasspathSnapshotFilesToDir(classpathSnapshot.files.toList(), classpathSnapshotDir.get().asFile)
}
}
} }
private fun validateKotlinAndJavaHasSameTargetCompatibility(args: K2JVMCompilerArguments) { private fun validateKotlinAndJavaHasSameTargetCompatibility(args: K2JVMCompilerArguments) {
@@ -786,14 +797,29 @@ abstract class KotlinCompile @Inject constructor(
return super.source(*sources) return super.source(*sources)
} }
private fun getClasspathChanges(): ClasspathChanges { private fun getClasspathChanges(knownChangedFiles: ChangedFiles.Known): ClasspathChanges {
val currentSnapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList() // Find current snapshot files that have been changed (added or modified)
val currentSnapshotFiles = classpathSnapshotProperties.classpathSnapshot.files
val addedOrModifiedFiles = knownChangedFiles.modified.toSet()
val (changedCurrentSnapshotFiles, unchangedCurrentSnapshotFiles) = currentSnapshotFiles.partition { it in addedOrModifiedFiles }
// Find previous snapshot files that have been changed (modified or removed)
val previousSnapshotFiles = getClasspathSnapshotFilesInDir(classpathSnapshotProperties.classpathSnapshotDir.get().asFile) val previousSnapshotFiles = getClasspathSnapshotFilesInDir(classpathSnapshotProperties.classpathSnapshotDir.get().asFile)
var unchangedSnapshotIndex = 0
var unchangedSnapshot: ByteArray? = unchangedCurrentSnapshotFiles.getOrNull(unchangedSnapshotIndex)?.readBytes()
val changedPreviousSnapshotFiles = previousSnapshotFiles.filter {
if (unchangedSnapshot != null && it.readBytes().contentEquals(unchangedSnapshot)) {
unchangedSnapshotIndex++
unchangedSnapshot = unchangedCurrentSnapshotFiles.getOrNull(unchangedSnapshotIndex)?.readBytes()
false
} else true
}
val currentSnapshot = ClasspathSnapshotSerializer.load(currentSnapshotFiles) // Compute changes for the changed snapshot files only, ignoring unchanged ones
val previousSnapshot = ClasspathSnapshotSerializer.load(previousSnapshotFiles) val changedCurrentSnapshot = ClasspathSnapshotSerializer.load(changedCurrentSnapshotFiles)
val changedPreviousSnapshot = ClasspathSnapshotSerializer.load(changedPreviousSnapshotFiles)
return ClasspathChangesComputer.getChanges(currentSnapshot, previousSnapshot) return ClasspathChangesComputer.compute(changedCurrentSnapshot, changedPreviousSnapshot)
} }
/** /**
@@ -5,8 +5,9 @@
package org.jetbrains.kotlin.gradle.incremental package org.jetbrains.kotlin.gradle.incremental
import org.jetbrains.kotlin.incremental.DirtyData import org.jetbrains.kotlin.incremental.ClasspathChanges
import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.LookupSymbol
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
@@ -24,6 +25,45 @@ abstract class ClasspathChangesComputerTest : ClasspathSnapshotTestCommon() {
originalSnapshot = testSourceFile.compileAndSnapshot() originalSnapshot = testSourceFile.compileAndSnapshot()
} }
/** Adapted version of [ClasspathChanges.Available] for readability in this test. */
private data class Changes(private val lookupSymbols: Set<LookupSymbol>, private val fqNames: Set<FqName>)
private fun computeClassChanges(current: ClassSnapshot, previous: ClassSnapshot): Changes {
val classChanges =
ClasspathChangesComputer.compute(current.toClasspathSnapshot(), previous.toClasspathSnapshot()) as ClasspathChanges.Available
return Changes(HashSet(classChanges.lookupSymbols), HashSet(classChanges.fqNames))
}
private fun ClassSnapshot.toClasspathSnapshot(): ClasspathSnapshot {
return ClasspathSnapshot(
classpathEntrySnapshots = listOf(
ClasspathEntrySnapshot(
LinkedHashMap<String, ClassSnapshot>(1).also {
it[getClassId()!!.getUnixStyleRelativePath()] = this
})
)
)
}
private fun ClassSnapshot.getClassId(): ClassId? {
return when (this) {
is KotlinClassSnapshot -> classInfo.classId
is RegularJavaClassSnapshot -> serializedJavaClass.classId
is EmptyJavaClassSnapshot -> null
}
}
private fun ClassId.getUnixStyleRelativePath() = asString().replace('.', '$') + ".class"
/**
* Returns the [FqName] of the class in this source file (e.g., "com/example/Foo$Bar.kt" or
* "com/example/Foo$Bar.java" has [FqName] "com.example.Foo.Bar").
*
* This source file must contain only 1 class.
*/
private fun SourceFile.getClassFqName() =
FqName(unixStyleRelativePath.substringBeforeLast('.').replace('/', '.').replace('$', '.'))
// TODO Add more test cases: // TODO Add more test cases:
// - private/non-private fields // - private/non-private fields
// - inline functions // - inline functions
@@ -31,31 +71,30 @@ abstract class ClasspathChangesComputerTest : ClasspathSnapshotTestCommon() {
// - adding an annotation // - adding an annotation
@Test @Test
fun testCollectClassChanges_changedPublicMethodSignature() { fun testComputeClassChanges_changedPublicMethodSignature() {
val updatedSnapshot = testSourceFile.changePublicMethodSignature().compileAndSnapshot() val updatedSnapshot = testSourceFile.changePublicMethodSignature().compileAndSnapshot()
val dirtyData = ClasspathChangesComputer.computeClassChanges(updatedSnapshot, originalSnapshot) val classChanges = computeClassChanges(updatedSnapshot, originalSnapshot)
val testClass = testSourceFile.sourceFile.unixStyleRelativePath.substringBeforeLast('.').replace('/', '.') val testClassFqName = testSourceFile.sourceFile.getClassFqName()
assertEquals( assertEquals(
DirtyData( Changes(
dirtyLookupSymbols = setOf( lookupSymbols = setOf(
LookupSymbol(name = SAM_LOOKUP_NAME.asString(), scope = testClass), LookupSymbol(name = SAM_LOOKUP_NAME.asString(), scope = testClassFqName.asString()),
LookupSymbol(name = "publicMethod", scope = testClass), LookupSymbol(name = "changedPublicMethod", scope = testClassFqName.asString()),
LookupSymbol(name = "changedPublicMethod", scope = testClass) LookupSymbol(name = "publicMethod", scope = testClassFqName.asString())
), ),
dirtyClassesFqNames = setOf(FqName(testClass)), fqNames = setOf(testClassFqName),
dirtyClassesFqNamesForceRecompile = emptySet()
), ),
dirtyData classChanges
) )
} }
@Test @Test
fun testCollectClassChanges_changedMethodImplementation() { fun testComputeClassChanges_changedMethodImplementation() {
val updatedSnapshot = testSourceFile.changeMethodImplementation().compileAndSnapshot() val updatedSnapshot = testSourceFile.changeMethodImplementation().compileAndSnapshot()
val dirtyData = ClasspathChangesComputer.computeClassChanges(updatedSnapshot, originalSnapshot) val classChanges = computeClassChanges(updatedSnapshot, originalSnapshot)
assertEquals(DirtyData(emptySet(), emptySet(), emptySet()), dirtyData) assertEquals(Changes(emptySet(), emptySet()), classChanges)
} }
} }