azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate components of the new incremental compilation

Browse Source
This commit is contained in:
Zalim Bashorov
2015-11-10 18:20:22 +03:00
parent 4caf1b54f7
commit d4a18a3ef2
4 changed files with 181 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+33 -3
View File
@@ -205,14 +205,17 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile)
val caches = filesToCompile.keySet().map { incrementalCaches[it]!! }
processChanges(context, chunk, allCompiledFiles, caches, changesInfo)
processChanges(context, chunk, filesToCompile.values(), allCompiledFiles, dataManager, caches, changesInfo)
return ADDITIONAL_PASS_REQUIRED
}
fun processChanges(
private fun processChanges(
context: CompileContext,
chunk: ModuleChunk,
compiledFiles: Collection<File>,
allCompiledFiles: MutableSet<File>,
dataManager: BuildDataManager,
caches: List<IncrementalCacheImpl>,
changesInfo: ChangesInfo
) {
@@ -250,7 +253,34 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
}
}
changesInfo.doProcessChanges()
fun ChangesInfo.doProcessChangesUsingLookups() {
val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, LookupStorageProvider)
// TODO group by fqName?
for (change in changes) {
if (change !is ChangeInfo.MembersChanged) continue
val files = change.names
.flatMap { lookupStorage.get(LookupSymbol(it, change.fqName.asString())) }
.asSequence()
.map { File(it) }
.filter { it !in compiledFiles && it.exists() }
files.forEach {
FSOperations.markDirty(context, CompilationRound.NEXT, it)
}
}
caches.forEach { it.cleanDirtyInlineFunctions() }
}
if (IncrementalCompilation.isExperimental()) {
changesInfo.doProcessChangesUsingLookups()
}
else {
changesInfo.doProcessChanges()
}
}
private fun doCompileModuleChunk(
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt
+118 -27
View File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.jps.incremental
import com.google.protobuf.MessageLite
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.io.BooleanDataDescriptor
import com.intellij.util.io.EnumeratorStringDescriptor
@@ -28,6 +29,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.incremental.storage.PathStringDescriptor
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.inline.inlineFunctionsJvmNames
import org.jetbrains.kotlin.jps.build.GeneratedJvmClass
import org.jetbrains.kotlin.jps.build.KotlinBuilder
@@ -36,8 +38,12 @@ import org.jetbrains.kotlin.load.kotlin.ModuleMapping
import org.jetbrains.kotlin.load.kotlin.header.*
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.NameResolver
import org.jetbrains.kotlin.serialization.jvm.BitEncoding
import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
import org.jetbrains.org.objectweb.asm.*
import java.io.File
import java.security.MessageDigest
@@ -122,10 +128,14 @@ public class IncrementalCacheImpl(
dependents.forEach(::addFilesAffectedByChangedInlineFuns)
}
dirtyInlineFunctionsMap.clean()
cleanDirtyInlineFunctions()
return result.map { File(it) }
}
public fun cleanDirtyInlineFunctions() {
dirtyInlineFunctionsMap.clean()
}
override fun getClassFilePath(internalClassName: String): String {
return File(outputDir, "$internalClassName.class").canonicalPath
}
@@ -160,9 +170,11 @@ public class IncrementalCacheImpl(
assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" }
packagePartMap.addPackagePart(className)
protoMap.process(kotlinClass, isPackage = true) +
val isPackage = true
protoMap.process(kotlinClass, isPackage) +
constantsMap.process(kotlinClass) +
inlineFunctionsMap.process(kotlinClass)
inlineFunctionsMap.process(kotlinClass, isPackage)
}
header.isCompatibleMultifileClassKind() -> {
val partNames = kotlinClass.classHeader.filePartClassNames?.toList()
@@ -171,21 +183,25 @@ public class IncrementalCacheImpl(
// TODO NO_CHANGES? (delegates only, see package facade)
constantsMap.process(kotlinClass) +
inlineFunctionsMap.process(kotlinClass)
inlineFunctionsMap.process(kotlinClass, isPackage = true)
}
header.isCompatibleMultifileClassPartKind() -> {
assert(sourceFiles.size() == 1) { "Multifile class part from several source files: $sourceFiles" }
packagePartMap.addPackagePart(className)
multifileClassPartMap.add(className.internalName, header.multifileClassName!!)
protoMap.process(kotlinClass, isPackage = true) +
val isPackage = true
protoMap.process(kotlinClass, isPackage) +
constantsMap.process(kotlinClass) +
inlineFunctionsMap.process(kotlinClass)
inlineFunctionsMap.process(kotlinClass, isPackage)
}
header.isCompatibleClassKind() && !header.isLocalClass -> {
protoMap.process(kotlinClass, isPackage = false) +
val isPackage = false
protoMap.process(kotlinClass, isPackage) +
constantsMap.process(kotlinClass) +
inlineFunctionsMap.process(kotlinClass)
inlineFunctionsMap.process(kotlinClass, isPackage)
}
else -> ChangesInfo.NO_CHANGES
}
@@ -201,12 +217,57 @@ public class IncrementalCacheImpl(
}
public fun clearCacheForRemovedClasses(): ChangesInfo {
fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>) =
members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()
fun createChangeInfo(className: JvmClassName): ChangeInfo? {
if (className.internalName == MODULE_MAPPING_FILE_NAME) return null
val mapValue = protoMap.get(className) ?: return null
return when {
mapValue.isPackageFacade -> {
val packageData = JvmProtoBufUtil.readPackageDataFrom(mapValue.bytes, mapValue.strings)
val memberNames =
packageData.packageProto.getNonPrivateNames(
packageData.nameResolver,
ProtoBuf.Package::getFunctionList,
ProtoBuf.Package::getPropertyList
)
ChangeInfo.Removed(className.packageFqName, memberNames)
}
else -> {
val classData = JvmProtoBufUtil.readClassDataFrom(mapValue.bytes, mapValue.strings)
val memberNames =
classData.classProto.getNonPrivateNames(
classData.nameResolver,
ProtoBuf.Class::getConstructorList,
ProtoBuf.Class::getFunctionList,
ProtoBuf.Class::getPropertyList
) +
classData.classProto.enumEntryList.map { classData.nameResolver.getString(it) }.toSet()
ChangeInfo.Removed(className.fqNameForClassNameWithoutDollars, memberNames)
}
}
}
val dirtyClasses = dirtyOutputClassesMap
.getDirtyOutputClasses()
.map(JvmClassName::byInternalName)
.toList()
val changesInfo = dirtyClasses.fold(ChangesInfo.NO_CHANGES) { info, className ->
val changes =
if (IncrementalCompilation.isExperimental())
dirtyClasses.map { createChangeInfo(it) }.asSequence().filterNotNull()
else
emptySequence<ChangeInfo>()
val changesInfo = dirtyClasses.fold(ChangesInfo(changes = changes)) { info, className ->
val newInfo = ChangesInfo(protoChanged = className in protoMap,
constantsChanged = className in constantsMap)
newInfo.logIfSomethingChanged(className)
@@ -268,10 +329,10 @@ public class IncrementalCacheImpl(
private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): ChangesInfo {
public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): ChangesInfo {
val header = kotlinClass.classHeader
val bytes = BitEncoding.decodeBytes(header.annotationData!!)
return put(kotlinClass.className, bytes, header.strings!!, isPackage, checkChangesIsOpenPart)
return put(kotlinClass.className, bytes, header.strings!!, isPackage, checkChangesIsOpenPart = true)
}
public fun process(className: JvmClassName, data: ByteArray, strings: Array<String>, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo {
@@ -288,13 +349,27 @@ public class IncrementalCacheImpl(
if (oldData == null ||
!Arrays.equals(bytes, oldData.bytes) ||
!Arrays.equals(strings, oldData.strings) ||
isPackage != oldData.isPackageFacade) {
isPackage != oldData.isPackageFacade
) {
storage[key] = data
}
return ChangesInfo(protoChanged = oldData == null ||
!checkChangesIsOpenPart ||
difference(oldData, data) != DifferenceKind.NONE)
if (oldData == null || !checkChangesIsOpenPart) return ChangesInfo(protoChanged = true)
val diff = difference(oldData, data)
if (!IncrementalCompilation.isExperimental()) return ChangesInfo(protoChanged = diff != DifferenceKind.NONE)
val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars
val changes =
when (diff) {
is DifferenceKind.NONE -> emptySequence<ChangeInfo>()
is DifferenceKind.CLASS_SIGNATURE -> sequenceOf(ChangeInfo.SignatureChanged(fqName))
is DifferenceKind.MEMBERS -> sequenceOf(ChangeInfo.MembersChanged(fqName, diff.names))
}
return ChangesInfo(protoChanged = diff != DifferenceKind.NONE, changes = changes)
}
public fun contains(className: JvmClassName): Boolean =
@@ -388,11 +463,11 @@ public class IncrementalCacheImpl(
return result
}
public fun process(kotlinClass: LocalFileKotlinClass): ChangesInfo {
return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.fileContents))
public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): ChangesInfo {
return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.fileContents), isPackage)
}
private fun put(className: JvmClassName, newMap: Map<String, Long>): ChangesInfo {
private fun put(className: JvmClassName, newMap: Map<String, Long>, isPackage: Boolean): ChangesInfo {
val internalName = className.internalName
val oldMap = storage[internalName] ?: emptyMap()
@@ -419,8 +494,19 @@ public class IncrementalCacheImpl(
dirtyInlineFunctionsMap.put(className, changed.toList())
}
val changes =
if (IncrementalCompilation.isExperimental()) {
val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars
// TODO get name in better way instead of using substringBefore
(added.asSequence() + changed.asSequence()).map { ChangeInfo.MembersChanged(fqName, listOf(it.substringBefore("("))) }
}
else {
emptySequence<ChangeInfo>()
}
return ChangesInfo(inlineChanged = changed.isNotEmpty(),
inlineAdded = added.isNotEmpty())
inlineAdded = added.isNotEmpty(),
changes = changes)
}
public fun remove(className: JvmClassName) {
@@ -559,11 +645,18 @@ public class IncrementalCacheImpl(
}
}
sealed class ChangeInfo(val fqName: FqName) {
open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName)
class Removed(fqName: FqName, names: Collection<String>) : MembersChanged(fqName, names)
class SignatureChanged(fqName: FqName) : ChangeInfo(fqName)
}
data class ChangesInfo(
public val protoChanged: Boolean = false,
public val constantsChanged: Boolean = false,
public val inlineChanged: Boolean = false,
public val inlineAdded: Boolean = false
val protoChanged: Boolean = false,
val constantsChanged: Boolean = false,
val inlineChanged: Boolean = false,
val inlineAdded: Boolean = false,
val changes: Sequence<ChangeInfo> = emptySequence()
) {
companion object {
public val NO_CHANGES: ChangesInfo = ChangesInfo()
@@ -573,7 +666,8 @@ data class ChangesInfo(
ChangesInfo(protoChanged || other.protoChanged,
constantsChanged || other.constantsChanged,
inlineChanged || other.inlineChanged,
inlineAdded || other.inlineAdded)
inlineAdded || other.inlineAdded,
changes + other.changes)
}
@@ -607,9 +701,6 @@ private fun ByteArray.md5(): Long {
)
}
private val File.normalizedPath: String
get() = FileUtil.toSystemIndependentName(canonicalPath)
@TestOnly
private fun <K : Comparable<K>, V> Map<K, V>.dumpMap(dumpValue: (V)->String): String =
StringBuilder {
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt
+10
View File
@@ -55,6 +55,16 @@ class LookupStorage(private val targetDataDir: File) : BasicMapsOwner() {
}
}
fun get(lookupSymbol: LookupSymbol): Collection<String> {
val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
val fileIds = lookupMap[key] ?: return emptySet()
return fileIds.map {
// null means it's outdated
idToFile[it]?.path
}.filterNotNull()
}
public fun add(lookupSymbol: LookupSymbol, containingPaths: Collection<String>) {
val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
val fileIds = containingPaths.map { addFileIfNeeded(File(it)) }.toHashSet()
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt
+20 -21
View File
@@ -44,6 +44,26 @@ public fun difference(oldData: ProtoMapValue, newData: ProtoMapValue): Differenc
return differenceObject.difference()
}
internal val MessageLite.isPrivate: Boolean
get() = Visibilities.isPrivate(Deserialization.visibility(
when (this) {
is ProtoBuf.Constructor -> Flags.VISIBILITY.get(flags)
is ProtoBuf.Function -> Flags.VISIBILITY.get(flags)
is ProtoBuf.Property -> Flags.VISIBILITY.get(flags)
else -> error("Unknown message: $this")
}))
private fun MessageLite.name(nameResolver: NameResolver): String {
return when (this) {
is ProtoBuf.Constructor -> "<init>"
is ProtoBuf.Function -> nameResolver.getString(name)
is ProtoBuf.Property -> nameResolver.getString(name)
else -> error("Unknown message: $this")
}
}
internal fun List<MessageLite>.names(nameResolver: NameResolver): List<String> = map { it.name(nameResolver) }
private abstract class DifferenceCalculator() {
protected abstract val oldNameResolver: NameResolver
protected abstract val newNameResolver: NameResolver
@@ -57,9 +77,6 @@ private abstract class DifferenceCalculator() {
protected fun calcDifferenceForMembers(oldList: List<MessageLite>, newList: List<MessageLite>): Collection<String> {
val result = hashSetOf<String>()
fun List<MessageLite>.names(nameResolver: NameResolver): List<String> =
map { it.name(nameResolver) }
val oldMap =
oldList.groupBy { it.getHashCode({ compareObject.oldGetIndexOfString(it) }, { compareObject.oldGetIndexOfClassId(it) }) }
val newMap =
@@ -114,15 +131,6 @@ private abstract class DifferenceCalculator() {
return HashSetUtil.symmetricDifference(oldNames, newNames)
}
protected val MessageLite.isPrivate: Boolean
get() = Visibilities.isPrivate(Deserialization.visibility(
when (this) {
is ProtoBuf.Constructor -> Flags.VISIBILITY.get(flags)
is ProtoBuf.Function -> Flags.VISIBILITY.get(flags)
is ProtoBuf.Property -> Flags.VISIBILITY.get(flags)
else -> error("Unknown message: $this")
}))
private fun MessageLite.getHashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (Int) -> Int): Int {
return when (this) {
is ProtoBuf.Constructor -> hashCode(stringIndexes, fqNameIndexes)
@@ -132,15 +140,6 @@ private abstract class DifferenceCalculator() {
}
}
private fun MessageLite.name(nameResolver: NameResolver): String {
return when (this) {
is ProtoBuf.Constructor -> "<init>"
is ProtoBuf.Function -> nameResolver.getString(name)
is ProtoBuf.Property -> nameResolver.getString(name)
else -> error("Unknown message: $this")
}
}
private fun ProtoCompareGenerated.checkEquals(old: MessageLite, new: MessageLite): Boolean {
return when {
old is ProtoBuf.Constructor && new is ProtoBuf.Constructor -> checkEquals(old, new)