azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jps: improve KotlinDirtySourceFilesHolder

Browse Source
This commit is contained in:
Sergey Rostov
2018-06-06 12:09:26 +03:00
parent b5acb83325
commit 3cfb200e07
7 changed files with 93 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt
+1 -1
View File
@@ -976,7 +976,7 @@ open class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() {
project.setTestingContext(TestingContext(LookupTracker.DO_NOTHING, object : BuildLogger {
override fun buildStarted(context: CompileContext, chunk: ModuleChunk) {
actual.append("Targets dependent on ${chunk.targets.joinToString() }:\n")
actual.append(getDependentTargets(chunk, context).map { it.toString() }.sorted().joinToString("\n"))
actual.append(getDependentTargets(chunk.targets, context).map { it.toString() }.sorted().joinToString("\n"))
actual.append("\n---------\n")
}
jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt.172
+1 -1
View File
@@ -975,7 +975,7 @@ open class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() {
project.setTestingContext(TestingContext(LookupTracker.DO_NOTHING, object : BuildLogger {
override fun buildStarted(context: CompileContext, chunk: ModuleChunk) {
actual.append("Targets dependent on ${chunk.targets.joinToString() }:\n")
actual.append(getDependentTargets(chunk, context).map { it.toString() }.sorted().joinToString("\n"))
actual.append(getDependentTargets(chunk.targets, context).map { it.toString() }.sorted().joinToString("\n"))
actual.append("\n---------\n")
}
jps-plugin/src/org/jetbrains/kotlin/jps/build/FSOperationsHelper.kt
+30 -6
View File
@@ -23,13 +23,18 @@ import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.java.dependencyView.Mappings
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.FSOperations
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.fs.CompilationRound
import java.io.File
import java.util.HashMap
/**
* Entry point for safely marking files as dirty.
*/
class FSOperationsHelper(
private val compileContext: CompileContext,
private val chunk: ModuleChunk,
private val dirtyFilesHolder: KotlinDirtySourceFilesHolder,
private val log: Logger
) {
private val moduleBasedFilter = ModulesBasedFileFilter(compileContext, chunk)
@@ -56,27 +61,46 @@ class FSOperationsHelper(
}
}
fun markFilesBeforeInitialRound(files: Iterable<File>) {
markFilesImpl(files, beforeRound = true) { it.exists() && moduleBasedFilter.accept(it) }
internal fun markFilesForCurrentRound(files: Iterable<File>) {
files.forEach {
val root = compileContext.projectDescriptor.buildRootIndex.findJavaRootDescriptor(compileContext, it)
if (root != null) dirtyFilesHolder.byTarget[root.target]?._markDirty(it)
}
markFilesImpl(files, currentRound = true) { it.exists() && moduleBasedFilter.accept(it) }
}
/**
* Marks given [files] as dirty for current round and given [target] of [chunk].
*/
fun markFilesForCurrentRound(target: ModuleBuildTarget, files: Iterable<File>) {
require(target in chunk.targets)
val targetDirtyFiles = dirtyFilesHolder.byTarget[target]!!
files.forEach {
targetDirtyFiles._markDirty(it)
}
markFilesImpl(files, currentRound = true) { it.exists() }
}
fun markFiles(files: Iterable<File>) {
markFilesImpl(files, beforeRound = false) { it.exists() }
markFilesImpl(files, currentRound = false) { it.exists() }
}
fun markInChunkOrDependents(files: Iterable<File>, excludeFiles: Set<File>) {
markFilesImpl(files, beforeRound = false) { it !in excludeFiles && it.exists() && moduleBasedFilter.accept(it) }
markFilesImpl(files, currentRound = false) { it !in excludeFiles && it.exists() && moduleBasedFilter.accept(it) }
}
private inline fun markFilesImpl(
files: Iterable<File>,
beforeRound: Boolean,
currentRound: Boolean,
shouldMark: (File) -> Boolean
) {
val filesToMark = files.filterTo(HashSet(), shouldMark)
if (filesToMark.isEmpty()) return
val compilationRound = if (beforeRound) {
val compilationRound = if (currentRound) {
buildLogger?.markedAsDirtyBeforeRound(filesToMark)
CompilationRound.CURRENT
} else {
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+33 -37
View File
@@ -118,7 +118,16 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
if (targets.none { hasKotlin[it] == true }) return
val fsOperations = FSOperationsHelper(context, chunk, LOG)
val roundDirtyFiles = KotlinDirtySourceFilesHolder(
chunk,
context,
object : DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {
override fun processDirtyFiles(processor: FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget>) {
FSOperations.processFilesToRecompile(context, chunk, processor)
}
}
)
val fsOperations = FSOperationsHelper(context, chunk, roundDirtyFiles, LOG)
if (System.getProperty(SKIP_CACHE_VERSION_CHECK_PROPERTY) == null) {
val cacheVersionsProvider = CacheVersionProvider(dataManager.dataPaths)
@@ -142,25 +151,16 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
return
}
markAdditionalFilesForInitialRound(chunk, context, fsOperations)
markAdditionalFilesForInitialRound(chunk, context, fsOperations, roundDirtyFiles)
buildLogger?.afterBuildStarted(context, chunk)
}
private fun markAdditionalFilesForInitialRound(
chunk: ModuleChunk,
context: CompileContext,
fsOperations: FSOperationsHelper
fsOperations: FSOperationsHelper,
dirtyFilesHolder: KotlinDirtySourceFilesHolder
) {
val roundDirtyFiles = KotlinDirtySourceFilesHolder(
chunk,
context,
fsOperations,
object : DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {
override fun processDirtyFiles(processor: FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget>) {
FSOperations.processFilesToRecompile(context, chunk, processor)
}
}
)
val representativeTarget = context.kotlinBuildTargets[chunk.representativeTarget()] ?: return
val incrementalCaches = getIncrementalCaches(chunk, context)
@@ -177,8 +177,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val removedClasses = HashSet<String>()
for (target in chunk.targets) {
val cache = incrementalCaches[target] ?: continue
val dirtyFiles = roundDirtyFiles.getDirtyFiles(target)
val removedFiles = roundDirtyFiles.getRemovedFilesSet(target)
val dirtyFiles = dirtyFilesHolder.getDirtyFiles(target)
val removedFiles = dirtyFilesHolder.getRemovedFiles(target)
val existingClasses = JpsKotlinCompilerRunner().classesFqNamesByFiles(environment, dirtyFiles)
val previousClasses = cache.classesFqNamesBySources(dirtyFiles + removedFiles)
@@ -194,7 +194,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) }
val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, context.projectDescriptor.dataManager)
fsOperations.markFilesBeforeInitialRound(affectedByRemovedClasses)
fsOperations.markFilesForCurrentRound(affectedByRemovedClasses)
}
private fun checkCachesVersions(
@@ -234,10 +234,11 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val kotlinTarget = context.kotlinBuildTargets[chunk.representativeTarget()] ?: return OK
val messageCollector = MessageCollectorAdapter(context, kotlinTarget)
val fsOperations = FSOperationsHelper(context, chunk, LOG)
val kotlinDirtyFilesHolder = KotlinDirtySourceFilesHolder(chunk, context, dirtyFilesHolder)
val fsOperations = FSOperationsHelper(context, chunk, kotlinDirtyFilesHolder, LOG)
try {
val proposedExitCode = doBuild(chunk, kotlinTarget, context, dirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
val proposedExitCode = doBuild(chunk, kotlinTarget, context, kotlinDirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
val actualExitCode = if (proposedExitCode == OK && fsOperations.hasMarkedDirty) ADDITIONAL_PASS_REQUIRED else proposedExitCode
@@ -260,7 +261,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
chunk: ModuleChunk,
representativeTarget: KotlinModuleBuildTarget,
context: CompileContext,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder,
messageCollector: MessageCollectorAdapter,
outputConsumer: OutputConsumer,
fsOperations: FSOperationsHelper
@@ -271,7 +272,6 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
return NOTHING_DONE
}
val roundDirtyFilesHolder = KotlinDirtySourceFilesHolder(chunk, context, fsOperations, dirtyFilesHolder)
val projectDescriptor = context.projectDescriptor
val dataManager = projectDescriptor.dataManager
val targets = chunk.targets
@@ -280,7 +280,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val isChunkRebuilding = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)
|| targets.any { rebuildAfterCacheVersionChanged[it] == true }
if (roundDirtyFilesHolder.hasDirtyOrRemovedFiles) {
if (kotlinDirtyFilesHolder.hasDirtyOrRemovedFiles) {
if (!isChunkRebuilding && !IncrementalCompilation.isEnabled()) {
targets.forEach { rebuildAfterCacheVersionChanged[it] = true }
return CHUNK_REBUILD_REQUIRED
@@ -319,13 +319,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
}
if (LOG.isDebugEnabled) {
LOG.debug("Compiling files: ${roundDirtyFilesHolder.dirtyFiles}")
LOG.debug("Compiling files: ${kotlinDirtyFilesHolder.allDirtyFiles}")
}
val start = System.nanoTime()
val outputItemCollector = doCompileModuleChunk(
chunk, representativeTarget, commonArguments, context, roundDirtyFilesHolder, environment,
incrementalCaches
chunk, representativeTarget, commonArguments, context, kotlinDirtyFilesHolder, fsOperations,
environment, incrementalCaches
)
statisticsLogger.registerStatistic(chunk, System.nanoTime() - start)
@@ -348,7 +348,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
saveVersions(context, chunk, commonArguments)
if (targets.any { hasKotlin[it] == null }) {
fsOperations.markChunk(recursively = false, kotlinOnly = true, excludeFiles = roundDirtyFilesHolder.dirtyFiles)
fsOperations.markChunk(recursively = false, kotlinOnly = true, excludeFiles = kotlinDirtyFilesHolder.allDirtyFiles)
}
for (target in targets) {
@@ -362,7 +362,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
representativeTarget.updateChunkMappings(
chunk,
roundDirtyFilesHolder,
kotlinDirtyFilesHolder,
generatedFiles,
incrementalCaches
)
@@ -383,11 +383,11 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
kotlinModuleBuilderTarget.updateCaches(incrementalCaches[target]!!, files, changesCollector, environment)
}
updateLookupStorage(lookupTracker, dataManager, roundDirtyFilesHolder)
updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder)
if (!isChunkRebuilding) {
changesCollector.processChangesUsingLookups(
roundDirtyFilesHolder.dirtyFiles,
kotlinDirtyFilesHolder.allDirtyFiles,
dataManager,
fsOperations,
incrementalCaches.values
@@ -499,6 +499,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
commonArguments: CommonCompilerArguments,
context: CompileContext,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
fsOperations: FSOperationsHelper,
environment: JpsCompilerEnvironment,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
): OutputItemsCollector? {
@@ -528,15 +529,10 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val targetDirtyFiles = dirtyFilesHolder.byTarget[target]
if (cache != null && targetDirtyFiles != null) {
val removedAndDirtyFiles: MutableSet<File> = mutableSetOf()
val complementaryFiles = cache.clearComplementaryFilesMapping(targetDirtyFiles.dirty + targetDirtyFiles.removed)
fsOperations.markFilesForCurrentRound(target, complementaryFiles)
removedAndDirtyFiles.addAll(dirtyFilesHolder.getDirtyFiles(target))
removedAndDirtyFiles.addAll(dirtyFilesHolder.getRemovedFilesSet(target))
val complementaryFiles = cache.clearComplementaryFilesMapping(targetDirtyFiles.dirtyOrRemovedFiles)
targetDirtyFiles.markDirtyForCurrentRound(complementaryFiles)
cache.markDirty(targetDirtyFiles.dirtyOrRemovedFiles)
cache.markDirty(targetDirtyFiles.dirty + targetDirtyFiles.removed)
}
}
}
@@ -644,7 +640,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
dataManager.withLookupStorage { lookupStorage ->
lookupStorage.removeLookupsFrom(dirtyFilesHolder.dirtyOrRemovedFilesSet.asSequence())
lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence())
lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values)
}
}
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinDirtySourceFilesHolder.kt
+23 -64
View File
@@ -19,8 +19,6 @@ package org.jetbrains.kotlin.jps.build
import com.intellij.openapi.util.io.FileUtilRt
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.FileProcessor
import org.jetbrains.jps.builders.impl.DirtyFilesHolderBase
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
@@ -30,59 +28,44 @@ import java.io.File
* Holding kotlin dirty files list for particular build round.
* Dirty and removed files set are initialized from [delegate].
*
* Probably should be merged with [FSOperationsHelper]
* Additional dirty files should be added only through [FSOperationsHelper.markFilesForCurrentRound]
*/
class KotlinDirtySourceFilesHolder(
val chunk: ModuleChunk,
val context: CompileContext,
val fsOperations: FSOperationsHelper,
delegate: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>
) : DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {
) {
val byTarget: Map<ModuleBuildTarget, TargetFiles>
inner class TargetFiles(val target: ModuleBuildTarget, val removed: Collection<String>) {
val dirty: MutableSet<DirtyFile> = mutableSetOf()
inner class TargetFiles(val target: ModuleBuildTarget, val removed: Collection<File>) {
private val _dirty: MutableSet<File> = mutableSetOf()
val dirtyOrRemovedFiles: Set<File>
get() {
val result = mutableSetOf<File>()
dirty.forEach { result.add(it.file) }
removed.forEach { result.add(File(it)) }
return result
}
val dirty: Set<File>
get() = _dirty
fun markDirtyForCurrentRound(files: Collection<File>) {
fsOperations.markFilesBeforeInitialRound(files)
files.forEach {
dirty.add(DirtyFile(it, null))
}
/**
* Should be called only from [FSOperationsHelper.markFilesForCurrentRound]
* and during KotlinDirtySourceFilesHolder initialization.
*/
internal fun _markDirty(file: File) {
_dirty.add(file)
}
}
data class DirtyFile(val file: File, val root: JavaSourceRootDescriptor?)
val hasRemovedFiles: Boolean
override fun hasRemovedFiles(): Boolean = hasRemovedFiles
get() = byTarget.any { it.value.removed.isNotEmpty() }
val hasDirtyFiles: Boolean
override fun hasDirtyFiles(): Boolean = hasDirtyFiles
get() = byTarget.any { it.value.dirty.isNotEmpty() }
val hasDirtyOrRemovedFiles: Boolean
get() = hasRemovedFiles || hasDirtyFiles
init {
val byTarget = mutableMapOf<ModuleBuildTarget, TargetFiles>()
var hasDirtyFiles = false
var hasRemovedFiles = false
chunk.targets.forEach { target ->
val removedFiles = delegate.getRemovedFiles(target)
if (removedFiles.isNotEmpty()) {
hasRemovedFiles = true
}
val removedFiles = delegate.getRemovedFiles(target).map { File(it) }
byTarget[target] = TargetFiles(target, removedFiles)
}
@@ -91,50 +74,26 @@ class KotlinDirtySourceFilesHolder(
?: error("processDirtyFiles should callback only on chunk `$chunk` targets (`$target` is not)")
if (file.isKotlinSourceFile) {
targetInfo.dirty.add(DirtyFile(file, root))
hasDirtyFiles = true
targetInfo._markDirty(file)
}
true
}
this.byTarget = byTarget
this.hasRemovedFiles = hasRemovedFiles
this.hasDirtyFiles = hasDirtyFiles
}
override fun processDirtyFiles(processor: FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget>) {
for ((target, info) in byTarget) {
info.dirty.forEach {
if (!processor.apply(target, it.file, it.root)) return
}
}
}
fun getDirtyFiles(target: ModuleBuildTarget): Set<File> =
byTarget[target]?.dirty?.mapTo(mutableSetOf()) { it.file } ?: setOf()
byTarget[target]?.dirty ?: setOf()
fun getRemovedFilesSet(target: ModuleBuildTarget): Set<File> =
byTarget[target]?.removed?.mapTo(mutableSetOf()) { File(it) } ?: setOf()
fun getRemovedFiles(target: ModuleBuildTarget): Collection<File> =
byTarget[target]?.removed ?: listOf()
override fun getRemovedFiles(target: ModuleBuildTarget): Collection<String> =
byTarget.flatMap { it.value.removed }
val allDirtyFiles: Set<File>
get() = byTarget.flatMapTo(mutableSetOf()) { it.value.dirty }
val removedFilesCount
get() = byTarget.values.flatMapTo(mutableSetOf()) { it.removed }.size
val dirtyFiles: Set<File>
get() = byTarget.flatMapTo(mutableSetOf()) { it.value.dirty.map { it.file } }
val dirtyOrRemovedFilesSet: Set<File>
get() {
val result = mutableSetOf<File>()
byTarget.forEach {
it.value.dirty.forEach { result.add(it.file) }
it.value.removed.forEach { result.add(File(it)) }
}
return result
}
val allRemovedFilesFiles
get() = byTarget.flatMapTo(mutableSetOf()) { it.value.removed }
}
val File.isKotlinSourceFile: Boolean
jps-plugin/src/org/jetbrains/kotlin/jps/build/MessageCollectorAdapter.kt
+1 -1
View File
@@ -17,7 +17,7 @@ import java.io.File
class MessageCollectorAdapter(
private val context: CompileContext,
val kotlinTarget: KotlinModuleBuildTarget?
private val kotlinTarget: KotlinModuleBuildTarget?
) : MessageCollector {
private var hasErrors = false
jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJvmModuleBuildTarget.kt
+4 -4
View File
@@ -133,7 +133,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
)
}
val filesSet = dirtyFilesHolder.dirtyFiles
val filesSet = dirtyFilesHolder.allDirtyFiles
val moduleFile = generateModuleDescription(chunk, dirtyFilesHolder)
if (moduleFile == null) {
@@ -150,7 +150,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
val module = chunk.representativeTarget().module
if (KotlinBuilder.LOG.isDebugEnabled) {
val totalRemovedFiles = dirtyFilesHolder.removedFilesCount
val totalRemovedFiles = dirtyFilesHolder.allRemovedFilesFiles.size
KotlinBuilder.LOG.debug("Compiling to JVM ${filesSet.size} files"
+ (if (totalRemovedFiles == 0) "" else " ($totalRemovedFiles removed files)")
+ " in " + chunk.targets.joinToString { it.presentableName })
@@ -303,7 +303,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
val targetDirtyFiles: Map<ModuleBuildTarget, Set<File>> = chunk.targets.keysToMap {
val files = HashSet<File>()
dirtyFilesHolder.getRemovedFiles(it).mapTo(files, ::File)
files.addAll(dirtyFilesHolder.getRemovedFiles(it))
files.addAll(dirtyFilesHolder.getDirtyFiles(it))
files
}
@@ -336,7 +336,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
}
}
val allCompiled = dirtyFilesHolder.dirtyFiles
val allCompiled = dirtyFilesHolder.allDirtyFiles
JavaBuilderUtil.registerFilesToCompile(context, allCompiled)
JavaBuilderUtil.registerSuccessfullyCompiled(context, allCompiled)
}