diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 3971e3ebd9b..fc9cb1352b4 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner import java.io.File -abstract class IncrementalCachesManager ( +abstract class IncrementalCachesManager( protected val cachesRootDir: File, protected val reporter: ICReporter ) { @@ -33,6 +33,7 @@ abstract class IncrementalCachesManager ( val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter).apply { registerCache() } val lookupCache: LookupStorage = LookupStorage(lookupCacheDir).apply { registerCache() } + abstract val platformCache: PlatformCache fun clean() { caches.forEach { it.clean() } @@ -70,8 +71,8 @@ class IncrementalJvmCachesManager( cacheDirectory: File, outputDir: File, reporter: ICReporter -) : IncrementalCachesManager(cacheDirectory, reporter) { +) : IncrementalCachesManager(cacheDirectory, reporter) { - private val jvmCacheFile = File(cacheDirectory, "jvm").apply { mkdirs() } - val jvmCache = IncrementalCacheImpl(jvmCacheFile, outputDir).apply { registerCache() } + private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() } + override val platformCache = IncrementalCacheImpl(jvmCacheDir, outputDir).apply { registerCache() } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index e009ae8a12b..6d368a7eb4e 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -19,8 +19,10 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.annotation.AnnotationFileUpdater import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.GeneratedJvmClass +import org.jetbrains.kotlin.build.isModuleMappingFile import org.jetbrains.kotlin.build.JvmSourceRoot import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity @@ -38,6 +40,7 @@ import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.modules.TargetId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.CompilationCanceledStatus +import org.jetbrains.kotlin.resolve.jvm.JvmClassName import java.io.File import java.util.* @@ -84,48 +87,54 @@ inline fun withIC(enabled: Boolean = true, fn: ()->R): R { } } -class IncrementalJvmCompilerRunner( - workingDir: File, - private val javaSourceRoots: Set, - private val cacheVersions: List, - private val reporter: ICReporter, - private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null, - private val artifactChangesProvider: ArtifactChangesProvider? = null, - private val changesRegistry: ChangesRegistry? = null +abstract class IncrementalCompilerRunner< + Args : CommonCompilerArguments, + CacheManager : IncrementalCachesManager<*> +>( + workingDir: File, + cacheDirName: String, + protected val cacheVersions: List, + protected val reporter: ICReporter, + protected val artifactChangesProvider: ArtifactChangesProvider?, + protected val changesRegistry: ChangesRegistry? ) { - private var anyClassesCompiled: Boolean = false - private val cacheDirectory = File(workingDir, CACHES_DIR_NAME) - private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) - private val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME) + + protected val cacheDirectory = File(workingDir, cacheDirName) + protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME) + protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME) + + protected abstract fun isICEnabled(): Boolean + protected abstract fun createCacheManager(args: Args): CacheManager + protected abstract fun destionationDir(args: Args): File fun compile( allKotlinSources: List, - args: K2JVMCompilerArguments, + args: Args, messageCollector: MessageCollector, - getChangedFiles: (IncrementalCachesManager)->ChangedFiles + getChangedFiles: (CacheManager)->ChangedFiles ): ExitCode { - var caches = IncrementalJvmCachesManager(cacheDirectory, File(args.destination), reporter) + assert(isICEnabled()) { "Incremental compilation is not enabled" } + var caches = createCacheManager(args) fun rebuild(): ExitCode { caches.clean() dirtySourcesSinceLastTimeFile.delete() - args.destinationAsFile.deleteRecursively() + destionationDir(args).deleteRecursively() - val javaFilesProcessor = ChangedJavaFilesProcessor(reporter) - caches = IncrementalJvmCachesManager(cacheDirectory, args.destinationAsFile, reporter) - return compileIncrementally(args, caches, javaFilesProcessor, allKotlinSources, CompilationMode.Rebuild, messageCollector) + caches = createCacheManager(args) + return compileIncrementally(args, caches, allKotlinSources, CompilationMode.Rebuild(), messageCollector) } return try { - val javaFilesProcessor = ChangedJavaFilesProcessor(reporter) val changedFiles = getChangedFiles(caches) - val compilationMode = calculateSourcesToCompile(javaFilesProcessor, caches, changedFiles, args) + val compilationMode = sourcesToCompile(caches, changedFiles, args) val exitCode = when (compilationMode) { is CompilationMode.Incremental -> { - compileIncrementally(args, caches, javaFilesProcessor, allKotlinSources, compilationMode, messageCollector) + compileIncrementally(args, caches, allKotlinSources, compilationMode, messageCollector) } is CompilationMode.Rebuild -> { + reporter.report { "Non-incremental compilation will be performed: ${compilationMode.reason}" } rebuild() } } @@ -141,52 +150,110 @@ class IncrementalJvmCompilerRunner( } } - private data class CompileChangedResults(val exitCode: ExitCode, val generatedFiles: List>) + private fun sourcesToCompile(caches: CacheManager, changedFiles: ChangedFiles, args: Args): CompilationMode = + when (changedFiles) { + is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args) + is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" } + } - private sealed class CompilationMode { - class Incremental(val dirtyFiles: Set) : CompilationMode() - object Rebuild : CompilationMode() - } + protected open fun calculateSourcesToCompile(caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args): CompilationMode = + CompilationMode.Incremental(getDirtyFiles(changedFiles)) - private fun calculateSourcesToCompile( - javaFilesProcessor: ChangedJavaFilesProcessor, - caches: IncrementalJvmCachesManager, - changedFiles: ChangedFiles, - args: K2JVMCompilerArguments - ): CompilationMode { - fun rebuild(reason: ()->String): CompilationMode { - reporter.report {"Non-incremental compilation will be performed: ${reason()}"} - return CompilationMode.Rebuild + protected fun getDirtyFiles(changedFiles: ChangedFiles.Known): HashSet { + val dirtyFiles = HashSet(with(changedFiles) { modified.size + removed.size }) + with(changedFiles) { + modified.asSequence() + removed.asSequence() + }.forEach { if (it.isKotlinFile()) dirtyFiles.add(it) } + + if (dirtySourcesSinceLastTimeFile.exists()) { + val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists) + if (files.isNotEmpty()) { + reporter.report { "Source files added since last compilation: ${reporter.pathsAsString(files)}" } + } + + dirtyFiles.addAll(files) } - if (changedFiles !is ChangedFiles.Known) return rebuild {"inputs' changes are unknown (first or clean build)"} + return dirtyFiles + } + protected open fun markOutputDirty(caches: CacheManager, dirtySources: List) { + } + + protected abstract fun compareAndUpdateCache(caches: CacheManager, generatedFiles: List>): CompilationResult + + protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {} + protected open fun postCompilationHook(exitCode: ExitCode) {} + protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List>): Iterable = + emptyList() + + protected abstract fun compileIncrementally(args: Args, caches: CacheManager, allKotlinSources: List, compilationMode: CompilationMode, messageCollector: MessageCollector): ExitCode + + protected data class CompileChangedResults(val exitCode: ExitCode, val generatedFiles: List>) + + protected sealed class CompilationMode { + class Incremental(val dirtyFiles: Set) : CompilationMode() + class Rebuild(getReason: ()->String = { "" }) : CompilationMode() { + val reason: String by lazy(getReason) + } + } + + companion object { + const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt" + const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin" + } +} + +class IncrementalJvmCompilerRunner( + workingDir: File, + private val javaSourceRoots: Set, + cacheVersions: List, + reporter: ICReporter, + private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null, + artifactChangesProvider: ArtifactChangesProvider? = null, + changesRegistry: ChangesRegistry? = null +) : IncrementalCompilerRunner( + workingDir, + CACHES_DIR_NAME, + cacheVersions, + reporter, + artifactChangesProvider, + changesRegistry +) { + override fun isICEnabled(): Boolean = + IncrementalCompilation.isEnabled() + + override fun createCacheManager(args: K2JVMCompilerArguments): IncrementalJvmCachesManager = + IncrementalJvmCachesManager(cacheDirectory, File(args.destination), reporter) + + override fun destionationDir(args: K2JVMCompilerArguments): File = + args.destinationAsFile + + private var javaFilesProcessor = ChangedJavaFilesProcessor(reporter) + + override fun calculateSourcesToCompile(caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments): CompilationMode { val removedClassFiles = changedFiles.removed.filter(File::isClassFile) - if (removedClassFiles.any()) return rebuild {"Removed class files: ${reporter.pathsAsString(removedClassFiles)}"} + if (removedClassFiles.any()) return CompilationMode.Rebuild { "Removed class files: ${reporter.pathsAsString(removedClassFiles)}" } val modifiedClassFiles = changedFiles.modified.filter(File::isClassFile) - if (modifiedClassFiles.any()) return rebuild {"Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}"} + if (modifiedClassFiles.any()) return CompilationMode.Rebuild { "Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}" } val classpathSet = args.classpathAsList.toHashSet() - val modifiedClasspathEntries = changedFiles.modified.filter {it in classpathSet} + val modifiedClasspathEntries = changedFiles.modified.filter { it in classpathSet } val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) reporter.report { "Last Kotlin Build info -- $lastBuildInfo" } val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo) if (classpathChanges !is ChangesEither.Known) { - return rebuild {"could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}"} + return CompilationMode.Rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" } } val javaFilesChanges = javaFilesProcessor.process(changedFiles) val affectedJavaSymbols = when (javaFilesChanges) { is ChangesEither.Known -> javaFilesChanges.lookupSymbols - is ChangesEither.Unknown -> return rebuild {"Could not get changes for java files"} + is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" } } - val dirtyFiles = HashSet(with(changedFiles) {modified.size + removed.size}) - with(changedFiles) { - modified.asSequence() + removed.asSequence() - }.forEach {if (it.isKotlinFile()) dirtyFiles.add(it)} - + val dirtyFiles = getDirtyFiles(changedFiles) val lookupSymbols = HashSet() lookupSymbols.addAll(affectedJavaSymbols) lookupSymbols.addAll(classpathChanges.lookupSymbols) @@ -196,21 +263,12 @@ class IncrementalJvmCompilerRunner( dirtyFiles.addAll(dirtyFilesFromLookups) } - val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.jvmCache)) } + val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.platformCache)) } if (dirtyClassesFqNames.any()) { - val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.jvmCache), dirtyClassesFqNames, reporter) + val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.platformCache), dirtyClassesFqNames, reporter) dirtyFiles.addAll(dirtyFilesFromFqNames) } - if (dirtySourcesSinceLastTimeFile.exists()) { - val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists) - if (files.isNotEmpty()) { - reporter.report {"Source files added since last compilation: ${reporter.pathsAsString(files)}"} - } - - dirtyFiles.addAll(files) - } - return CompilationMode.Incremental(dirtyFiles) } @@ -248,29 +306,49 @@ class IncrementalJvmCompilerRunner( return ChangesEither.Known(symbols, fqNames) } - private fun compileIncrementally( + override fun preBuildHook(args: K2JVMCompilerArguments, compilationMode: CompilationMode) { + when (compilationMode) { + is CompilationMode.Incremental -> { + args.classpathAsList += args.destinationAsFile.apply { mkdirs() } + } + is CompilationMode.Rebuild -> { + // there is no point in updating annotation file since all files will be compiled anyway + kaptAnnotationsFileUpdater = null + } + } + } + + private var outdatedClasses: Iterable = emptyList() + override fun markOutputDirty(caches: IncrementalJvmCachesManager, dirtySources: List) { + outdatedClasses = caches.platformCache.classesBySources(dirtySources) + caches.platformCache.markOutputClassesDirty(dirtySources) + super.markOutputDirty(caches, dirtySources) + } + + override fun postCompilationHook(exitCode: ExitCode) { + if (exitCode == ExitCode.OK) { + kaptAnnotationsFileUpdater?.updateAnnotations(outdatedClasses) + } + else { + kaptAnnotationsFileUpdater?.revert() + } + } + + override fun compareAndUpdateCache(caches: IncrementalJvmCachesManager, generatedFiles: List>): CompilationResult = + updateIncrementalCache(generatedFiles, caches.platformCache) + + override fun compileIncrementally( args: K2JVMCompilerArguments, caches: IncrementalJvmCachesManager, - javaFilesProcessor: ChangedJavaFilesProcessor, allKotlinSources: List, compilationMode: CompilationMode, messageCollector: MessageCollector ): ExitCode { - assert(IncrementalCompilation.isEnabled()) { "Incremental compilation is not enabled" } + preBuildHook(args, compilationMode) - val allGeneratedFiles = hashSetOf>() - val dirtySources: MutableList - - when (compilationMode) { - is CompilationMode.Incremental -> { - dirtySources = ArrayList(compilationMode.dirtyFiles) - args.classpathAsList += args.destinationAsFile.apply { mkdirs() } - } - is CompilationMode.Rebuild -> { - dirtySources = allKotlinSources.toMutableList() - // there is no point in updating annotation file since all files will be compiled anyway - kaptAnnotationsFileUpdater = null - } + val dirtySources = when (compilationMode) { + is CompilationMode.Incremental -> ArrayList(compilationMode.dirtyFiles) + is CompilationMode.Rebuild -> allKotlinSources.toMutableList() } val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis()) @@ -280,12 +358,13 @@ class IncrementalJvmCompilerRunner( val allSourcesToCompile = HashSet() var exitCode = ExitCode.OK + val allGeneratedFiles = hashSetOf>() + while (dirtySources.any()) { - val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) - val outdatedClasses = caches.jvmCache.classesBySources(dirtySources) - caches.jvmCache.markOutputClassesDirty(dirtySources) + markOutputDirty(caches, dirtySources) caches.inputsCache.removeOutputForSourceFiles(dirtySources) + val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) val (sourcesToCompile, removedKotlinSources) = dirtySources.partition(File::exists) // todo: more optimal to save only last iteration, but it will require adding standalone-ic specific logs @@ -294,22 +373,20 @@ class IncrementalJvmCompilerRunner( val text = allSourcesToCompile.joinToString(separator = System.getProperty("line.separator")) { it.canonicalPath } dirtySourcesSinceLastTimeFile.writeText(text) - val compilerOutput = compileChanged(sourcesToCompile.toSet(), args, caches.jvmCache, lookupTracker, messageCollector) + val compilerOutput = compileChanged(sourcesToCompile.toSet(), args, caches.platformCache, lookupTracker, messageCollector) + exitCode = compilerOutput.exitCode + postCompilationHook(exitCode) + + if (exitCode != ExitCode.OK) break + + dirtySourcesSinceLastTimeFile.delete() + val generatedFiles = compilerOutput.generatedFiles - anyClassesCompiled = anyClassesCompiled || generatedFiles.isNotEmpty() || removedKotlinSources.isNotEmpty() - - if (exitCode == ExitCode.OK) { - dirtySourcesSinceLastTimeFile.delete() - kaptAnnotationsFileUpdater?.updateAnnotations(outdatedClasses) - } else { - kaptAnnotationsFileUpdater?.revert() - break - } - if (compilationMode is CompilationMode.Incremental) { + // todo: feels dirty, can this be refactored? val dirtySourcesSet = dirtySources.toHashSet() - val additionalDirtyFiles = additionalDirtyFiles(caches.jvmCache, generatedFiles).filter { it !in dirtySourcesSet } + val additionalDirtyFiles = additionalDirtyFiles(caches, generatedFiles).filter { it !in dirtySourcesSet } if (additionalDirtyFiles.isNotEmpty()) { dirtySources.addAll(additionalDirtyFiles) continue @@ -318,21 +395,18 @@ class IncrementalJvmCompilerRunner( allGeneratedFiles.addAll(generatedFiles) caches.inputsCache.registerOutputForSourceFiles(generatedFiles) - val compilationResult = updateIncrementalCache(generatedFiles, caches.platformCache) - caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) + val compilationResult = compareAndUpdateCache(caches, generatedFiles) - if (compilationMode is CompilationMode.Rebuild) { - break - } + if (compilationMode is CompilationMode.Rebuild) break - val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.jvmCache), reporter) + val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.platformCache), reporter) val compiledInThisIterationSet = sourcesToCompile.toHashSet() with (dirtySources) { clear() addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = compiledInThisIterationSet)) - addAll(mapClassesFqNamesToFiles(listOf(caches.jvmCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet)) + addAll(mapClassesFqNamesToFiles(listOf(caches.platformCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet)) } buildDirtyLookupSymbols.addAll(dirtyLookupSymbols) @@ -360,10 +434,11 @@ class IncrementalJvmCompilerRunner( return exitCode } - private fun additionalDirtyFiles( - cache: IncrementalCacheImpl, - generatedFiles: List> - ): Collection { + override fun additionalDirtyFiles( + caches: IncrementalJvmCachesManager, + generatedFiles: List> + ): Iterable { + val cache = caches.platformCache val result = HashSet() fun partsByFacadeName(facadeInternalName: String): List { @@ -382,6 +457,7 @@ class IncrementalJvmCompilerRunner( val cachedSourceFile = cache.getSourceFileIfClass(fqName) if (cachedSourceFile != null) { + // todo: seems useless, remove? result.add(cachedSourceFile) } } @@ -424,7 +500,7 @@ class IncrementalJvmCompilerRunner( val messageCollector = MessageCollectorWrapper(messageCollector, outputItemCollector) try { - val targetId = TargetId(args.moduleName, "java-production") + val targetId = TargetId(args.buildFile!!, "java-production") val incrementalCaches = mapOf(targetId to cache) val compilationCanceledStatus = object : CompilationCanceledStatus { override fun checkCanceled() { @@ -467,8 +543,6 @@ class IncrementalJvmCompilerRunner( companion object { const val CACHES_DIR_NAME = "caches" - const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt" - const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin" } }