Begin extracting common IC code

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental
 import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
 import java.io.File
 
-abstract class IncrementalCachesManager (
+abstract class IncrementalCachesManager<PlatformCache : IncrementalCacheCommon>(
     protected val cachesRootDir: File,
     protected val reporter: ICReporter
 ) {
@@ -33,6 +33,7 @@ abstract class IncrementalCachesManager (
 
     val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter).apply { registerCache() }
     val lookupCache: LookupStorage = LookupStorage(lookupCacheDir).apply { registerCache() }
+    abstract val platformCache: PlatformCache
 
     fun clean() {
         caches.forEach { it.clean() }
@@ -70,8 +71,8 @@ class IncrementalJvmCachesManager(
     cacheDirectory: File,
     outputDir: File,
     reporter: ICReporter
-) : IncrementalCachesManager(cacheDirectory, reporter) {
+) : IncrementalCachesManager<IncrementalCacheImpl>(cacheDirectory, reporter) {
 
-    private val jvmCacheFile = File(cacheDirectory, "jvm").apply { mkdirs() }
+    private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() }
-    val jvmCache = IncrementalCacheImpl(jvmCacheFile, outputDir).apply { registerCache() }
+    override val platformCache = IncrementalCacheImpl(jvmCacheDir, outputDir).apply { registerCache() }
 }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -19,8 +19,10 @@ package org.jetbrains.kotlin.incremental
 import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.build.GeneratedJvmClass
+import org.jetbrains.kotlin.build.isModuleMappingFile
 import org.jetbrains.kotlin.build.JvmSourceRoot
 import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
@@ -38,6 +40,7 @@ import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.modules.TargetId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
+import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import java.io.File
 import java.util.*
 
@@ -84,48 +87,54 @@ inline fun <R> withIC(enabled: Boolean = true, fn: ()->R): R {
     }
 }
 
-class IncrementalJvmCompilerRunner(
+abstract class IncrementalCompilerRunner<
-        workingDir: File,
+    Args : CommonCompilerArguments,
-        private val javaSourceRoots: Set<JvmSourceRoot>,
+    CacheManager : IncrementalCachesManager<*>
-        private val cacheVersions: List<CacheVersion>,
+>(
-        private val reporter: ICReporter,
+    workingDir: File,
-        private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
+    cacheDirName: String,
-        private val artifactChangesProvider: ArtifactChangesProvider? = null,
+    protected val cacheVersions: List<CacheVersion>,
-        private val changesRegistry: ChangesRegistry? = null
+    protected val reporter: ICReporter,
+    protected val artifactChangesProvider: ArtifactChangesProvider?,
+    protected val changesRegistry: ChangesRegistry?
 ) {
-    private var anyClassesCompiled: Boolean = false
+
-    private val cacheDirectory = File(workingDir, CACHES_DIR_NAME)
+    protected val cacheDirectory = File(workingDir, cacheDirName)
-    private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
+    protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
-    private val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
+    protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
+
+    protected abstract fun isICEnabled(): Boolean
+    protected abstract fun createCacheManager(args: Args): CacheManager
+    protected abstract fun destionationDir(args: Args): File
 
     fun compile(
             allKotlinSources: List<File>,
-            args: K2JVMCompilerArguments,
+            args: Args,
             messageCollector: MessageCollector,
-            getChangedFiles: (IncrementalCachesManager)->ChangedFiles
+            getChangedFiles: (CacheManager)->ChangedFiles
     ): ExitCode {
-        var caches = IncrementalJvmCachesManager(cacheDirectory, File(args.destination), reporter)
+        assert(isICEnabled()) { "Incremental compilation is not enabled" }
+        var caches = createCacheManager(args)
 
         fun rebuild(): ExitCode {
             caches.clean()
             dirtySourcesSinceLastTimeFile.delete()
-            args.destinationAsFile.deleteRecursively()
+            destionationDir(args).deleteRecursively()
 
-            val javaFilesProcessor = ChangedJavaFilesProcessor(reporter)
+            caches = createCacheManager(args)
-            caches = IncrementalJvmCachesManager(cacheDirectory, args.destinationAsFile, reporter)
+            return compileIncrementally(args, caches, allKotlinSources, CompilationMode.Rebuild(), messageCollector)
-            return compileIncrementally(args, caches, javaFilesProcessor, allKotlinSources, CompilationMode.Rebuild, messageCollector)
         }
 
         return try {
-            val javaFilesProcessor = ChangedJavaFilesProcessor(reporter)
             val changedFiles = getChangedFiles(caches)
-            val compilationMode = calculateSourcesToCompile(javaFilesProcessor, caches, changedFiles, args)
+            val compilationMode = sourcesToCompile(caches, changedFiles, args)
 
             val exitCode = when (compilationMode) {
                 is CompilationMode.Incremental -> {
-                    compileIncrementally(args, caches, javaFilesProcessor, allKotlinSources, compilationMode, messageCollector)
+                    compileIncrementally(args, caches, allKotlinSources, compilationMode, messageCollector)
                 }
                 is CompilationMode.Rebuild -> {
+                    reporter.report { "Non-incremental compilation will be performed: ${compilationMode.reason}" }
                     rebuild()
                 }
             }
@@ -141,52 +150,110 @@ class IncrementalJvmCompilerRunner(
         }
     }
 
-    private data class CompileChangedResults(val exitCode: ExitCode, val generatedFiles: List<GeneratedFile<TargetId>>)
+    private fun sourcesToCompile(caches: CacheManager, changedFiles: ChangedFiles, args: Args): CompilationMode =
+            when (changedFiles) {
+                is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args)
+                is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" }
+            }
 
-    private sealed class CompilationMode {
+    protected open fun calculateSourcesToCompile(caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args): CompilationMode =
-        class Incremental(val dirtyFiles: Set<File>) : CompilationMode()
+            CompilationMode.Incremental(getDirtyFiles(changedFiles))
-        object Rebuild : CompilationMode()
-    }
 
-    private fun calculateSourcesToCompile(
+    protected fun getDirtyFiles(changedFiles: ChangedFiles.Known): HashSet<File> {
-            javaFilesProcessor: ChangedJavaFilesProcessor,
+        val dirtyFiles = HashSet<File>(with(changedFiles) { modified.size + removed.size })
-            caches: IncrementalJvmCachesManager,
+        with(changedFiles) {
-            changedFiles: ChangedFiles,
+            modified.asSequence() + removed.asSequence()
-            args: K2JVMCompilerArguments
+        }.forEach { if (it.isKotlinFile()) dirtyFiles.add(it) }
-    ): CompilationMode {
+
-        fun rebuild(reason: ()->String): CompilationMode {
+        if (dirtySourcesSinceLastTimeFile.exists()) {
-            reporter.report {"Non-incremental compilation will be performed: ${reason()}"}
+            val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists)
-            return CompilationMode.Rebuild
+            if (files.isNotEmpty()) {
+                reporter.report { "Source files added since last compilation: ${reporter.pathsAsString(files)}" }
+            }
+
+            dirtyFiles.addAll(files)
         }
 
-        if (changedFiles !is ChangedFiles.Known) return rebuild {"inputs' changes are unknown (first or clean build)"}
+        return dirtyFiles
+    }
 
+    protected open fun markOutputDirty(caches: CacheManager, dirtySources: List<File>) {
+    }
+
+    protected abstract fun compareAndUpdateCache(caches: CacheManager, generatedFiles: List<GeneratedFile<*>>): CompilationResult
+
+    protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {}
+    protected open fun postCompilationHook(exitCode: ExitCode) {}
+    protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List<GeneratedFile<*>>): Iterable<File> =
+            emptyList()
+
+    protected abstract fun compileIncrementally(args: Args, caches: CacheManager, allKotlinSources: List<File>, compilationMode: CompilationMode, messageCollector: MessageCollector): ExitCode
+
+    protected data class CompileChangedResults(val exitCode: ExitCode, val generatedFiles: List<GeneratedFile<TargetId>>)
+
+    protected sealed class CompilationMode {
+        class Incremental(val dirtyFiles: Set<File>) : CompilationMode()
+        class Rebuild(getReason: ()->String = { "" }) : CompilationMode() {
+            val reason: String by lazy(getReason)
+        }
+    }
+
+    companion object {
+        const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
+        const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
+    }
+}
+
+class IncrementalJvmCompilerRunner(
+        workingDir: File,
+        private val javaSourceRoots: Set<JvmSourceRoot>,
+        cacheVersions: List<CacheVersion>,
+        reporter: ICReporter,
+        private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
+        artifactChangesProvider: ArtifactChangesProvider? = null,
+        changesRegistry: ChangesRegistry? = null
+) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
+        workingDir,
+        CACHES_DIR_NAME,
+        cacheVersions,
+        reporter,
+        artifactChangesProvider,
+        changesRegistry
+) {
+    override fun isICEnabled(): Boolean =
+            IncrementalCompilation.isEnabled()
+
+    override fun createCacheManager(args: K2JVMCompilerArguments): IncrementalJvmCachesManager =
+            IncrementalJvmCachesManager(cacheDirectory, File(args.destination), reporter)
+
+    override fun destionationDir(args: K2JVMCompilerArguments): File =
+            args.destinationAsFile
+
+    private var javaFilesProcessor = ChangedJavaFilesProcessor(reporter)
+
+    override fun calculateSourcesToCompile(caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments): CompilationMode {
         val removedClassFiles = changedFiles.removed.filter(File::isClassFile)
-        if (removedClassFiles.any()) return rebuild {"Removed class files: ${reporter.pathsAsString(removedClassFiles)}"}
+        if (removedClassFiles.any()) return CompilationMode.Rebuild { "Removed class files: ${reporter.pathsAsString(removedClassFiles)}" }
 
         val modifiedClassFiles = changedFiles.modified.filter(File::isClassFile)
-        if (modifiedClassFiles.any()) return rebuild {"Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}"}
+        if (modifiedClassFiles.any()) return CompilationMode.Rebuild { "Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}" }
 
         val classpathSet = args.classpathAsList.toHashSet()
-        val modifiedClasspathEntries = changedFiles.modified.filter {it in classpathSet}
+        val modifiedClasspathEntries = changedFiles.modified.filter { it in classpathSet }
         val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
         reporter.report { "Last Kotlin Build info -- $lastBuildInfo" }
         val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo)
         if (classpathChanges !is ChangesEither.Known) {
-            return rebuild {"could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}"}
+            return CompilationMode.Rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
         }
 
         val javaFilesChanges = javaFilesProcessor.process(changedFiles)
         val affectedJavaSymbols = when (javaFilesChanges) {
             is ChangesEither.Known -> javaFilesChanges.lookupSymbols
-            is ChangesEither.Unknown -> return rebuild {"Could not get changes for java files"}
+            is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" }
         }
 
-        val dirtyFiles = HashSet<File>(with(changedFiles) {modified.size + removed.size})
+        val dirtyFiles = getDirtyFiles(changedFiles)
-        with(changedFiles) {
-            modified.asSequence() + removed.asSequence()
-        }.forEach {if (it.isKotlinFile()) dirtyFiles.add(it)}
-
         val lookupSymbols = HashSet<LookupSymbol>()
         lookupSymbols.addAll(affectedJavaSymbols)
         lookupSymbols.addAll(classpathChanges.lookupSymbols)
@@ -196,21 +263,12 @@ class IncrementalJvmCompilerRunner(
             dirtyFiles.addAll(dirtyFilesFromLookups)
         }
 
-        val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.jvmCache)) }
+        val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.platformCache)) }
         if (dirtyClassesFqNames.any()) {
-            val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.jvmCache), dirtyClassesFqNames, reporter)
+            val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.platformCache), dirtyClassesFqNames, reporter)
             dirtyFiles.addAll(dirtyFilesFromFqNames)
         }
 
-        if (dirtySourcesSinceLastTimeFile.exists()) {
-            val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists)
-            if (files.isNotEmpty()) {
-                reporter.report {"Source files added since last compilation: ${reporter.pathsAsString(files)}"}
-            }
-
-            dirtyFiles.addAll(files)
-        }
-
         return CompilationMode.Incremental(dirtyFiles)
     }
 
@@ -248,29 +306,49 @@ class IncrementalJvmCompilerRunner(
         return ChangesEither.Known(symbols, fqNames)
     }
 
-    private fun compileIncrementally(
+    override fun preBuildHook(args: K2JVMCompilerArguments, compilationMode: CompilationMode) {
+        when (compilationMode) {
+            is CompilationMode.Incremental -> {
+                args.classpathAsList += args.destinationAsFile.apply { mkdirs() }
+            }
+            is CompilationMode.Rebuild -> {
+                // there is no point in updating annotation file since all files will be compiled anyway
+                kaptAnnotationsFileUpdater = null
+            }
+        }
+    }
+
+    private var outdatedClasses: Iterable<JvmClassName> = emptyList()
+    override fun markOutputDirty(caches: IncrementalJvmCachesManager, dirtySources: List<File>) {
+        outdatedClasses = caches.platformCache.classesBySources(dirtySources)
+        caches.platformCache.markOutputClassesDirty(dirtySources)
+        super.markOutputDirty(caches, dirtySources)
+    }
+
+    override fun postCompilationHook(exitCode: ExitCode) {
+        if (exitCode == ExitCode.OK) {
+            kaptAnnotationsFileUpdater?.updateAnnotations(outdatedClasses)
+        }
+        else {
+            kaptAnnotationsFileUpdater?.revert()
+        }
+    }
+
+    override fun compareAndUpdateCache(caches: IncrementalJvmCachesManager, generatedFiles: List<GeneratedFile<*>>): CompilationResult =
+        updateIncrementalCache(generatedFiles, caches.platformCache)
+
+    override fun compileIncrementally(
             args: K2JVMCompilerArguments,
             caches: IncrementalJvmCachesManager,
-            javaFilesProcessor: ChangedJavaFilesProcessor,
             allKotlinSources: List<File>,
             compilationMode: CompilationMode,
             messageCollector: MessageCollector
     ): ExitCode {
-        assert(IncrementalCompilation.isEnabled()) { "Incremental compilation is not enabled" }
+        preBuildHook(args, compilationMode)
 
-        val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
+        val dirtySources = when (compilationMode) {
-        val dirtySources: MutableList<File>
+            is CompilationMode.Incremental -> ArrayList(compilationMode.dirtyFiles)
-
+            is CompilationMode.Rebuild -> allKotlinSources.toMutableList()
-        when (compilationMode) {
-            is CompilationMode.Incremental -> {
-                dirtySources = ArrayList(compilationMode.dirtyFiles)
-                args.classpathAsList += args.destinationAsFile.apply { mkdirs() }
-            }
-            is CompilationMode.Rebuild -> {
-                dirtySources = allKotlinSources.toMutableList()
-                // there is no point in updating annotation file since all files will be compiled anyway
-                kaptAnnotationsFileUpdater = null
-            }
         }
 
         val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
@@ -280,12 +358,13 @@ class IncrementalJvmCompilerRunner(
         val allSourcesToCompile = HashSet<File>()
 
         var exitCode = ExitCode.OK
+        val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
+
         while (dirtySources.any()) {
-            val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
+            markOutputDirty(caches, dirtySources)
-            val outdatedClasses = caches.jvmCache.classesBySources(dirtySources)
-            caches.jvmCache.markOutputClassesDirty(dirtySources)
             caches.inputsCache.removeOutputForSourceFiles(dirtySources)
 
+            val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
             val (sourcesToCompile, removedKotlinSources) = dirtySources.partition(File::exists)
 
             // todo: more optimal to save only last iteration, but it will require adding standalone-ic specific logs
@@ -294,22 +373,20 @@ class IncrementalJvmCompilerRunner(
             val text = allSourcesToCompile.joinToString(separator = System.getProperty("line.separator")) { it.canonicalPath }
             dirtySourcesSinceLastTimeFile.writeText(text)
 
-            val compilerOutput = compileChanged(sourcesToCompile.toSet(), args, caches.jvmCache, lookupTracker, messageCollector)
+            val compilerOutput = compileChanged(sourcesToCompile.toSet(), args, caches.platformCache, lookupTracker, messageCollector)
+
             exitCode = compilerOutput.exitCode
+            postCompilationHook(exitCode)
+
+            if (exitCode != ExitCode.OK) break
+
+            dirtySourcesSinceLastTimeFile.delete()
+
             val generatedFiles = compilerOutput.generatedFiles
-            anyClassesCompiled = anyClassesCompiled || generatedFiles.isNotEmpty() || removedKotlinSources.isNotEmpty()
-
-            if (exitCode == ExitCode.OK) {
-                dirtySourcesSinceLastTimeFile.delete()
-                kaptAnnotationsFileUpdater?.updateAnnotations(outdatedClasses)
-            } else {
-                kaptAnnotationsFileUpdater?.revert()
-                break
-            }
-
             if (compilationMode is CompilationMode.Incremental) {
+                // todo: feels dirty, can this be refactored?
                 val dirtySourcesSet = dirtySources.toHashSet()
-                val additionalDirtyFiles = additionalDirtyFiles(caches.jvmCache, generatedFiles).filter { it !in dirtySourcesSet }
+                val additionalDirtyFiles = additionalDirtyFiles(caches, generatedFiles).filter { it !in dirtySourcesSet }
                 if (additionalDirtyFiles.isNotEmpty()) {
                     dirtySources.addAll(additionalDirtyFiles)
                     continue
@@ -318,21 +395,18 @@ class IncrementalJvmCompilerRunner(
 
             allGeneratedFiles.addAll(generatedFiles)
             caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
-            val compilationResult = updateIncrementalCache(generatedFiles, caches.platformCache)
-
             caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
+            val compilationResult = compareAndUpdateCache(caches, generatedFiles)
 
-            if (compilationMode is CompilationMode.Rebuild) {
+            if (compilationMode is CompilationMode.Rebuild) break
-                break
-            }
 
-            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.jvmCache), reporter)
+            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.platformCache), reporter)
             val compiledInThisIterationSet = sourcesToCompile.toHashSet()
 
             with (dirtySources) {
                 clear()
                 addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = compiledInThisIterationSet))
-                addAll(mapClassesFqNamesToFiles(listOf(caches.jvmCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet))
+                addAll(mapClassesFqNamesToFiles(listOf(caches.platformCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet))
             }
 
             buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
@@ -360,10 +434,11 @@ class IncrementalJvmCompilerRunner(
         return exitCode
     }
 
-    private fun additionalDirtyFiles(
+    override fun additionalDirtyFiles(
-            cache: IncrementalCacheImpl,
+            caches: IncrementalJvmCachesManager,
-            generatedFiles: List<GeneratedFile<TargetId>>
+            generatedFiles: List<GeneratedFile<*>>
-    ): Collection<File> {
+    ): Iterable<File> {
+        val cache = caches.platformCache
         val result = HashSet<File>()
 
         fun partsByFacadeName(facadeInternalName: String): List<File> {
@@ -382,6 +457,7 @@ class IncrementalJvmCompilerRunner(
                     val cachedSourceFile = cache.getSourceFileIfClass(fqName)
 
                     if (cachedSourceFile != null) {
+                        // todo: seems useless, remove?
                         result.add(cachedSourceFile)
                     }
                 }
@@ -424,7 +500,7 @@ class IncrementalJvmCompilerRunner(
         val messageCollector = MessageCollectorWrapper(messageCollector, outputItemCollector)
 
         try {
-            val targetId = TargetId(args.moduleName, "java-production")
+            val targetId = TargetId(args.buildFile!!, "java-production")
             val incrementalCaches = mapOf(targetId to cache)
             val compilationCanceledStatus = object : CompilationCanceledStatus {
                 override fun checkCanceled() {
@@ -467,8 +543,6 @@ class IncrementalJvmCompilerRunner(
 
     companion object {
         const val CACHES_DIR_NAME = "caches"
-        const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
-        const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
     }
 }