Format and cleanup incremental-compilation-impl

This commit is contained in:
Alexey Tsvetkov
2019-08-06 22:28:28 +03:00
parent 5816722a64
commit a950226602
11 changed files with 119 additions and 112 deletions
@@ -55,8 +55,7 @@ data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
}
return result
}
}
catch (e: IOException) {
} catch (e: IOException) {
reportFail(e.toString())
}
@@ -76,8 +75,7 @@ data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
output.writeBuildDifference(diff)
}
}
}
catch (e: IOException) {
} catch (e: IOException) {
reporter?.report { "Could not write diff to file $file: $e" }
}
}
@@ -129,7 +127,7 @@ data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
}
}
internal val MAX_DIFFS_ENTRIES: Int = 10
internal const val MAX_DIFFS_ENTRIES: Int = 10
@set:TestOnly
var CURRENT_VERSION: Int = 0
@@ -21,14 +21,13 @@ import java.io.*
data class BuildInfo(val startTS: Long) : Serializable {
companion object {
fun read(file: File): BuildInfo? =
try {
ObjectInputStream(FileInputStream(file)).use {
it.readObject() as BuildInfo
}
}
catch (e: Exception) {
null
try {
ObjectInputStream(FileInputStream(file)).use {
it.readObject() as BuildInfo
}
} catch (e: Exception) {
null
}
fun write(buildInfo: BuildInfo, file: File) {
ObjectOutputStream(FileOutputStream(file)).use {
@@ -23,8 +23,8 @@ import java.io.File
import java.util.*
internal class ChangedJavaFilesProcessor(
private val reporter: ICReporter,
private val psiFileFactory: (File) -> PsiFile?
private val reporter: ICReporter,
private val psiFileFactory: (File) -> PsiFile?
) {
private val allSymbols = HashSet<LookupSymbol>()
@@ -20,8 +20,9 @@ import org.jetbrains.kotlin.name.FqName
internal sealed class ChangesEither {
internal class Known(
val lookupSymbols: Collection<LookupSymbol> = emptyList(),
val fqNames: Collection<FqName> = emptyList()
val lookupSymbols: Collection<LookupSymbol> = emptyList(),
val fqNames: Collection<FqName> = emptyList()
) : ChangesEither()
internal class Unknown(val reason: String? = null) : ChangesEither()
}
@@ -35,9 +35,9 @@ import java.io.File
import java.util.*
abstract class IncrementalCompilerRunner<
Args : CommonCompilerArguments,
CacheManager : IncrementalCachesManager<*>
>(
Args : CommonCompilerArguments,
CacheManager : IncrementalCachesManager<*>
>(
private val workingDir: File,
cacheDirName: String,
protected val reporter: ICReporter,
@@ -48,7 +48,7 @@ abstract class IncrementalCompilerRunner<
) {
protected val cacheDirectory = File(workingDir, cacheDirName)
protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
@@ -57,12 +57,12 @@ abstract class IncrementalCompilerRunner<
protected abstract fun destinationDir(args: Args): File
fun compile(
allSourceFiles: List<File>,
args: Args,
messageCollector: MessageCollector,
// when [providedChangedFiles] is not null, changes are provided by external system (e.g. Gradle)
// otherwise we track source files changes ourselves.
providedChangedFiles: ChangedFiles?
allSourceFiles: List<File>,
args: Args,
messageCollector: MessageCollector,
// when [providedChangedFiles] is not null, changes are provided by external system (e.g. Gradle)
// otherwise we track source files changes ourselves.
providedChangedFiles: ChangedFiles?
): ExitCode {
assert(isICEnabled()) { "Incremental compilation is not enabled" }
var caches = createCacheManager(args)
@@ -95,8 +95,7 @@ abstract class IncrementalCompilerRunner<
if (!caches.close(flush = true)) throw RuntimeException("Could not flush caches")
return exitCode
}
catch (e: Exception) {
} catch (e: Exception) {
// todo: warn?
rebuild { "Possible cache corruption. Rebuilding. $e" }
}
@@ -131,10 +130,10 @@ abstract class IncrementalCompilerRunner<
}
private fun sourcesToCompile(caches: CacheManager, changedFiles: ChangedFiles, args: Args): CompilationMode =
when (changedFiles) {
is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args)
is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" }
}
when (changedFiles) {
is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args)
is ChangedFiles.Unknown -> CompilationMode.Rebuild { "inputs' changes are unknown (first or clean build)" }
}
protected abstract fun calculateSourcesToCompile(caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args): CompilationMode
@@ -156,26 +155,26 @@ abstract class IncrementalCompilerRunner<
}
protected abstract fun updateCaches(
services: Services,
caches: CacheManager,
generatedFiles: List<GeneratedFile>,
changesCollector: ChangesCollector
services: Services,
caches: CacheManager,
generatedFiles: List<GeneratedFile>,
changesCollector: ChangesCollector
)
protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {}
protected open fun postCompilationHook(exitCode: ExitCode) {}
protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List<GeneratedFile>): Iterable<File> =
emptyList()
emptyList()
protected open fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
emptyList()
emptyList()
protected open fun makeServices(
args: Args,
lookupTracker: LookupTracker,
expectActualTracker: ExpectActualTracker,
caches: CacheManager,
compilationMode: CompilationMode
args: Args,
lookupTracker: LookupTracker,
expectActualTracker: ExpectActualTracker,
caches: CacheManager,
compilationMode: CompilationMode
): Services.Builder =
Services.Builder().apply {
register(LookupTracker::class.java, lookupTracker)
@@ -184,19 +183,19 @@ abstract class IncrementalCompilerRunner<
}
protected abstract fun runCompiler(
sourcesToCompile: Set<File>,
args: Args,
caches: CacheManager,
services: Services,
messageCollector: MessageCollector
sourcesToCompile: Set<File>,
args: Args,
caches: CacheManager,
services: Services,
messageCollector: MessageCollector
): ExitCode
private fun compileIncrementally(
args: Args,
caches: CacheManager,
allKotlinSources: List<File>,
compilationMode: CompilationMode,
messageCollector: MessageCollector
args: Args,
caches: CacheManager,
allKotlinSources: List<File>,
compilationMode: CompilationMode,
messageCollector: MessageCollector
): ExitCode {
preBuildHook(args, compilationMode)
@@ -264,10 +263,17 @@ abstract class IncrementalCompilerRunner<
val (dirtyLookupSymbols, dirtyClassFqNames) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
val compiledInThisIterationSet = sourcesToCompile.toHashSet()
with (dirtySources) {
with(dirtySources) {
clear()
addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = compiledInThisIterationSet))
addAll(mapClassesFqNamesToFiles(listOf(caches.platformCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet))
addAll(
mapClassesFqNamesToFiles(
listOf(caches.platformCache),
dirtyClassFqNames,
reporter,
excludes = compiledInThisIterationSet
)
)
}
buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
@@ -26,7 +26,10 @@ import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.js.*
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
import org.jetbrains.kotlin.incremental.js.IncrementalDataProviderFromCache
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
import org.jetbrains.kotlin.library.metadata.KlibMetadataSerializerProtocol
@@ -145,7 +148,7 @@ class IncrementalJsCompilerRunner(
generatedFiles: List<GeneratedFile>,
changesCollector: ChangesCollector
) {
val incrementalResults = services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl
val incrementalResults = services[IncrementalResultsConsumer::class.java] as IncrementalResultsConsumerImpl
val jsCache = caches.platformCache
jsCache.header = incrementalResults.headerMetadata
@@ -25,7 +25,6 @@ import com.intellij.psi.PsiJavaFile
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.JvmSourceRoot
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -47,14 +46,13 @@ import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import java.io.File
import java.io.ObjectOutputStream
fun makeIncrementally(
cachesDir: File,
sourceRoots: Iterable<File>,
args: K2JVMCompilerArguments,
messageCollector: MessageCollector = MessageCollector.NONE,
reporter: ICReporter = EmptyICReporter
cachesDir: File,
sourceRoots: Iterable<File>,
args: K2JVMCompilerArguments,
messageCollector: MessageCollector = MessageCollector.NONE,
reporter: ICReporter = EmptyICReporter
) {
val kotlinExtensions = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
val allExtensions = kotlinExtensions + "java"
@@ -90,14 +88,13 @@ object EmptyICReporter : ICReporterBase() {
}
}
inline fun <R> withIC(enabled: Boolean = true, fn: ()->R): R {
inline fun <R> withIC(enabled: Boolean = true, fn: () -> R): R {
val isEnabledBackup = IncrementalCompilation.isEnabledForJvm()
IncrementalCompilation.setIsEnabledForJvm(enabled)
try {
return fn()
}
finally {
} finally {
IncrementalCompilation.setIsEnabledForJvm(isEnabledBackup)
}
}
@@ -118,15 +115,15 @@ class IncrementalJvmCompilerRunner(
buildHistoryFile = buildHistoryFile
) {
override fun isICEnabled(): Boolean =
IncrementalCompilation.isEnabledForJvm()
IncrementalCompilation.isEnabledForJvm()
override fun createCacheManager(args: K2JVMCompilerArguments): IncrementalJvmCachesManager =
IncrementalJvmCachesManager(cacheDirectory, File(args.destination), reporter)
IncrementalJvmCachesManager(cacheDirectory, File(args.destination), reporter)
override fun destinationDir(args: K2JVMCompilerArguments): File =
args.destinationAsFile
args.destinationAsFile
private var dirtyClasspathChanges: Collection<FqName> = emptySet<FqName>()
private var dirtyClasspathChanges: Collection<FqName> = emptySet()
private val psiFileFactory: PsiFileFactory by lazy {
val rootDisposable = Disposer.newDisposable()
@@ -139,10 +136,10 @@ class IncrementalJvmCompilerRunner(
private val changedUntrackedJavaClasses = mutableSetOf<ClassId>()
private var javaFilesProcessor =
if (!usePreciseJavaTracking)
ChangedJavaFilesProcessor(reporter) { it.psiFile() }
else
null
if (!usePreciseJavaTracking)
ChangedJavaFilesProcessor(reporter) { it.psiFile() }
else
null
override fun calculateSourcesToCompile(
caches: IncrementalJvmCachesManager,
@@ -227,7 +224,7 @@ class IncrementalJvmCompilerRunner(
}
private fun File.psiFile(): PsiFile? =
psiFileFactory.createFileFromText(nameWithoutExtension, JavaLanguage.INSTANCE, readText())
psiFileFactory.createFileFromText(nameWithoutExtension, JavaLanguage.INSTANCE, readText())
private fun processChangedUntrackedJavaClass(psiClass: PsiClass, classId: ClassId) {
changedUntrackedJavaClasses.add(classId)
@@ -259,23 +256,23 @@ class IncrementalJvmCompilerRunner(
override fun postCompilationHook(exitCode: ExitCode) {}
override fun updateCaches(
services: Services,
caches: IncrementalJvmCachesManager,
generatedFiles: List<GeneratedFile>,
changesCollector: ChangesCollector
services: Services,
caches: IncrementalJvmCachesManager,
generatedFiles: List<GeneratedFile>,
changesCollector: ChangesCollector
) {
updateIncrementalCache(
generatedFiles, caches.platformCache, changesCollector,
services[JavaClassesTracker::class.java] as? JavaClassesTrackerImpl
generatedFiles, caches.platformCache, changesCollector,
services[JavaClassesTracker::class.java] as? JavaClassesTrackerImpl
)
}
override fun runWithNoDirtyKotlinSources(caches: IncrementalJvmCachesManager): Boolean =
caches.platformCache.getObsoleteJavaClasses().isNotEmpty() || changedUntrackedJavaClasses.isNotEmpty()
caches.platformCache.getObsoleteJavaClasses().isNotEmpty() || changedUntrackedJavaClasses.isNotEmpty()
override fun additionalDirtyFiles(
caches: IncrementalJvmCachesManager,
generatedFiles: List<GeneratedFile>
caches: IncrementalJvmCachesManager,
generatedFiles: List<GeneratedFile>
): Iterable<File> {
val cache = caches.platformCache
val result = HashSet<File>()
@@ -314,14 +311,14 @@ class IncrementalJvmCompilerRunner(
}
override fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
javaFilesProcessor?.allChangedSymbols ?: emptyList()
javaFilesProcessor?.allChangedSymbols ?: emptyList()
override fun makeServices(
args: K2JVMCompilerArguments,
lookupTracker: LookupTracker,
expectActualTracker: ExpectActualTracker,
caches: IncrementalJvmCachesManager,
compilationMode: CompilationMode
args: K2JVMCompilerArguments,
lookupTracker: LookupTracker,
expectActualTracker: ExpectActualTracker,
caches: IncrementalJvmCachesManager,
compilationMode: CompilationMode
): Services.Builder =
super.makeServices(args, lookupTracker, expectActualTracker, caches, compilationMode).apply {
val targetId = TargetId(args.moduleName!!, "java-production")
@@ -336,11 +333,11 @@ class IncrementalJvmCompilerRunner(
}
override fun runCompiler(
sourcesToCompile: Set<File>,
args: K2JVMCompilerArguments,
caches: IncrementalJvmCachesManager,
services: Services,
messageCollector: MessageCollector
sourcesToCompile: Set<File>,
args: K2JVMCompilerArguments,
caches: IncrementalJvmCachesManager,
services: Services,
messageCollector: MessageCollector
): ExitCode {
val compiler = K2JVMCompiler()
val freeArgsBackup = args.freeArgs.toList()
@@ -353,9 +350,13 @@ class IncrementalJvmCompilerRunner(
}
var K2JVMCompilerArguments.destinationAsFile: File
get() = File(destination)
set(value) { destination = value.path }
get() = File(destination)
set(value) {
destination = value.path
}
var K2JVMCompilerArguments.classpathAsList: List<File>
get() = classpath.orEmpty().split(File.pathSeparator).map(::File)
set(value) { classpath = value.joinToString(separator = File.pathSeparator, transform = { it.path }) }
set(value) {
classpath = value.joinToString(separator = File.pathSeparator, transform = { it.path })
}
@@ -19,18 +19,17 @@ package org.jetbrains.kotlin.incremental
import com.intellij.util.containers.MultiMap
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap
import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.SourceToOutputFilesMap
import java.io.File
import java.util.*
import kotlin.collections.HashSet
class InputsCache(
workingDir: File,
private val reporter: ICReporter
workingDir: File,
private val reporter: ICReporter
) : BasicMapsOwner(workingDir) {
companion object {
private val SOURCE_SNAPSHOTS = "source-snapshot"
private val SOURCE_TO_OUTPUT_FILES = "source-to-output"
private const val SOURCE_SNAPSHOTS = "source-snapshot"
private const val SOURCE_TO_OUTPUT_FILES = "source-to-output"
}
internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile))
@@ -38,7 +37,7 @@ class InputsCache(
fun removeOutputForSourceFiles(sources: Iterable<File>) {
for (sourceFile in sources) {
sourceToOutputMap.remove(sourceFile).forEach { it ->
sourceToOutputMap.remove(sourceFile).forEach {
reporter.reportVerbose { "Deleting $it on clearing cache for $sourceFile" }
it.delete()
}
@@ -192,7 +192,7 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH
private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable<String>): Either<Set<File>> {
val possibleModules =
moduleNames.flatMapTo(HashSet<IncrementalModuleEntry>()) { modulesInfo.nameToModules[it] ?: emptySet() }
moduleNames.flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() }
val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(path) }
if (modules.isEmpty()) return Either.Error("Unknown module for $path (candidates: ${possibleModules.joinToString()})")
@@ -20,9 +20,9 @@ import java.io.File
import java.util.*
class FileSnapshot(
val file: File,
val length: Long,
val hash: ByteArray
val file: File,
val length: Long,
val hash: ByteArray
) {
init {
assert(!file.isDirectory)
@@ -24,7 +24,7 @@ import java.util.*
class FileSnapshotMap(storageFile: File) : BasicStringMap<FileSnapshot>(storageFile, PathStringDescriptor, FileSnapshotExternalizer) {
override fun dumpValue(value: FileSnapshot): String =
value.toString()
value.toString()
fun compareAndUpdate(newFiles: Iterable<File>): ChangedFiles.Known {
val snapshotProvider = SimpleFileSnapshotProviderImpl()