azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring: introduce IncReporter to report IC progress

Browse Source
This commit is contained in:
Alexey Tsvetkov
2016-10-03 21:35:41 +03:00
parent e32c19ce4e
commit fafde1e948
7 changed files with 110 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/IncReporter.kt
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.incremental
import java.io.File
abstract class IncReporter {
abstract fun report(message: ()->String)
abstract fun pathsAsString(files: Iterable<File>): String
fun pathsAsString(vararg files: File): String =
pathsAsString(files.toList())
}
build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt
+6 -8
View File
@@ -187,13 +187,13 @@ data class DirtyData(
fun <Target> CompilationResult.getDirtyData(
caches: Iterable<IncrementalCacheImpl<Target>>,
log: (String)->Unit
reporter: IncReporter
): DirtyData {
val dirtyLookupSymbols = HashSet<LookupSymbol>()
val dirtyClassesFqNames = HashSet<FqName>()
for (change in changes) {
log("Process $change")
reporter.report { "Process $change" }
if (change is ChangeInfo.SignatureChanged) {
val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, caches)
@@ -225,15 +225,14 @@ fun <Target> CompilationResult.getDirtyData(
fun mapLookupSymbolsToFiles(
lookupStorage: LookupStorage,
lookupSymbols: Iterable<LookupSymbol>,
log: (String)->Unit,
getLogFilePath: (File)->String = { it.canonicalPath },
reporter: IncReporter,
excludes: Set<File> = emptySet()
): Set<File> {
val dirtyFiles = HashSet<File>()
for (lookup in lookupSymbols) {
val affectedFiles = lookupStorage.get(lookup).map(::File).filter { it !in excludes }
log("${lookup.scope}#${lookup.name} caused recompilation of: ${affectedFiles.map(getLogFilePath)}")
reporter.report { "${lookup.scope}#${lookup.name} caused recompilation of: ${reporter.pathsAsString(affectedFiles)}" }
dirtyFiles.addAll(affectedFiles)
}
@@ -243,8 +242,7 @@ fun mapLookupSymbolsToFiles(
fun <Target> mapClassesFqNamesToFiles(
caches: Iterable<IncrementalCacheImpl<Target>>,
classesFqNames: Iterable<FqName>,
log: (String)->Unit,
getLogFilePath: (File)->String = { it.canonicalPath },
reporter: IncReporter,
excludes: Set<File> = emptySet()
): Set<File> {
val dirtyFiles = HashSet<File>()
@@ -254,7 +252,7 @@ fun <Target> mapClassesFqNamesToFiles(
val srcFile = cache.getSourceFileIfClass(dirtyClassFqName)
if (srcFile == null || srcFile in excludes) continue
log("Class $dirtyClassFqName caused recompilation of: ${getLogFilePath(srcFile)}")
reporter.report { ("Class $dirtyClassFqName caused recompilation of: ${reporter.pathsAsString(srcFile)}") }
dirtyFiles.add(srcFile)
}
}
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+17 -6
View File
@@ -788,6 +788,17 @@ private fun CompilationResult.doProcessChanges(
}
}
private class JpsIncReporter : IncReporter() {
override fun report(message: ()->String) {
if (KotlinBuilder.LOG.isDebugEnabled) {
KotlinBuilder.LOG.debug(message())
}
}
override fun pathsAsString(files: Iterable<File>): String =
files.map { it.canonicalPath }.joinToString()
}
private fun CompilationResult.doProcessChangesUsingLookups(
compiledFiles: Set<File>,
dataManager: BuildDataManager,
@@ -796,16 +807,16 @@ private fun CompilationResult.doProcessChangesUsingLookups(
) {
val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider)
val allCaches = caches.flatMap { it.thisWithDependentCaches }
val logAction = { logStr: String -> KotlinBuilder.LOG.debug(logStr) }
val reporter = JpsIncReporter()
logAction("Start processing changes")
reporter.report { "Start processing changes" }
val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(allCaches, logAction)
val dirtyFiles = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction) +
mapClassesFqNamesToFiles(allCaches, dirtyClassFqNames, logAction)
val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(allCaches, reporter)
val dirtyFiles = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, reporter) +
mapClassesFqNamesToFiles(allCaches, dirtyClassFqNames, reporter)
fsOperations.markFiles(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
logAction("End of processing changes")
reporter.report { "End of processing changes" }
}
private fun getLookupTracker(project: JpsProject): LookupTracker {
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncReporter.kt
+17
View File
@@ -0,0 +1,17 @@
package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.logging.Logging
import org.jetbrains.kotlin.incremental.IncReporter
import java.io.File
internal class GradleIncReporter(private val projectRootFile: File) : IncReporter() {
private val log = Logging.getLogger(GradleIncReporter::class.java)
override fun report(message: ()->String) {
log.kotlinDebug(message)
}
override fun pathsAsString(files: Iterable<File>): String =
files.pathsAsStringRelativeTo(projectRootFile)
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt
+17 -28
View File
@@ -78,7 +78,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
@TaskAction
fun execute(inputs: IncrementalTaskInputs): Unit {
val allKotlinSources = getKotlinSources()
logger.kotlinDebug { "all kotlin sources: ${filesToString(allKotlinSources)}" }
logger.kotlinDebug { "all kotlin sources: ${allKotlinSources.pathsAsStringRelativeTo(project.rootProject.projectDir)}" }
if (allKotlinSources.isEmpty()) {
logger.kotlinDebug { "No Kotlin files found, skipping Kotlin compiler task" }
@@ -93,13 +93,6 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
private fun getKotlinSources(): List<File> = (getSource() as Iterable<File>).filter(File::isKotlinFile)
internal abstract fun callCompiler(args: T, allKotlinSources: List<File>, changedFiles: ChangedFiles)
internal fun relativePathOrCanonical(file: File): String =
file.relativeToOrNull(project.rootProject.projectDir)?.path
?: file.canonicalPath
internal fun filesToString(files: Iterable<File>) =
"[" + files.map { relativePathOrCanonical(it) }.sorted().joinToString(separator = ", \n") + "]"
}
open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), KotlinJvmCompile {
@@ -175,8 +168,6 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
override fun callCompiler(args: K2JVMCompilerArguments, allKotlinSources: List<File>, changedFiles: ChangedFiles) {
val outputDir = destinationDir
val logAction = { logStr: String -> logger.kotlinInfo(logStr) }
val relativePathOrCanonical = { file: File -> relativePathOrCanonical(file) }
if (!incremental) {
anyClassesCompiled = true
@@ -190,33 +181,31 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
val targetId = TargetId(name = args.moduleName, type = "java-production")
val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
logger.kotlinDebug { "Last Kotlin Build info for task $path -- $lastBuildInfo" }
val caches = IncrementalCachesManager(targetId, cacheDirectory, outputDir)
val reporter = GradleIncReporter(project.rootProject.projectDir)
reporter.report { "Last Kotlin Build info for task $path -- $lastBuildInfo" }
val compilationMode = calculateSourcesToCompile(javaFilesProcessor,
caches,
lastBuildInfo,
changedFiles,
logger,
classpath.toList(),
dirtySourcesSinceLastTimeFile,
{ filesToString(it) },
logAction, { relativePathOrCanonical(it) },
artifactDifferenceRegistry)
compileIncrementally(args, caches, javaFilesProcessor, logAction, lookupTracker, outputDir, relativePathOrCanonical, allKotlinSources, targetId, compilationMode)
artifactDifferenceRegistry,
reporter)
compileIncrementally(args, caches, javaFilesProcessor, lookupTracker, outputDir, allKotlinSources, targetId, compilationMode, reporter)
}
private fun compileIncrementally(
args: K2JVMCompilerArguments,
caches: IncrementalCachesManager,
javaFilesProcessor: ChangedJavaFilesProcessor,
logAction: (String) -> Unit,
lookupTracker: LookupTrackerImpl,
outputDir: File,
relativePathOrCanonical: (File) -> String,
allKotlinSources: List<File>,
targetId: TargetId,
compilationMode: CompilationMode
compilationMode: CompilationMode,
reporter: IncReporter
) {
val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
val dirtySources: MutableList<File>
@@ -237,7 +226,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
@Suppress("NAME_SHADOWING")
var compilationMode = compilationMode
logger.kotlinDebug { "Artifact to register difference for task $path: $artifactFile" }
reporter.report { "Artifact to register difference for task $path: $artifactFile" }
val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
@@ -251,7 +240,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
val (sourcesToCompile, removedKotlinSources) = dirtySources.partition { it.isFile }
if (sourcesToCompile.isNotEmpty()) {
logger.kotlinDebug { "compile iteration: ${sourcesToCompile.joinToString(transform = relativePathOrCanonical)}" }
reporter.report { "compile iteration: ${reporter.report { reporter.pathsAsString(sourcesToCompile) }}" }
}
val text = sourcesToCompile.map { it.canonicalPath }.joinToString(separator = System.getProperty("line.separator"))
@@ -284,17 +273,17 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
break
}
val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.incrementalCache), logAction)
val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.incrementalCache), reporter)
val generatedJavaFilesChanges = javaFilesProcessor.process(generatedJavaFilesDiff)
val compiledInThisIterationSet = sourcesToCompile.toHashSet()
val dirtyKotlinFilesFromJava = when (generatedJavaFilesChanges) {
is ChangesEither.Unknown -> {
logger.kotlinDebug { "Could not get changes for generated java files, recompiling all kotlin" }
reporter.report { "Could not get changes for generated java files, recompiling all kotlin" }
compilationMode = CompilationMode.Rebuild()
allKotlinSources.toSet()
}
is ChangesEither.Known -> {
mapLookupSymbolsToFiles(caches.lookupCache, generatedJavaFilesChanges.lookupSymbols, logAction, relativePathOrCanonical, excludes = compiledInThisIterationSet)
mapLookupSymbolsToFiles(caches.lookupCache, generatedJavaFilesChanges.lookupSymbols, reporter, excludes = compiledInThisIterationSet)
}
else -> throw IllegalStateException("Unknown ChangesEither implementation: $generatedJavaFiles")
}
@@ -302,8 +291,8 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
with (dirtySources) {
clear()
addAll(dirtyKotlinFilesFromJava)
addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, logAction, relativePathOrCanonical, excludes = compiledInThisIterationSet))
addAll(mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassFqNames, logAction, relativePathOrCanonical, excludes = compiledInThisIterationSet))
addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = compiledInThisIterationSet))
addAll(mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet))
}
buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
@@ -319,7 +308,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
val artifactDifference = ArtifactDifference(currentBuildInfo.startTS, dirtyData)
artifactDifferenceRegistry!!.add(artifactFile!!, artifactDifference)
logger.kotlinDebug {
reporter.report {
val dirtySymbolsSorted = buildDirtyLookupSymbols.map { it.scope + "#" + it.name }.sorted()
"Added artifact difference for $artifactFile (ts: ${currentBuildInfo.startTS}): " +
"[\n\t${dirtySymbolsSorted.joinToString(",\n\t")}]"
@@ -327,7 +316,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
}
caches.close(flush = true)
logger.kotlinDebug { "flushed incremental caches" }
reporter.report { "flushed incremental caches" }
processCompilerExitCode(exitCode)
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/fileUtils.kt
+5
View File
@@ -17,3 +17,8 @@ internal fun File.isClassFile(): Boolean =
internal fun listClassFiles(path: String): Sequence<File> =
File(path).walk().filter { it.isFile && it.isClassFile() }
internal fun File.relativeOrCanonical(base: File): String =
relativeToOrNull(base)?.path ?: canonicalPath
internal fun Iterable<File>.pathsAsStringRelativeTo(base: File): String =
map { it.relativeOrCanonical(base) }.sorted().joinToString()
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/incrementalCompilation.kt
+21 -28
View File
@@ -1,11 +1,7 @@
package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.incremental.LookupSymbol
import org.jetbrains.kotlin.incremental.mapClassesFqNamesToFiles
import org.jetbrains.kotlin.incremental.mapLookupSymbolsToFiles
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff
import org.jetbrains.kotlin.incremental.withSubtypes
import org.jetbrains.kotlin.name.FqName
import java.io.File
import java.util.*
@@ -20,34 +16,31 @@ internal fun calculateSourcesToCompile(
caches: IncrementalCachesManager,
lastBuildInfo: BuildInfo?,
changedFiles: ChangedFiles,
logger: Logger,
classpath: Iterable<File>,
dirtySourcesSinceLastTimeFile: File,
filesToString: (Iterable<File>)-> String,
logAction: (String)->Unit,
relativePathOrCanonical: (File)->String,
artifactDifferenceRegistry: ArtifactDifferenceRegistry?
artifactDifferenceRegistry: ArtifactDifferenceRegistry?,
reporter: IncReporter
): CompilationMode {
fun rebuild(reason: String): CompilationMode {
logger.kotlinInfo("Non-incremental compilation will be performed: $reason")
fun rebuild(reason: ()->String): CompilationMode {
reporter.report { "Non-incremental compilation will be performed: ${reason()}" }
caches.clean()
dirtySourcesSinceLastTimeFile.delete()
return CompilationMode.Rebuild()
}
if (changedFiles !is ChangedFiles.Known) return rebuild("inputs' changes are unknown (first or clean build)")
if (changedFiles !is ChangedFiles.Known) return rebuild { "inputs' changes are unknown (first or clean build)" }
val removedClassFiles = changedFiles.removed.filter(File::isClassFile)
if (removedClassFiles.any()) return rebuild("Removed class files: ${filesToString(removedClassFiles)}")
if (removedClassFiles.any()) return rebuild { "Removed class files: ${reporter.pathsAsString(removedClassFiles)}" }
val modifiedClassFiles = changedFiles.modified.filter(File::isClassFile)
if (modifiedClassFiles.any()) return rebuild("Modified class files: ${filesToString(modifiedClassFiles)}")
if (modifiedClassFiles.any()) return rebuild { "Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}" }
val classpathSet = classpath.toHashSet()
val modifiedClasspathEntries = changedFiles.modified.filter { it in classpathSet }
val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo, logger, artifactDifferenceRegistry)
val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo, artifactDifferenceRegistry, reporter)
if (classpathChanges is ChangesEither.Unknown) {
return rebuild("could not get changes from modified classpath entries: ${filesToString(modifiedClasspathEntries)}")
return rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
}
if (classpathChanges !is ChangesEither.Known) {
throw AssertionError("Unknown implementation of ChangesEither: ${classpathChanges.javaClass}")
@@ -58,7 +51,7 @@ internal fun calculateSourcesToCompile(
val javaFilesChanges = javaFilesProcessor.process(javaFilesDiff)
val affectedJavaSymbols = when (javaFilesChanges) {
is ChangesEither.Known -> javaFilesChanges.lookupSymbols
is ChangesEither.Unknown -> return rebuild("Could not get changes for java files")
is ChangesEither.Unknown -> return rebuild { "Could not get changes for java files" }
}
val dirtyFiles = HashSet<File>(with(changedFiles) { modified.size + removed.size })
@@ -71,20 +64,20 @@ internal fun calculateSourcesToCompile(
lookupSymbols.addAll(classpathChanges.lookupSymbols)
if (lookupSymbols.any()) {
val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, logAction, relativePathOrCanonical)
val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, reporter)
dirtyFiles.addAll(dirtyFilesFromLookups)
}
val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.incrementalCache)) }
if (dirtyClassesFqNames.any()) {
val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassesFqNames, logAction, relativePathOrCanonical)
val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassesFqNames, reporter)
dirtyFiles.addAll(dirtyFilesFromFqNames)
}
if (dirtySourcesSinceLastTimeFile.exists()) {
val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists)
if (files.isNotEmpty()) {
logger.kotlinDebug { "Source files added since last compilation: $files" }
reporter.report { "Source files added since last compilation: ${reporter.pathsAsString(files)}" }
}
dirtyFiles.addAll(files)
@@ -96,21 +89,21 @@ internal fun calculateSourcesToCompile(
private fun getClasspathChanges(
modifiedClasspath: List<File>,
lastBuildInfo: BuildInfo?,
logger: Logger,
artifactDifferenceRegistry: ArtifactDifferenceRegistry?
artifactDifferenceRegistry: ArtifactDifferenceRegistry?,
reporter: IncReporter
): ChangesEither {
if (modifiedClasspath.isEmpty()) {
logger.kotlinDebug { "No classpath changes" }
reporter.report { "No classpath changes" }
return ChangesEither.Known()
}
if (artifactDifferenceRegistry == null) {
logger.kotlinDebug { "No artifact history provider" }
reporter.report { "No artifact history provider" }
return ChangesEither.Unknown()
}
val lastBuildTS = lastBuildInfo?.startTS
if (lastBuildTS == null) {
logger.kotlinDebug { "Could not determine last build timestamp" }
reporter.report { "Could not determine last build timestamp" }
return ChangesEither.Unknown()
}
@@ -119,13 +112,13 @@ private fun getClasspathChanges(
for (file in modifiedClasspath) {
val diffs = artifactDifferenceRegistry[file]
if (diffs == null) {
logger.kotlinDebug { "Could not get changes for file: $file" }
reporter.report { "Could not get changes for file: $file" }
return ChangesEither.Unknown()
}
val (beforeLastBuild, afterLastBuild) = diffs.partition { it.buildTS < lastBuildTS }
if (beforeLastBuild.isEmpty()) {
logger.kotlinDebug { "No known build preceding timestamp $lastBuildTS for file $file" }
reporter.report { "No known build preceding timestamp $lastBuildTS for file $file" }
return ChangesEither.Unknown()
}