Refactoring: introduce IncReporter to report IC progress

build-common/src/org/jetbrains/kotlin/incremental/IncReporter.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import java.io.File
+
+abstract class IncReporter {
+    abstract fun report(message: ()->String)
+    abstract fun pathsAsString(files: Iterable<File>): String
+
+    fun pathsAsString(vararg files: File): String =
+            pathsAsString(files.toList())
+}

build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt

@@ -187,13 +187,13 @@ data class DirtyData(
 
 fun <Target> CompilationResult.getDirtyData(
         caches: Iterable<IncrementalCacheImpl<Target>>,
-        log: (String)->Unit
+        reporter: IncReporter
 ): DirtyData {
     val dirtyLookupSymbols = HashSet<LookupSymbol>()
     val dirtyClassesFqNames = HashSet<FqName>()
 
     for (change in changes) {
-        log("Process $change")
+        reporter.report { "Process $change" }
 
         if (change is ChangeInfo.SignatureChanged) {
             val fqNames = if (!change.areSubclassesAffected) listOf(change.fqName) else withSubtypes(change.fqName, caches)
@@ -225,15 +225,14 @@ fun <Target> CompilationResult.getDirtyData(
 fun mapLookupSymbolsToFiles(
         lookupStorage: LookupStorage,
         lookupSymbols: Iterable<LookupSymbol>,
-        log: (String)->Unit,
+        reporter: IncReporter,
-        getLogFilePath: (File)->String = { it.canonicalPath },
         excludes: Set<File> = emptySet()
 ): Set<File> {
     val dirtyFiles = HashSet<File>()
 
     for (lookup in lookupSymbols) {
         val affectedFiles = lookupStorage.get(lookup).map(::File).filter { it !in excludes }
-        log("${lookup.scope}#${lookup.name} caused recompilation of: ${affectedFiles.map(getLogFilePath)}")
+        reporter.report { "${lookup.scope}#${lookup.name} caused recompilation of: ${reporter.pathsAsString(affectedFiles)}" }
         dirtyFiles.addAll(affectedFiles)
     }
 
@@ -243,8 +242,7 @@ fun mapLookupSymbolsToFiles(
 fun <Target> mapClassesFqNamesToFiles(
         caches: Iterable<IncrementalCacheImpl<Target>>,
         classesFqNames: Iterable<FqName>,
-        log: (String)->Unit,
+        reporter: IncReporter,
-        getLogFilePath: (File)->String = { it.canonicalPath },
         excludes: Set<File> = emptySet()
 ): Set<File> {
     val dirtyFiles = HashSet<File>()
@@ -254,7 +252,7 @@ fun <Target> mapClassesFqNamesToFiles(
             val srcFile = cache.getSourceFileIfClass(dirtyClassFqName)
             if (srcFile == null || srcFile in excludes) continue
 
-            log("Class $dirtyClassFqName caused recompilation of: ${getLogFilePath(srcFile)}")
+            reporter.report { ("Class $dirtyClassFqName caused recompilation of: ${reporter.pathsAsString(srcFile)}") }
             dirtyFiles.add(srcFile)
         }
     }

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -788,6 +788,17 @@ private fun CompilationResult.doProcessChanges(
     }
 }
 
+private class JpsIncReporter : IncReporter() {
+    override fun report(message: ()->String) {
+        if (KotlinBuilder.LOG.isDebugEnabled) {
+            KotlinBuilder.LOG.debug(message())
+        }
+    }
+
+    override fun pathsAsString(files: Iterable<File>): String =
+            files.map { it.canonicalPath }.joinToString()
+}
+
 private fun CompilationResult.doProcessChangesUsingLookups(
         compiledFiles: Set<File>,
         dataManager: BuildDataManager,
@@ -796,16 +807,16 @@ private fun CompilationResult.doProcessChangesUsingLookups(
 ) {
     val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider)
     val allCaches = caches.flatMap { it.thisWithDependentCaches }
-    val logAction = { logStr: String -> KotlinBuilder.LOG.debug(logStr) }
+    val reporter = JpsIncReporter()
 
-    logAction("Start processing changes")
+    reporter.report { "Start processing changes" }
 
-    val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(allCaches, logAction)
+    val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(allCaches, reporter)
-    val dirtyFiles = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction) +
+    val dirtyFiles = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, reporter) +
-                     mapClassesFqNamesToFiles(allCaches, dirtyClassFqNames, logAction)
+                     mapClassesFqNamesToFiles(allCaches, dirtyClassFqNames, reporter)
     fsOperations.markFiles(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
 
-    logAction("End of processing changes")
+    reporter.report { "End of processing changes" }
 }
 
 private fun getLookupTracker(project: JpsProject): LookupTracker {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncReporter.kt

@@ -0,0 +1,17 @@
+package org.jetbrains.kotlin.gradle.tasks
+
+import org.gradle.api.logging.Logging
+import org.jetbrains.kotlin.incremental.IncReporter
+import java.io.File
+
+internal class GradleIncReporter(private val projectRootFile: File) : IncReporter() {
+    private val log = Logging.getLogger(GradleIncReporter::class.java)
+
+    override fun report(message: ()->String) {
+        log.kotlinDebug(message)
+    }
+
+    override fun pathsAsString(files: Iterable<File>): String =
+            files.pathsAsStringRelativeTo(projectRootFile)
+}
+

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -78,7 +78,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
     @TaskAction
     fun execute(inputs: IncrementalTaskInputs): Unit {
         val allKotlinSources = getKotlinSources()
-        logger.kotlinDebug { "all kotlin sources: ${filesToString(allKotlinSources)}" }
+        logger.kotlinDebug { "all kotlin sources: ${allKotlinSources.pathsAsStringRelativeTo(project.rootProject.projectDir)}" }
 
         if (allKotlinSources.isEmpty()) {
             logger.kotlinDebug { "No Kotlin files found, skipping Kotlin compiler task" }
@@ -93,13 +93,6 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCo
     private fun getKotlinSources(): List<File> = (getSource() as Iterable<File>).filter(File::isKotlinFile)
 
     internal abstract fun callCompiler(args: T, allKotlinSources: List<File>, changedFiles: ChangedFiles)
-
-    internal fun relativePathOrCanonical(file: File): String =
-            file.relativeToOrNull(project.rootProject.projectDir)?.path
-                    ?: file.canonicalPath
-
-    internal fun filesToString(files: Iterable<File>) =
-            "[" + files.map { relativePathOrCanonical(it) }.sorted().joinToString(separator = ", \n") + "]"
 }
 
 open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), KotlinJvmCompile {
@@ -175,8 +168,6 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
 
     override fun callCompiler(args: K2JVMCompilerArguments, allKotlinSources: List<File>, changedFiles: ChangedFiles) {
         val outputDir = destinationDir
-        val logAction = { logStr: String -> logger.kotlinInfo(logStr) }
-        val relativePathOrCanonical = { file: File -> relativePathOrCanonical(file) }
 
         if (!incremental) {
             anyClassesCompiled = true
@@ -190,33 +181,31 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
         val targetId = TargetId(name = args.moduleName, type = "java-production")
         val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
         val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
-        logger.kotlinDebug { "Last Kotlin Build info for task $path -- $lastBuildInfo" }
         val caches = IncrementalCachesManager(targetId, cacheDirectory, outputDir)
+        val reporter = GradleIncReporter(project.rootProject.projectDir)
+        reporter.report { "Last Kotlin Build info for task $path -- $lastBuildInfo" }
 
         val compilationMode = calculateSourcesToCompile(javaFilesProcessor,
                 caches,
                 lastBuildInfo,
                 changedFiles,
-                logger,
                 classpath.toList(),
                 dirtySourcesSinceLastTimeFile,
-                { filesToString(it) },
+                artifactDifferenceRegistry,
-                logAction, { relativePathOrCanonical(it) },
+                reporter)
-                artifactDifferenceRegistry)
+        compileIncrementally(args, caches, javaFilesProcessor, lookupTracker, outputDir, allKotlinSources, targetId, compilationMode, reporter)
-        compileIncrementally(args, caches, javaFilesProcessor, logAction, lookupTracker, outputDir, relativePathOrCanonical, allKotlinSources, targetId, compilationMode)
     }
 
     private fun compileIncrementally(
             args: K2JVMCompilerArguments,
             caches: IncrementalCachesManager,
             javaFilesProcessor: ChangedJavaFilesProcessor,
-            logAction: (String) -> Unit,
             lookupTracker: LookupTrackerImpl,
             outputDir: File,
-            relativePathOrCanonical: (File) -> String,
             allKotlinSources: List<File>,
             targetId: TargetId,
-            compilationMode: CompilationMode
+            compilationMode: CompilationMode,
+            reporter: IncReporter
     ) {
         val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
         val dirtySources: MutableList<File>
@@ -237,7 +226,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
         @Suppress("NAME_SHADOWING")
         var compilationMode = compilationMode
 
-        logger.kotlinDebug { "Artifact to register difference for task $path: $artifactFile" }
+        reporter.report { "Artifact to register difference for task $path: $artifactFile" }
         val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
         BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
         val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
@@ -251,7 +240,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
 
             val (sourcesToCompile, removedKotlinSources) = dirtySources.partition { it.isFile }
             if (sourcesToCompile.isNotEmpty()) {
-                logger.kotlinDebug { "compile iteration: ${sourcesToCompile.joinToString(transform = relativePathOrCanonical)}" }
+                reporter.report { "compile iteration: ${reporter.report { reporter.pathsAsString(sourcesToCompile) }}" }
             }
 
             val text = sourcesToCompile.map { it.canonicalPath }.joinToString(separator = System.getProperty("line.separator"))
@@ -284,17 +273,17 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
                 break
             }
 
-            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.incrementalCache), logAction)
+            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.incrementalCache), reporter)
             val generatedJavaFilesChanges = javaFilesProcessor.process(generatedJavaFilesDiff)
             val compiledInThisIterationSet = sourcesToCompile.toHashSet()
             val dirtyKotlinFilesFromJava = when (generatedJavaFilesChanges) {
                 is ChangesEither.Unknown -> {
-                    logger.kotlinDebug { "Could not get changes for generated java files, recompiling all kotlin" }
+                    reporter.report { "Could not get changes for generated java files, recompiling all kotlin" }
                     compilationMode = CompilationMode.Rebuild()
                     allKotlinSources.toSet()
                 }
                 is ChangesEither.Known -> {
-                    mapLookupSymbolsToFiles(caches.lookupCache, generatedJavaFilesChanges.lookupSymbols, logAction, relativePathOrCanonical, excludes = compiledInThisIterationSet)
+                    mapLookupSymbolsToFiles(caches.lookupCache, generatedJavaFilesChanges.lookupSymbols, reporter, excludes = compiledInThisIterationSet)
                 }
                 else -> throw IllegalStateException("Unknown ChangesEither implementation: $generatedJavaFiles")
             }
@@ -302,8 +291,8 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
             with (dirtySources) {
                 clear()
                 addAll(dirtyKotlinFilesFromJava)
-                addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, logAction, relativePathOrCanonical, excludes = compiledInThisIterationSet))
+                addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = compiledInThisIterationSet))
-                addAll(mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassFqNames, logAction, relativePathOrCanonical, excludes = compiledInThisIterationSet))
+                addAll(mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassFqNames, reporter, excludes = compiledInThisIterationSet))
             }
 
             buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
@@ -319,7 +308,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
             val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
             val artifactDifference = ArtifactDifference(currentBuildInfo.startTS, dirtyData)
             artifactDifferenceRegistry!!.add(artifactFile!!, artifactDifference)
-            logger.kotlinDebug {
+            reporter.report {
                 val dirtySymbolsSorted = buildDirtyLookupSymbols.map { it.scope + "#" + it.name }.sorted()
                 "Added artifact difference for $artifactFile (ts: ${currentBuildInfo.startTS}): " +
                         "[\n\t${dirtySymbolsSorted.joinToString(",\n\t")}]"
@@ -327,7 +316,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
         }
 
         caches.close(flush = true)
-        logger.kotlinDebug { "flushed incremental caches" }
+        reporter.report { "flushed incremental caches" }
         processCompilerExitCode(exitCode)
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/fileUtils.kt

@@ -17,3 +17,8 @@ internal fun File.isClassFile(): Boolean =
 internal fun listClassFiles(path: String): Sequence<File> =
         File(path).walk().filter { it.isFile && it.isClassFile() }
 
+internal fun File.relativeOrCanonical(base: File): String =
+        relativeToOrNull(base)?.path ?: canonicalPath
+
+internal fun Iterable<File>.pathsAsStringRelativeTo(base: File): String =
+        map { it.relativeOrCanonical(base) }.sorted().joinToString()

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/incrementalCompilation.kt

@@ -1,11 +1,7 @@
 package org.jetbrains.kotlin.gradle.tasks
 
-import org.gradle.api.logging.Logger
+import org.jetbrains.kotlin.incremental.*
-import org.jetbrains.kotlin.incremental.LookupSymbol
-import org.jetbrains.kotlin.incremental.mapClassesFqNamesToFiles
-import org.jetbrains.kotlin.incremental.mapLookupSymbolsToFiles
 import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff
-import org.jetbrains.kotlin.incremental.withSubtypes
 import org.jetbrains.kotlin.name.FqName
 import java.io.File
 import java.util.*
@@ -20,34 +16,31 @@ internal fun calculateSourcesToCompile(
         caches: IncrementalCachesManager,
         lastBuildInfo: BuildInfo?,
         changedFiles: ChangedFiles,
-        logger: Logger,
         classpath: Iterable<File>,
         dirtySourcesSinceLastTimeFile: File,
-        filesToString: (Iterable<File>)-> String,
+        artifactDifferenceRegistry: ArtifactDifferenceRegistry?,
-        logAction: (String)->Unit,
+        reporter: IncReporter
-        relativePathOrCanonical: (File)->String,
-        artifactDifferenceRegistry: ArtifactDifferenceRegistry?
 ): CompilationMode {
-    fun rebuild(reason: String): CompilationMode {
+    fun rebuild(reason: ()->String): CompilationMode {
-        logger.kotlinInfo("Non-incremental compilation will be performed: $reason")
+        reporter.report { "Non-incremental compilation will be performed: ${reason()}" }
         caches.clean()
         dirtySourcesSinceLastTimeFile.delete()
         return CompilationMode.Rebuild()
     }
 
-    if (changedFiles !is ChangedFiles.Known) return rebuild("inputs' changes are unknown (first or clean build)")
+    if (changedFiles !is ChangedFiles.Known) return rebuild { "inputs' changes are unknown (first or clean build)" }
 
     val removedClassFiles = changedFiles.removed.filter(File::isClassFile)
-    if (removedClassFiles.any()) return rebuild("Removed class files: ${filesToString(removedClassFiles)}")
+    if (removedClassFiles.any()) return rebuild { "Removed class files: ${reporter.pathsAsString(removedClassFiles)}" }
 
     val modifiedClassFiles = changedFiles.modified.filter(File::isClassFile)
-    if (modifiedClassFiles.any()) return rebuild("Modified class files: ${filesToString(modifiedClassFiles)}")
+    if (modifiedClassFiles.any()) return rebuild { "Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}" }
 
     val classpathSet = classpath.toHashSet()
     val modifiedClasspathEntries = changedFiles.modified.filter { it in classpathSet }
-    val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo, logger, artifactDifferenceRegistry)
+    val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo, artifactDifferenceRegistry, reporter)
     if (classpathChanges is ChangesEither.Unknown) {
-        return rebuild("could not get changes from modified classpath entries: ${filesToString(modifiedClasspathEntries)}")
+        return rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
     }
     if (classpathChanges !is ChangesEither.Known) {
         throw AssertionError("Unknown implementation of ChangesEither: ${classpathChanges.javaClass}")
@@ -58,7 +51,7 @@ internal fun calculateSourcesToCompile(
     val javaFilesChanges = javaFilesProcessor.process(javaFilesDiff)
     val affectedJavaSymbols = when (javaFilesChanges) {
         is ChangesEither.Known -> javaFilesChanges.lookupSymbols
-        is ChangesEither.Unknown -> return rebuild("Could not get changes for java files")
+        is ChangesEither.Unknown -> return rebuild { "Could not get changes for java files" }
     }
 
     val dirtyFiles = HashSet<File>(with(changedFiles) { modified.size + removed.size })
@@ -71,20 +64,20 @@ internal fun calculateSourcesToCompile(
     lookupSymbols.addAll(classpathChanges.lookupSymbols)
 
     if (lookupSymbols.any()) {
-        val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, logAction, relativePathOrCanonical)
+        val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, reporter)
         dirtyFiles.addAll(dirtyFilesFromLookups)
     }
 
     val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.incrementalCache)) }
     if (dirtyClassesFqNames.any()) {
-        val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassesFqNames, logAction, relativePathOrCanonical)
+        val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassesFqNames, reporter)
         dirtyFiles.addAll(dirtyFilesFromFqNames)
     }
 
     if (dirtySourcesSinceLastTimeFile.exists()) {
         val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists)
         if (files.isNotEmpty()) {
-            logger.kotlinDebug { "Source files added since last compilation: $files" }
+            reporter.report { "Source files added since last compilation: ${reporter.pathsAsString(files)}" }
         }
 
         dirtyFiles.addAll(files)
@@ -96,21 +89,21 @@ internal fun calculateSourcesToCompile(
 private fun getClasspathChanges(
         modifiedClasspath: List<File>,
         lastBuildInfo: BuildInfo?,
-        logger: Logger,
+        artifactDifferenceRegistry: ArtifactDifferenceRegistry?,
-        artifactDifferenceRegistry: ArtifactDifferenceRegistry?
+        reporter: IncReporter
 ): ChangesEither {
     if (modifiedClasspath.isEmpty()) {
-        logger.kotlinDebug { "No classpath changes" }
+        reporter.report { "No classpath changes" }
         return ChangesEither.Known()
     }
     if (artifactDifferenceRegistry == null) {
-        logger.kotlinDebug { "No artifact history provider" }
+        reporter.report { "No artifact history provider" }
         return ChangesEither.Unknown()
     }
 
     val lastBuildTS = lastBuildInfo?.startTS
     if (lastBuildTS == null) {
-        logger.kotlinDebug { "Could not determine last build timestamp" }
+        reporter.report { "Could not determine last build timestamp" }
         return ChangesEither.Unknown()
     }
 
@@ -119,13 +112,13 @@ private fun getClasspathChanges(
     for (file in modifiedClasspath) {
         val diffs = artifactDifferenceRegistry[file]
         if (diffs == null) {
-            logger.kotlinDebug { "Could not get changes for file: $file" }
+            reporter.report { "Could not get changes for file: $file" }
             return ChangesEither.Unknown()
         }
 
         val (beforeLastBuild, afterLastBuild) = diffs.partition { it.buildTS < lastBuildTS }
         if (beforeLastBuild.isEmpty()) {
-            logger.kotlinDebug { "No known build preceding timestamp $lastBuildTS for file $file" }
+            reporter.report { "No known build preceding timestamp $lastBuildTS for file $file" }
             return ChangesEither.Unknown()
         }