Ensure all output directories are cleared on IC rebuild

In some cases IC needs to perform a rebuild.
Before this change IC was not clearing output directories
besides destination dir for classes, so for example
kapt stubs were not cleared.

Stalled stubs might lead to compile errors.
For example:
1. foo/XGen.java is generated from annotated class foo/X (XGen also
references X).
2. foo/X is moved to bar/X and some other change forces IC to rebuild.
3. kapt generates bar/X stub, but foo/X stub
was not removed because stubs dir is not cleared.
4. kapt runs annotation processors, foo/XGen.java is generated from
foo/X stub, bar/XGen.java is generated from bar/X stub.
5. kotlinc rebuilds. Since destination dir is cleared properly,
only bar/X.class exists.
6. javac tries to compile foo/XGen and fails, because it
compiles against actual Kotlin classes, not stubs.

This commit fixes the issue by passing all output directories
of a task from Gradle to Kotlin IC.

   #KT-21735 fixed

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt

@@ -62,7 +62,11 @@ class IncrementalCompilationOptions(
         requestedCompilationResults: Array<Int>,
         val resultDifferenceFile: File? = null,
         val friendDifferenceFile: File? = null,
-        val usePreciseJavaTracking: Boolean
+        val usePreciseJavaTracking: Boolean,
+        /**
+         * Directories that should be cleared when IC decides to rebuild
+         */
+        val localStateDirs: List<File>
 ) : CompilationOptions(compilerMode, targetPlatform, reportCategories, reportSeverity, requestedCompilationResults) {
     companion object {
         const val serialVersionUID: Long = 0
@@ -80,6 +84,7 @@ class IncrementalCompilationOptions(
                "resultDifferenceFile=$resultDifferenceFile, " +
                "friendDifferenceFile=$friendDifferenceFile, " +
                "usePreciseJavaTracking=$usePreciseJavaTracking" +
+               "localStateDirs=$localStateDirs" +
                ")"
     }
 }

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -522,7 +522,8 @@ class CompileServiceImpl(
                                                     artifactChanges, changesRegistry,
                                                     buildHistoryFile = incrementalCompilationOptions.resultDifferenceFile,
                                                     friendBuildHistoryFile = incrementalCompilationOptions.friendDifferenceFile,
-                                                    usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking
+                                                    usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
+                                                    localStateDirs = incrementalCompilationOptions.localStateDirs
         )
         return compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles)
     }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -43,7 +43,8 @@ abstract class IncrementalCompilerRunner<
         protected val cacheVersions: List<CacheVersion>,
         protected val reporter: ICReporter,
         protected val artifactChangesProvider: ArtifactChangesProvider?,
-        protected val changesRegistry: ChangesRegistry?
+        protected val changesRegistry: ChangesRegistry?,
+        private val localStateDirs: Collection<File> = emptyList()
 ) {
 
     protected val cacheDirectory = File(workingDir, cacheDirName)
@@ -70,7 +71,15 @@ abstract class IncrementalCompilerRunner<
 
             caches.clean()
             dirtySourcesSinceLastTimeFile.delete()
-            destinationDir(args).deleteRecursively()
+
+            reporter.report { "Deleting output directories on rebuild:" }
+            for (dir in sequenceOf(destinationDir(args)) + localStateDirs.asSequence()) {
+                if (!dir.isDirectory) continue
+
+                dir.deleteRecursively()
+                dir.mkdirs()
+                reporter.report { "deleted $dir" }
+            }
 
             caches = createCacheManager(args)
             if (providedChangedFiles == null) {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -71,7 +71,8 @@ fun makeIncrementally(
                 sourceRoots.map { JvmSourceRoot(it, null) }.toSet(),
                 versions, reporter,
                 // Use precise setting in case of non-Gradle build
-                usePreciseJavaTracking = true
+                usePreciseJavaTracking = true,
+                localStateDirs = emptyList()
         )
         compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null)
     }
@@ -104,14 +105,16 @@ class IncrementalJvmCompilerRunner(
         changesRegistry: ChangesRegistry? = null,
         private val buildHistoryFile: File? = null,
         private val friendBuildHistoryFile: File? = null,
-        private val usePreciseJavaTracking: Boolean
+        private val usePreciseJavaTracking: Boolean,
+        localStateDirs: Collection<File>
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
         workingDir,
         "caches-jvm",
         cacheVersions,
         reporter,
         artifactChangesProvider,
-        changesRegistry
+        changesRegistry,
+        localStateDirs = localStateDirs
 ) {
     override fun isICEnabled(): Boolean =
             IncrementalCompilation.isEnabled()

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt3IT.kt

@@ -193,6 +193,54 @@ open class Kapt3IT : Kapt3BaseIT() {
         }
     }
 
+    @Test
+    fun testRemoveJavaClassICRebuild() {
+        testICRebuild { project ->
+            project.projectFile("Foo.java").delete()
+        }
+    }
+
+    @Test
+    fun testChangeClasspathICRebuild() {
+        testICRebuild { project ->
+            project.projectFile("build.gradle").modify {
+                "$it\ndependencies { compile 'org.jetbrains.kotlin:kotlin-reflect:' + kotlin_version }"
+            }
+        }
+    }
+
+    // tests all output directories are cleared when IC rebuilds
+    private fun testICRebuild(performChange: (Project) -> Unit) {
+        val project = Project("incrementalRebuild", directoryPrefix = "kapt2")
+        val options = defaultBuildOptions().copy(incremental = true)
+        val generatedSrc = "build/generated/source/kapt/main"
+
+        project.build("build", options = options) {
+            assertSuccessful()
+
+            // generated sources
+            assertFileExists("$generatedSrc/bar/UseBar_MembersInjector.java")
+        }
+
+        performChange(project)
+        project.projectFile("UseBar.kt").modify { it.replace("package bar", "package foo.bar") }
+
+        project.build("build", options = options) {
+            assertSuccessful()
+            assertTasksExecuted(listOf(":kaptGenerateStubsKotlin", ":kaptKotlin", ":compileKotlin", ":compileJava"))
+
+            // generated sources
+            assertFileExists("$generatedSrc/foo/bar/UseBar_MembersInjector.java")
+            assertNoSuchFile("$generatedSrc/bar/UseBar_MembersInjector.java")
+
+            // classes
+            assertFileExists(kotlinClassesDir() + "foo/bar/UseBar.class")
+            assertNoSuchFile(kotlinClassesDir() + "bar/UseBar.class")
+            assertFileExists(javaClassesDir() + "foo/bar/UseBar_MembersInjector.class")
+            assertNoSuchFile(javaClassesDir() + "bar/UseBar_MembersInjector.class")
+        }
+    }
+
     @Test
     fun testRemoveAnnotationIC() {
         val project = Project("simple", directoryPrefix = "kapt2")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/kapt2/incrementalRebuild/build.gradle

@@ -0,0 +1,24 @@
+buildscript {
+    repositories {
+        mavenLocal()
+        mavenCentral()
+    }
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+apply plugin: "java"
+apply plugin: "kotlin"
+apply plugin: "kotlin-kapt"
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+dependencies {
+    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    compile 'com.google.dagger:dagger:2.14.1'
+    kapt 'com.google.dagger:dagger-compiler:2.14.1'
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/kapt2/incrementalRebuild/src/main/java/bar/UseBar.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package bar
+
+import javax.inject.Inject
+import foo.*
+
+class UseBar {
+    @Inject
+    lateinit var bar: Bar
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/kapt2/incrementalRebuild/src/main/java/foo/Bar.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package foo;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+@Singleton
+public class Bar {
+    @Inject
+    public Bar(){
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/kapt2/incrementalRebuild/src/main/java/foo/Foo.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package foo;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+@Singleton
+public class Foo {
+    @Inject
+    public Foo(){
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt

@@ -39,5 +39,6 @@ internal class GradleIncrementalCompilerEnvironment(
         val artifactFile: File? = null,
         val buildHistoryFile: File? = null,
         val friendBuildHistoryFile: File? = null,
-        val usePreciseJavaTracking: Boolean = false
+        val usePreciseJavaTracking: Boolean = false,
+        val localStateDirs: List<File> = emptyList()
 ) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -269,7 +269,8 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                 targetPlatform = targetPlatform,
                 resultDifferenceFile = environment.buildHistoryFile,
                 friendDifferenceFile = environment.friendBuildHistoryFile,
-                usePreciseJavaTracking = environment.usePreciseJavaTracking
+                usePreciseJavaTracking = environment.usePreciseJavaTracking,
+                localStateDirs = environment.localStateDirs
         )
 
         log.info("Options for KOTLIN DAEMON: $compilationOptions")

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/KaptGenerateStubsTask.kt

@@ -84,16 +84,6 @@ open class KaptGenerateStubsTask : KotlinCompile() {
         args.destinationAsFile = this.destinationDir
     }
 
-    override fun clearOutputsBeforeNonIncrementalBuild() {
-        super.clearOutputsBeforeNonIncrementalBuild()
-
-        if (!stubsDir.deleteRecursively()) {
-            logger.kotlinWarn("Could not delete $stubsDir")
-        }
-
-        stubsDir.mkdirs()
-    }
-
     override fun execute(inputs: IncrementalTaskInputs) {
         val sourceRoots = kotlinCompileTask.getSourceRoots()
         val allKotlinSources = sourceRoots.kotlinSourceFiles

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/KaptTask.kt

@@ -97,9 +97,7 @@ open class KaptTask : ConventionTask(), CompilerArgumentAwareWithInput<K2JVMComp
     fun compile() {
         /** Delete everything inside generated sources and classes output directory
          * (annotation processing is not incremental) */
-        destinationDir.clearDirectory()
+        clearOutputDirectories()
-        classesDir.clearDirectory()
-        kotlinSourcesDestinationDir.clearDirectory()
 
         val sourceRootsFromKotlin = kotlinCompileTask.sourceRootsContainer.sourceRoots
         val rawSourceRoots = FilteringSourceRootsContainer(sourceRootsFromKotlin, { !isInsideDestinationDirs(it) })
@@ -124,11 +122,6 @@ open class KaptTask : ConventionTask(), CompilerArgumentAwareWithInput<K2JVMComp
         throwGradleExceptionIfError(exitCode)
     }
 
-    private fun File.clearDirectory() {
-        deleteRecursively()
-        mkdirs()
-    }
-
     private val isAtLeastJava9: Boolean
         get() = compareVersionNumbers(getJavaRuntimeVersion(), "9") >= 0
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -341,13 +341,6 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
     @Internal
     override fun getSourceRoots() = SourceRoots.ForJvm.create(getSource(), sourceRootsContainer)
 
-    protected open fun clearOutputsBeforeNonIncrementalBuild() {
-        logger.kotlinDebug { "Removing all kotlin classes in $destinationDir" }
-
-        destinationDir.deleteRecursively()
-        destinationDir.mkdirs()
-    }
-
     override fun callCompiler(args: K2JVMCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) {
         sourceRoots as SourceRoots.ForJvm
 
@@ -371,13 +364,14 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
                         artifactFile = artifactFile,
                         buildHistoryFile = buildHistoryFile,
                         friendBuildHistoryFile = friendTask?.buildHistoryFile,
-                        usePreciseJavaTracking = usePreciseJavaTracking
+                        usePreciseJavaTracking = usePreciseJavaTracking,
+                        localStateDirs = outputDirectories
                 )
             }
         }
 
         if (!incremental) {
-            clearOutputsBeforeNonIncrementalBuild()
+            clearOutputDirectories(reason = "IC is disabled for the task")
         }
 
         try {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt

@@ -1,7 +1,15 @@
 package org.jetbrains.kotlin.gradle.tasks
 
 import org.gradle.api.GradleException
+import org.gradle.api.Task
+import org.gradle.api.tasks.OutputDirectory
 import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
+import org.jetbrains.kotlin.gradle.utils.outputsCompatible
+import java.io.File
+import kotlin.reflect.KProperty1
+import kotlin.reflect.full.findAnnotation
+import kotlin.reflect.full.memberProperties
 
 fun throwGradleExceptionIfError(exitCode: ExitCode) {
     when (exitCode) {
@@ -12,3 +20,24 @@ fun throwGradleExceptionIfError(exitCode: ExitCode) {
         else -> throw IllegalStateException("Unexpected exit code: $exitCode")
     }
 }
+
+internal val <T : Task> T.outputDirectories: List<File>
+    get() = outputsCompatible.files.files.filter { it.isDirectory }
+
+internal fun <T : Task> T.clearOutputDirectories(reason: String? = null) {
+    logger.kotlinDebug {
+        val suffix = reason?.let { " ($it)" }.orEmpty()
+        "Clearing output directories for task '$path'$suffix:"
+    }
+    val outputDirectories = outputDirectories
+    for (dir in outputDirectories) {
+        when {
+            dir.isDirectory -> {
+                dir.deleteRecursively()
+                dir.mkdirs()
+                logger.kotlinDebug { "  deleted $dir" }
+            }
+            else -> logger.kotlinDebug { "  skipping $dir (not a directory)" }
+        }
+    }
+}