Track changes in java files generated by kapt2

#KT-13500 fixed

libraries/tools/kotlin-gradle-plugin-core/pom.xml

@@ -62,6 +62,7 @@
 
     <build>
         <sourceDirectory>${project.basedir}/src/main/kotlin</sourceDirectory>
+        <testSourceDirectory>${project.basedir}/src/test/kotlin</testSourceDirectory>
 
         <resources>
             <resource>
@@ -89,6 +90,16 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.19.1</version>
+                <configuration>
+                    <excludes>
+                        <exclude>src/test/resources/**</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/ChangedJavaFilesProcessor.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks
+
+import org.gradle.api.logging.Logging
+import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.com.intellij.lang.java.JavaLanguage
+import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
+import org.jetbrains.kotlin.com.intellij.psi.PsiClass
+import org.jetbrains.kotlin.com.intellij.psi.PsiFile
+import org.jetbrains.kotlin.com.intellij.psi.PsiFileFactory
+import org.jetbrains.kotlin.com.intellij.psi.PsiJavaFile
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.incremental.LookupSymbol
+import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff
+import java.io.File
+import java.util.*
+
+internal class ChangedJavaFilesProcessor {
+    private val log = Logging.getLogger(this.javaClass.simpleName)
+    private val allSymbols = HashSet<LookupSymbol>()
+    private val javaLang = JavaLanguage.INSTANCE
+    private val psiFileFactory: PsiFileFactory by lazy {
+        val rootDisposable = Disposer.newDisposable()
+        val configuration = CompilerConfiguration()
+        val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
+        val project = environment.project
+        PsiFileFactory.getInstance(project)
+    }
+
+    val allChangedSymbols: Collection<LookupSymbol>
+            get() = allSymbols
+
+    fun process(filesDiff: FileCollectionDiff): ChangesEither {
+        if (filesDiff.removed.any()) {
+            log.kotlinDebug { "Some java files are removed: [${filesDiff.removed.joinToString()}]" }
+            return ChangesEither.Unknown()
+        }
+
+        val symbols = HashSet<LookupSymbol>()
+        for (javaFile in filesDiff.newOrModified) {
+            assert(javaFile.extension.equals("java", ignoreCase = true))
+
+            val psiFile = javaFile.psiFile()
+            if (psiFile !is PsiJavaFile) {
+                log.kotlinDebug { "Expected PsiJavaFile, got ${psiFile?.javaClass}" }
+                return ChangesEither.Unknown()
+            }
+
+            psiFile.classes.forEach { it.addLookupSymbols(symbols) }
+        }
+        allSymbols.addAll(symbols)
+        return ChangesEither.Known(lookupSymbols = symbols)
+    }
+
+    private fun PsiClass.addLookupSymbols(symbols: MutableSet<LookupSymbol>) {
+        val fqn = qualifiedName.orEmpty()
+
+        symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!)))
+        methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) }
+        fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) }
+        innerClasses.forEach { it.addLookupSymbols(symbols) }
+    }
+
+    private fun File.psiFile(): PsiFile? =
+            psiFileFactory.createFileFromText(nameWithoutExtension, javaLang, readText())
+}

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt

@@ -21,22 +21,28 @@ import org.jetbrains.kotlin.build.GeneratedJvmClass
 import org.jetbrains.kotlin.incremental.CompilationResult
 import org.jetbrains.kotlin.incremental.IncrementalCacheImpl
 import org.jetbrains.kotlin.incremental.dumpCollection
+import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff
+import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap
+import org.jetbrains.kotlin.incremental.snapshots.SimpleFileSnapshotProviderImpl
 import org.jetbrains.kotlin.incremental.storage.BasicStringMap
 import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
 import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer
 import org.jetbrains.kotlin.modules.TargetId
 import java.io.File
 
-class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl<TargetId>(targetDataRoot, targetOutputDir, target) {
+internal class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl<TargetId>(targetDataRoot, targetOutputDir, target) {
-
     companion object {
         private val SOURCES_TO_CLASSFILES = "sources-to-classfiles"
+        private val FILE_SNAPSHOT = "file-snapshot"
     }
 
-    private val loggerInstance = Logging.getLogger(this.javaClass)
+    private val log = Logging.getLogger(this.javaClass)
-    fun getLogger() = loggerInstance
 
     private val sourceToClassfilesMap = registerMap(SourceToClassfilesMap(SOURCES_TO_CLASSFILES.storageFile))
+    private val fileSnapshotMap = registerMap(FileSnapshotMap(FILE_SNAPSHOT.storageFile))
+
+    fun compareAndUpdateFileSnapshots(files: Iterable<File>): FileCollectionDiff =
+            fileSnapshotMap.compareAndUpdate(files, SimpleFileSnapshotProviderImpl())
 
     fun removeClassfilesBySources(sources: Iterable<File>): Unit =
             sources.forEach { sourceToClassfilesMap.remove(it) }
@@ -60,11 +66,10 @@ class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, t
             // TODO: do it in the code that uses cache, since cache should not generally delete anything outside of it!
             // but for a moment it is an easiest solution to implement
             get(file).forEach {
-                getLogger().debug("[KOTLIN] Deleting $it on clearing cache for $file")
+                log.kotlinDebug { "Deleting $it on clearing cache for $file" }
                 it.delete()
             }
             storage.remove(file.absolutePath)
         }
     }
 }
-

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/IncrementalCachesManager.kt

@@ -0,0 +1,49 @@
+package org.jetbrains.kotlin.gradle.tasks
+
+import org.jetbrains.kotlin.incremental.LookupStorage
+import org.jetbrains.kotlin.modules.TargetId
+import java.io.File
+
+internal class IncrementalCachesManager (
+        private val targetId: TargetId,
+        private val cacheDirectory: File,
+        private val outputDir: File
+) {
+    private val incrementalCacheDir = File(cacheDirectory, "increCache.${targetId.name}")
+    private val lookupCacheDir = File(cacheDirectory, "lookups")
+    private var incrementalCacheOpen = false
+    private var lookupCacheOpen = false
+
+    val incrementalCache: GradleIncrementalCacheImpl by lazy {
+        incrementalCacheOpen = true
+        GradleIncrementalCacheImpl(targetDataRoot = incrementalCacheDir.apply { mkdirs() }, targetOutputDir = outputDir, target = targetId)
+    }
+
+    val lookupCache: LookupStorage by lazy {
+        lookupCacheOpen = true
+        LookupStorage(lookupCacheDir.apply { mkdirs() })
+    }
+
+    fun clean() {
+        close(flush = false)
+        cacheDirectory.deleteRecursively()
+    }
+
+    fun close(flush: Boolean = false) {
+        if (incrementalCacheOpen) {
+            if (flush) {
+                incrementalCache.flush(false)
+            }
+            incrementalCache.close()
+            incrementalCacheOpen = false
+        }
+
+        if (lookupCacheOpen) {
+            if (flush) {
+                lookupCache.flush(false)
+            }
+            lookupCache.close()
+            lookupCacheOpen = false
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -43,6 +43,7 @@ import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.*
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.components.SourceRetentionAnnotationHandler
+import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff
 import org.jetbrains.kotlin.modules.TargetId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
@@ -155,7 +156,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
     private val sourceRoots = HashSet<File>()
 
     // lazy because name is probably not available when constructor is called
-    val taskBuildDirectory: File by lazy { File(File(project.buildDir, KOTLIN_BUILD_DIR_NAME), name) }
+    val taskBuildDirectory: File by lazy { File(File(project.buildDir, KOTLIN_BUILD_DIR_NAME), name).apply { mkdirs() } }
     private val cacheDirectory: File by lazy { File(taskBuildDirectory, CACHES_DIR_NAME) }
     private val dirtySourcesSinceLastTimeFile: File by lazy { File(taskBuildDirectory, DIRTY_SOURCES_FILE_NAME) }
     private val lastBuildInfoFile: File by lazy { File(taskBuildDirectory, LAST_BUILD_INFO_FILE_NAME) }
@@ -177,6 +178,9 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
     private val compileClasspath: Iterable<File>
             get() = (classpath + additionalClasspath)
                     .filterTo(LinkedHashSet()) { it.exists() }
+    private val kapt2GeneratedSourcesDir: File
+            get() = File(project.buildDir, "generated/source/kapt2")
+
 
     val kaptOptions = KaptOptions()
     val pluginOptions = CompilerPluginOptions()
@@ -252,10 +256,9 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
 
         val targetType = "java-production"
         val moduleName = args.moduleName
-        val targets = listOf(TargetId(moduleName, targetType))
+        val targetId = TargetId(moduleName, targetType)
         val outputDir = destinationDir
-        val caches = hashMapOf<TargetId, GradleIncrementalCacheImpl>()
+        val caches = IncrementalCachesManager(targetId, cacheDirectory, outputDir)
-        val lookupStorage = LookupStorage(File(cacheDirectory, "lookups"))
         val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
         var currentRemoved = removed.filter { it.isKotlinFile() }
         val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
@@ -263,43 +266,6 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
         val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
         logger.kotlinDebug { "Last Kotlin Build info for task $path -- $lastBuildInfo" }
 
-        fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl {
-            val cacheDir = File(cacheDirectory, "increCache.${target.name}")
-            cacheDir.mkdirs()
-            return GradleIncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target)
-        }
-
-        fun getIncrementalCache(it: TargetId) = caches.getOrPut(it, { getOrCreateIncrementalCache(it) })
-
-        fun PsiClass.addLookupSymbols(symbols: MutableSet<LookupSymbol>) {
-            val fqn = qualifiedName.orEmpty()
-
-            symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!)))
-            methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) }
-            fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) }
-            innerClasses.forEach { it.addLookupSymbols(symbols) }
-        }
-
-        val dirtyJavaLookupSymbols: Lazy<Collection<LookupSymbol>> = lazy {
-            val symbols = HashSet<LookupSymbol>()
-            val modifiedJavaFiles = modified.filter { it.isJavaFile() }
-            if (modifiedJavaFiles.isEmpty()) return@lazy symbols
-
-            val rootDisposable = Disposer.newDisposable()
-            val configuration = CompilerConfiguration()
-            val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
-            val project = environment.project
-            val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
-
-            modifiedJavaFiles.forEach {
-                val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText())
-                if (javaFile is PsiJavaFile) {
-                    javaFile.classes.forEach { it.addLookupSymbols(symbols) }
-                }
-            }
-            symbols
-        }
-
         fun getClasspathChanges(modifiedClasspath: List<File>): ChangesEither {
             if (modifiedClasspath.isEmpty()) {
                 logger.kotlinDebug { "No classpath changes" }
@@ -340,11 +306,10 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             return ChangesEither.Known(symbols, fqNames)
         }
 
-        fun calculateSourcesToCompile(): Pair<Set<File>, Boolean> {
+        fun calculateSourcesToCompile(javaFilesProcessor: ChangedJavaFilesProcessor): Pair<Set<File>, Boolean> {
             fun rebuild(reason: String): Pair<Set<File>, Boolean> {
                 logger.kotlinInfo("Non-incremental compilation will be performed: $reason")
-                targets.forEach { getIncrementalCache(it).clean() }
+                caches.clean()
-                lookupStorage.clean()
                 dirtySourcesSinceLastTimeFile.delete()
                 return Pair(sources.toSet(), false)
             }
@@ -352,12 +317,11 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             if (!incremental) return rebuild("incremental compilation is not enabled")
             if (!isIncrementalRequested) return rebuild("inputs' changes are unknown (first or clean build)")
 
-            // TODO: store java files in the cache and extract removed symbols from it here
+            val removedClassFiles = removed.filter(File::hasClassFileExtension)
-            val illegalRemovedFiles = removed.filter { it.isJavaFile() || it.hasClassFileExtension() }
+            if (removedClassFiles.any()) return rebuild("Removed class files: ${filesToString(removedClassFiles)}")
-            if (illegalRemovedFiles.any()) return rebuild("unsupported removed files: ${filesToString(illegalRemovedFiles)}")
 
-            val illegalModifiedFiles = modified.filter(File::hasClassFileExtension)
+            val modifiedClassFiles = modified.filter(File::hasClassFileExtension)
-            if (illegalModifiedFiles.any()) return rebuild("unsupported modified files: ${filesToString(illegalModifiedFiles)}")
+            if (modifiedClassFiles.any()) return rebuild("Modified class files: ${filesToString(modifiedClassFiles)}")
 
             val modifiedClasspathEntries = modified.filter { it in classpath }
             val classpathChanges = getClasspathChanges(modifiedClasspathEntries)
@@ -367,21 +331,28 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             if (classpathChanges !is ChangesEither.Known) {
                 throw AssertionError("Unknown implementation of ChangesEither: ${classpathChanges.javaClass}")
             }
+            val javaFilesDiff = FileCollectionDiff(
+                    newOrModified = modified.filter { it.isJavaFile() },
+                    removed = removed.filter { it.isJavaFile() })
+            val javaFilesChanges = javaFilesProcessor.process(javaFilesDiff)
+            val affectedJavaSymbols = when (javaFilesChanges) {
+                is ChangesEither.Known -> javaFilesChanges.lookupSymbols
+                is ChangesEither.Unknown -> return rebuild("Could not get changes for java files")
+            }
 
             val dirtyFiles = modified.filter { it.isKotlinFile() }.toMutableSet()
             val lookupSymbols = HashSet<LookupSymbol>()
-            lookupSymbols.addAll(dirtyJavaLookupSymbols.value)
+            lookupSymbols.addAll(affectedJavaSymbols)
             lookupSymbols.addAll(classpathChanges.lookupSymbols)
 
             if (lookupSymbols.any()) {
-                val dirtyFilesFromLookups = mapLookupSymbolsToFiles(lookupStorage, lookupSymbols, logAction, ::projectRelativePath)
+                val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, logAction, ::projectRelativePath)
                 dirtyFiles.addAll(dirtyFilesFromLookups)
             }
 
-            val allCaches = targets.map(::getIncrementalCache)
+            val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.incrementalCache)) }
-            val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, allCaches) }
             if (dirtyClassesFqNames.any()) {
-                val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(allCaches, dirtyClassesFqNames, logAction, ::projectRelativePath)
+                val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassesFqNames, logAction, ::projectRelativePath)
                 dirtyFiles.addAll(dirtyFilesFromFqNames)
             }
 
@@ -407,9 +378,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
                 cleanupOnError()
             }
 
-            lookupStorage.flush(false)
+            caches.close(flush = true)
-            lookupStorage.close()
-            caches.values.forEach { it.flush(false); it.close() }
             logger.debug("flushed incremental caches")
 
             when (exitCode) {
@@ -432,8 +401,9 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
 
         logger.warn(USING_EXPERIMENTAL_INCREMENTAL_MESSAGE)
         anyClassesCompiled = false
+        val javaFilesProcessor = ChangedJavaFilesProcessor()
         // TODO: decide what to do if no files are considered dirty - rebuild or skip the module
-        var (sourcesToCompile, isIncrementalDecided) = calculateSourcesToCompile()
+        var (sourcesToCompile, isIncrementalDecided) = calculateSourcesToCompile(javaFilesProcessor)
 
         if (isIncrementalDecided) {
             additionalClasspath.add(destinationDir)
@@ -452,12 +422,9 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
         var exitCode = ExitCode.OK
         while (sourcesToCompile.any() || currentRemoved.any()) {
             val removedAndModified = (sourcesToCompile + currentRemoved).toList()
-            val outdatedClasses = targets.flatMap { getIncrementalCache(it).classesBySources(removedAndModified) }
+            val outdatedClasses = caches.incrementalCache.classesBySources(removedAndModified)
-
+            caches.incrementalCache.markOutputClassesDirty(removedAndModified)
-            targets.forEach { getIncrementalCache(it).let {
+            caches.incrementalCache.removeClassfilesBySources(removedAndModified)
-                it.markOutputClassesDirty(removedAndModified)
-                it.removeClassfilesBySources(removedAndModified)
-            }}
 
             // can be empty if only removed sources are present
             if (sourcesToCompile.isNotEmpty()) {
@@ -470,7 +437,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             val text = existingSource.map { it.canonicalPath }.joinToString(separator = System.getProperty("line.separator"))
             dirtySourcesSinceLastTimeFile.writeText(text)
 
-            val compilerOutput = compileChanged(targets, existingSource.toSet(), outputDir, args, ::getIncrementalCache, lookupTracker)
+            val compilerOutput = compileChanged(listOf(targetId), existingSource.toSet(), outputDir, args, { caches.incrementalCache }, lookupTracker)
             exitCode = compilerOutput.exitCode
 
             if (exitCode == ExitCode.OK) {
@@ -483,20 +450,36 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             }
 
             allGeneratedFiles.addAll(compilerOutput.generatedFiles)
-            val compilationResult = updateIncrementalCaches(targets, compilerOutput.generatedFiles,
+            val compilationResult = updateIncrementalCaches(listOf(targetId), compilerOutput.generatedFiles,
                                                             compiledWithErrors = exitCode != ExitCode.OK,
-                                                            getIncrementalCache = { caches[it]!! })
+                                                            getIncrementalCache = { caches.incrementalCache })
 
-            lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved)
+            caches.lookupCache.update(lookupTracker, sourcesToCompile, currentRemoved)
+
+            val generatedJavaFiles = kapt2GeneratedSourcesDir.walk().filter { it.isJavaFile() }.toList()
+            val generatedJavaFilesDiff = caches.incrementalCache.compareAndUpdateFileSnapshots(generatedJavaFiles)
 
             if (!isIncrementalDecided) {
                 artifactFile?.let { artifactDifferenceRegistry?.remove(it) }
                 break
             }
 
-            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(caches.values, logAction)
+            val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.incrementalCache), logAction)
-            sourcesToCompile = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile) +
+            val generatedJavaFilesChanges = javaFilesProcessor.process(generatedJavaFilesDiff)
-                               mapClassesFqNamesToFiles(caches.values, dirtyClassFqNames, logAction, ::projectRelativePath, excludes = sourcesToCompile)
+            val dirtyKotlinFilesFromJava = when (generatedJavaFilesChanges) {
+                is ChangesEither.Unknown -> {
+                    logger.kotlinDebug { "Could not get changes for generated java files, recompiling all kotlin" }
+                    isIncrementalDecided = false
+                    sources.toSet()
+                }
+                is ChangesEither.Known -> {
+                    mapLookupSymbolsToFiles(caches.lookupCache, generatedJavaFilesChanges.lookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile)
+                }
+                else -> throw IllegalStateException("Unknown ChangesEither implementation: $generatedJavaFiles")
+            }
+            sourcesToCompile = dirtyKotlinFilesFromJava +
+                               mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile) +
+                               mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassFqNames, logAction, ::projectRelativePath, excludes = sourcesToCompile)
 
             buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
             buildDirtyFqNames.addAll(dirtyClassFqNames)
@@ -507,8 +490,8 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             }
         }
 
-        if (exitCode == ExitCode.OK) {
+        if (exitCode == ExitCode.OK && isIncrementalDecided) {
-            buildDirtyLookupSymbols.addAll(dirtyJavaLookupSymbols.value)
+            buildDirtyLookupSymbols.addAll(javaFilesProcessor.allChangedSymbols)
         }
         if (artifactFile != null && artifactDifferenceRegistry != null) {
             val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
@@ -634,7 +617,6 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
             extension.equals(JavaFileType.INSTANCE.defaultExtension, ignoreCase = true)
     
     private fun File.isKapt2GeneratedDirectory(): Boolean {
-        val kapt2GeneratedSourcesDir = File(project.buildDir, "generated/source/kapt2")
         if (!kapt2GeneratedSourcesDir.isDirectory) return false
         return FileUtil.isAncestor(kapt2GeneratedSourcesDir, this, /* strict = */ false)
     }

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/gradleCacheVersion.kt

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.incremental.CacheVersion
 import java.io.File
 
-const val GRADLE_CACHE_VERSION = 0
+const val GRADLE_CACHE_VERSION = 1
 const val GRADLE_CACHE_VERSION_FILE_NAME = "gradle-format-version.txt"
 
 fun gradleCacheVersion(dataRoot: File): CacheVersion =

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.snapshots
+
+import java.io.File
+import java.util.*
+
+internal class FileSnapshot(
+        val file: File,
+        val length: Long,
+        val hash: ByteArray
+) {
+    init {
+        assert(!file.isDirectory)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other?.javaClass != javaClass) return false
+
+        other as FileSnapshot
+
+        if (file != other.file) return false
+        if (length != other.length) return false
+        if (!Arrays.equals(hash, other.hash)) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = file.hashCode()
+        result = 31 * result + length.hashCode()
+        result = 31 * result + Arrays.hashCode(hash)
+        return result
+    }
+
+    override fun toString(): String {
+        return "FileSnapshot(file=$file, length=$length, hash=${Arrays.toString(hash)})"
+    }
+}

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.snapshots
+
+import org.jetbrains.kotlin.com.intellij.util.io.DataExternalizer
+import java.io.DataInput
+import java.io.DataOutput
+import java.io.File
+
+internal object FileSnapshotExternalizer : DataExternalizer<FileSnapshot> {
+    override fun save(out: DataOutput, value: FileSnapshot) {
+        out.writeUTF(value.file.canonicalPath)
+        out.writeLong(value.length)
+        out.writeInt(value.hash.size)
+        out.write(value.hash)
+    }
+
+    override fun read(input: DataInput): FileSnapshot {
+        val file = File(input.readUTF())
+        val length = input.readLong()
+        val hashSize = input.readInt()
+        val hash = ByteArray(hashSize)
+        input.readFully(hash)
+        return FileSnapshot(file, length, hash)
+    }
+}

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.snapshots
+
+import org.jetbrains.kotlin.incremental.storage.BasicStringMap
+import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
+import java.io.File
+import java.util.*
+
+internal class FileSnapshotMap(storageFile: File) : BasicStringMap<FileSnapshot>(storageFile, PathStringDescriptor, FileSnapshotExternalizer) {
+    override fun dumpValue(value: FileSnapshot): String =
+            value.toString()
+
+    fun compareAndUpdate(newFiles: Iterable<File>, snapshotProvider: FileSnapshotProvider): FileCollectionDiff {
+        val newOrModified = ArrayList<File>()
+        val removed = ArrayList<File>()
+
+        val newPaths = newFiles.mapTo(HashSet()) { it.canonicalPath }
+        for (oldPath in storage.keys) {
+            if (oldPath !in newPaths) {
+                storage.remove(oldPath)
+                removed.add(File(oldPath))
+            }
+        }
+
+        for (path in newPaths) {
+            val file = File(path)
+            val oldSnapshot = storage[path]
+            val newSnapshot = snapshotProvider[file]
+
+            if (oldSnapshot == null || oldSnapshot != newSnapshot) {
+                newOrModified.add(file)
+                storage[path] = newSnapshot
+            }
+        }
+
+        return FileCollectionDiff(newOrModified, removed)
+    }
+}
+
+internal class FileCollectionDiff(val newOrModified: Iterable<File>, val removed: Iterable<File>)

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.snapshots
+
+import java.io.File
+
+internal interface FileSnapshotProvider {
+    operator fun get(file: File): FileSnapshot
+}
+
+internal class SimpleFileSnapshotProviderImpl : FileSnapshotProvider {
+    override fun get(file: File): FileSnapshot {
+        val length = file.length()
+        val hash = file.md5
+        return FileSnapshot(file, length, hash)
+    }
+}

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.snapshots
+
+import java.io.File
+import java.security.MessageDigest
+
+val File.md5: ByteArray
+    get() {
+        val messageDigest = MessageDigest.getInstance("MD5")
+        val buffer = ByteArray(4048)
+        inputStream().use { input ->
+            while (true) {
+                val len = input.read(buffer)
+                if (len < 0) {
+                    break
+                }
+                messageDigest.update(buffer, 0, len)
+            }
+        }
+        return messageDigest.digest()
+    }

libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/TestWithWorkingDir.kt

@@ -0,0 +1,22 @@
+package org.jetbrains.kotlin
+
+import org.jetbrains.kotlin.com.intellij.openapi.util.io.FileUtil
+import org.junit.After
+import org.junit.Before
+import java.io.File
+import kotlin.properties.Delegates
+
+abstract class TestWithWorkingDir {
+    protected var workingDir: File by Delegates.notNull()
+        private set
+
+    @Before
+    open fun setUp() {
+        workingDir = FileUtil.createTempDirectory(this.javaClass.simpleName, null)
+    }
+
+    @After
+    open fun tearDown() {
+        workingDir.deleteRecursively()
+    }
+}

libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt

@@ -0,0 +1,67 @@
+package org.jetbrains.kotlin.incremental.snapshots
+
+import org.jetbrains.kotlin.TestWithWorkingDir
+import org.junit.After
+import org.junit.Assert.*
+import org.junit.Before
+import org.junit.Test
+import java.io.File
+import kotlin.properties.Delegates
+
+class FileSnapshotMapTest : TestWithWorkingDir() {
+    private var snapshotMap: FileSnapshotMap by Delegates.notNull()
+
+    @Before
+    override fun setUp() {
+        super.setUp()
+        val caches = File(workingDir, "caches").apply { mkdirs() }
+        val snapshotMapFile = File(caches, "snapshots.tab")
+        snapshotMap = FileSnapshotMap(snapshotMapFile)
+    }
+
+    @After
+    override fun tearDown() {
+        snapshotMap.flush(false)
+        snapshotMap.close()
+        super.tearDown()
+    }
+
+    @Test
+    fun testSnapshotMap() {
+        val src = File(workingDir, "src").apply { mkdirs() }
+        val foo = File(src, "foo").apply { mkdirs() }
+
+        val removedTxt = File(foo, "removed.txt").apply { writeText("removed") }
+        val unchangedTxt = File(foo, "unchanged.txt").apply { writeText("unchanged") }
+        val changedTxt = File(foo, "changed.txt").apply { writeText("changed") }
+
+        val snapshotProvider = SimpleFileSnapshotProviderImpl()
+        val diff1 = snapshotMap.compareAndUpdate(src.filesWithExt("txt"), snapshotProvider)
+
+        assertArrayEquals("diff1.removed",
+                diff1.removed.toSortedPaths(),
+                emptyArray<String>())
+        assertArrayEquals("diff1.newOrModified",
+                diff1.newOrModified.toSortedPaths(),
+                listOf(removedTxt, unchangedTxt, changedTxt).toSortedPaths())
+
+        removedTxt.delete()
+        unchangedTxt.writeText("unchanged")
+        changedTxt.writeText("degnahc")
+        val newTxt = File(foo, "new.txt").apply { writeText("new") }
+
+        val diff2 = snapshotMap.compareAndUpdate(src.filesWithExt("txt"), snapshotProvider)
+        assertArrayEquals("diff2.removed",
+                diff2.removed.toSortedPaths(),
+                listOf(removedTxt).toSortedPaths())
+        assertArrayEquals("diff2.newOrModified",
+                diff2.newOrModified.toSortedPaths(),
+                listOf(newTxt, changedTxt).toSortedPaths())
+    }
+
+    private fun Iterable<File>.toSortedPaths(): Array<String> =
+            map { it.canonicalPath }.sorted().toTypedArray()
+
+    private fun File.filesWithExt(ext: String): Iterable<File> =
+            walk().filter { it.isFile && it.extension.equals(ext, ignoreCase = true) }.toList()
+}

libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotTest.kt

@@ -0,0 +1,76 @@
+package org.jetbrains.kotlin.incremental.snapshots
+
+import org.jetbrains.kotlin.TestWithWorkingDir
+import org.junit.Assert.*
+import org.junit.Test
+import java.io.*
+
+class FileSnapshotTest : TestWithWorkingDir() {
+    private val fileSnapshotProvider: FileSnapshotProvider
+            get() = SimpleFileSnapshotProviderImpl()
+
+    @Test
+    fun testExternalizer() {
+        val file = File(workingDir, "1.txt")
+        file.writeText("test")
+        val snapshot = fileSnapshotProvider[file]
+        val deserializedSnapshot = saveAndReadBack(snapshot)
+        assertEquals(snapshot, deserializedSnapshot)
+    }
+
+    @Test
+    fun testEqualityNoChanges() {
+        val file = File(workingDir, "1.txt").apply { writeText("file") }
+        val oldSnapshot = fileSnapshotProvider[file]
+        val newSnapshot = fileSnapshotProvider[file]
+        assertEquals(oldSnapshot, newSnapshot)
+    }
+
+    @Test
+    fun testEqualityDifferentFile() {
+        val file1 = File(workingDir, "1.txt").apply { writeText("file1") }
+        val file2 = File(workingDir, "2.txt").apply {
+            writeText(file1.readText())
+            setLastModified(file1.lastModified())
+        }
+        val oldSnapshot = fileSnapshotProvider[file1]
+        val newSnapshot = fileSnapshotProvider[file2]
+        assertNotEquals(oldSnapshot, newSnapshot)
+    }
+
+    @Test
+    fun testEqualityDifferentTimestamp() {
+        val text = "file"
+        val file = File(workingDir, "1.txt").apply { writeText(text) }
+        val oldSnapshot = fileSnapshotProvider[file]
+        Thread.sleep(1000)
+        file.writeText(text)
+        val newSnapshot = fileSnapshotProvider[file]
+        assertEquals(oldSnapshot, newSnapshot)
+    }
+
+    @Test
+    fun testEqualityDifferentSize() {
+        val file = File(workingDir, "1.txt").apply { writeText("file") }
+        val oldSnapshot = fileSnapshotProvider[file]
+        file.writeText("file modified")
+        val newSnapshot = fileSnapshotProvider[file]
+        assertNotEquals(oldSnapshot, newSnapshot)
+    }
+
+    @Test
+    fun testEqualityDifferentHash() {
+        val file = File(workingDir, "1.txt").apply { writeText("file") }
+        val oldSnapshot = fileSnapshotProvider[file]
+        file.writeText("main")
+        val newSnapshot = fileSnapshotProvider[file]
+        assertNotEquals(oldSnapshot, newSnapshot)
+    }
+
+    private fun saveAndReadBack(snapshot: FileSnapshot): FileSnapshot {
+        val byteOut = ByteArrayOutputStream()
+        DataOutputStream(byteOut).use { FileSnapshotExternalizer.save(it, snapshot) }
+        val byteIn = ByteArrayInputStream(byteOut.toByteArray())
+        return DataInputStream(byteIn).use { FileSnapshotExternalizer.read(it) }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt2IT.kt

@@ -217,4 +217,29 @@ class Kapt2IT: BaseGradleIT() {
             assertFileExists("build/classes/main/example/RuntimeAnnotatedTestClassGenerated.class")
         }
     }
+
+    @Test
+    fun testRemoveAnnotationIC() {
+        val project = Project("simple", GRADLE_2_14_VERSION, directoryPrefix = "kapt2")
+        val options = defaultBuildOptions().copy(incremental = true)
+        project.setupWorkingDir()
+        val internalDummyKt = project.projectDir.getFileByName("InternalDummy.kt")
+
+        // add annotation
+        val exampleAnn = "@example.ExampleAnnotation "
+        internalDummyKt.modify { it.addBeforeSubstring(exampleAnn, "internal class InternalDummy")}
+
+        project.build("classes", options = options) {
+            assertSuccessful()
+        }
+
+        // remove annotation
+        internalDummyKt.modify { it.replace(exampleAnn, "")}
+
+        project.build("classes", options = options) {
+            assertSuccessful()
+            val allMainKotlinSrc = File(project.projectDir, "src/main").allKotlinFiles()
+            assertCompiledKotlinSources(project.relativize(allMainKotlinSrc))
+        }
+    }
 }

libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/stringUtil.kt

@@ -0,0 +1,4 @@
+package org.jetbrains.kotlin.gradle.util
+
+fun String.addBeforeSubstring(prefix: String, substring: String): String =
+        replace(substring, prefix + substring)