Add property for precise version of Java tracking in Gradle IC

The flag name is kotlin.incremental.usePreciseJavaTracking
By default precise version is disabled

 #KT-17621 Fixed

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt

@@ -61,7 +61,8 @@ class IncrementalCompilationOptions(
         /** @See [CompilationResultCategory]] */
         requestedCompilationResults: Array<Int>,
         val resultDifferenceFile: File? = null,
-        val friendDifferenceFile: File? = null
+        val friendDifferenceFile: File? = null,
+        val usePreciseJavaTracking: Boolean
 ) : CompilationOptions(compilerMode, targetPlatform, reportCategories, reportSeverity, requestedCompilationResults) {
     companion object {
         const val serialVersionUID: Long = 0
@@ -77,7 +78,8 @@ class IncrementalCompilationOptions(
                "customCacheVersionFileName='$customCacheVersionFileName', " +
                "customCacheVersion=$customCacheVersion, " +
                "resultDifferenceFile=$resultDifferenceFile, " +
-               "friendDifferenceFile=$friendDifferenceFile" +
+               "friendDifferenceFile=$friendDifferenceFile, " +
+               "usePreciseJavaTracking=$usePreciseJavaTracking" +
                ")"
     }
 }

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -511,7 +511,9 @@ class CompileServiceImpl(
                                                     reporter, annotationFileUpdater,
                                                     artifactChanges, changesRegistry,
                                                     buildHistoryFile = incrementalCompilationOptions.resultDifferenceFile,
-                                                    friendBuildHistoryFile = incrementalCompilationOptions.friendDifferenceFile)
+                                                    friendBuildHistoryFile = incrementalCompilationOptions.friendDifferenceFile,
+                                                    usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking
+        )
         return compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, { changedFiles })
     }
 

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import com.intellij.psi.PsiClass
+import com.intellij.psi.PsiFile
+import com.intellij.psi.PsiJavaFile
+import java.io.File
+import java.util.*
+
+internal class ChangedJavaFilesProcessor(
+        private val reporter: ICReporter,
+        private val psiFileFactory: (File) -> PsiFile?
+) {
+    private val allSymbols = HashSet<LookupSymbol>()
+
+    val allChangedSymbols: Collection<LookupSymbol>
+        get() = allSymbols
+
+    fun process(filesDiff: ChangedFiles.Known): ChangesEither {
+        val modifiedJava = filesDiff.modified.filter(File::isJavaFile)
+        val removedJava = filesDiff.removed.filter(File::isJavaFile)
+
+        if (removedJava.any()) {
+            reporter.report { "Some java files are removed: [${removedJava.joinToString()}]" }
+            return ChangesEither.Unknown()
+        }
+
+        val symbols = HashSet<LookupSymbol>()
+        for (javaFile in modifiedJava) {
+            assert(javaFile.extension.equals("java", ignoreCase = true))
+
+            val psiFile = psiFileFactory(javaFile)
+            if (psiFile !is PsiJavaFile) {
+                reporter.report { "Expected PsiJavaFile, got ${psiFile?.javaClass}" }
+                return ChangesEither.Unknown()
+            }
+
+            psiFile.classes.forEach { it.addLookupSymbols(symbols) }
+        }
+        allSymbols.addAll(symbols)
+        return ChangesEither.Known(lookupSymbols = symbols)
+    }
+
+    private fun PsiClass.addLookupSymbols(symbols: MutableSet<LookupSymbol>) {
+        val fqn = qualifiedName.orEmpty()
+
+        symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!)))
+        methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) }
+        fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) }
+        innerClasses.forEach { it.addLookupSymbols(symbols) }
+    }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -143,6 +143,9 @@ abstract class IncrementalCompilerRunner<
     protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List<GeneratedFile>): Iterable<File> =
             emptyList()
 
+    protected open fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
+            emptyList()
+
     protected open fun makeServices(
             args: Args,
             lookupTracker: LookupTracker,
@@ -245,6 +248,9 @@ abstract class IncrementalCompilerRunner<
         if (exitCode == ExitCode.OK) {
             BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
         }
+        if (exitCode == ExitCode.OK && compilationMode is CompilationMode.Incremental) {
+            buildDirtyLookupSymbols.addAll(additionalDirtyLookupSymbols())
+        }
 
         val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
         processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -66,9 +66,13 @@ fun makeIncrementally(
     val kotlinFiles = sourceFiles.filter { it.extension.toLowerCase() in kotlinExtensions }
 
     withIC {
-        val compiler = IncrementalJvmCompilerRunner(cachesDir,
+        val compiler = IncrementalJvmCompilerRunner(
-                                                    sourceRoots.map { JvmSourceRoot(it, null) }.toSet(),
+                cachesDir,
-                                                    versions, reporter)
+                sourceRoots.map { JvmSourceRoot(it, null) }.toSet(),
+                versions, reporter,
+                // Use precise setting in case of non-Gradle build
+                usePreciseJavaTracking = true
+        )
         compiler.compile(kotlinFiles, args, messageCollector) {
             it.inputsCache.sourceSnapshotMap.compareAndUpdate(sourceFiles)
         }
@@ -101,7 +105,8 @@ class IncrementalJvmCompilerRunner(
         artifactChangesProvider: ArtifactChangesProvider? = null,
         changesRegistry: ChangesRegistry? = null,
         private val buildHistoryFile: File? = null,
-        private val friendBuildHistoryFile: File? = null
+        private val friendBuildHistoryFile: File? = null,
+        private val usePreciseJavaTracking: Boolean
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
         workingDir,
         "caches-jvm",
@@ -129,6 +134,12 @@ class IncrementalJvmCompilerRunner(
 
     private val changedUntrackedJavaClasses = mutableSetOf<ClassId>()
 
+    private var javaFilesProcessor =
+            if (!usePreciseJavaTracking)
+                ChangedJavaFilesProcessor(reporter) { it.psiFile() }
+            else
+                null
+
     override fun calculateSourcesToCompile(caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments): CompilationMode {
         val dirtyFiles = getDirtyFiles(changedFiles)
 
@@ -187,8 +198,18 @@ class IncrementalJvmCompilerRunner(
             return CompilationMode.Rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
         }
 
-        if (!processChangedJava(changedFiles, caches)) {
+        if (!usePreciseJavaTracking) {
-            return CompilationMode.Rebuild { "Could not get changes for java files" }
+            val javaFilesChanges = javaFilesProcessor!!.process(changedFiles)
+            val affectedJavaSymbols = when (javaFilesChanges) {
+                is ChangesEither.Known -> javaFilesChanges.lookupSymbols
+                is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" }
+            }
+            markDirtyBy(affectedJavaSymbols)
+        }
+        else {
+            if (!processChangedJava(changedFiles, caches)) {
+                return CompilationMode.Rebuild { "Could not get changes for java files" }
+            }
         }
 
         if ((changedFiles.modified + changedFiles.removed).any { it.extension.toLowerCase() == "xml" }) {
@@ -350,6 +371,9 @@ class IncrementalJvmCompilerRunner(
         return result
     }
 
+    override fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
+            javaFilesProcessor?.allChangedSymbols ?: emptyList()
+
     override fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
         super.processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)
 
@@ -378,9 +402,11 @@ class IncrementalJvmCompilerRunner(
             val targetToCache = mapOf(targetId to caches.platformCache)
             val incrementalComponents = IncrementalCompilationComponentsImpl(targetToCache)
             register(IncrementalCompilationComponents::class.java, incrementalComponents)
-            val changesTracker = JavaClassesTrackerImpl(caches.platformCache, changedUntrackedJavaClasses.toSet())
+            if (usePreciseJavaTracking) {
-            changedUntrackedJavaClasses.clear()
+                val changesTracker = JavaClassesTrackerImpl(caches.platformCache, changedUntrackedJavaClasses.toSet())
-            register(JavaClassesTracker::class.java, changesTracker)
+                changedUntrackedJavaClasses.clear()
+                register(JavaClassesTracker::class.java, changesTracker)
+            }
         }
 
     override fun runCompiler(

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt

@@ -1,7 +1,7 @@
 package org.jetbrains.kotlin.gradle
 
-import org.gradle.api.logging.LogLevel
 import com.intellij.openapi.util.io.FileUtil
+import org.gradle.api.logging.LogLevel
 import org.jetbrains.kotlin.gradle.util.*
 import org.junit.After
 import org.junit.AfterClass
@@ -146,7 +146,9 @@ abstract class BaseGradleIT {
             val debug: Boolean = false,
             val freeCommandLineArgs: List<String> = emptyList(),
             val kotlinVersion: String = KOTLIN_VERSION,
-            val kotlinDaemonDebugPort: Int? = null)
+            val kotlinDaemonDebugPort: Int? = null,
+            val usePreciseJavaTracking: Boolean? = null
+    )
 
     open inner class Project(
             val projectName: String,
@@ -441,6 +443,7 @@ abstract class BaseGradleIT {
 
                 add("-Pkotlin_version=" + options.kotlinVersion)
                 options.incremental?.let { add("-Pkotlin.incremental=$it") }
+                options.usePreciseJavaTracking?.let { add("-Pkotlin.incremental.usePreciseJavaTracking=$it") }
                 options.androidGradlePluginVersion?.let { add("-Pandroid_tools_version=$it")}
                 if (options.debug) {
                     add("-Dorg.gradle.debug=true")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt

@@ -88,44 +88,6 @@ open class A {
         }
     }
 
-    @Test
-    fun testModifyJavaInLib() {
-        val project = Project("incrementalMultiproject", GRADLE_VERSION)
-        project.build("build") {
-            assertSuccessful()
-        }
-
-        val javaClassJava = project.projectDir.getFileByName("JavaClass.java")
-        javaClassJava.modify { it.replace("String getString", "Object getString") }
-
-        project.build("build") {
-            assertSuccessful()
-            val affectedSources = project.projectDir.getFilesByNames("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
-            val relativePaths = project.relativize(affectedSources)
-            assertCompiledKotlinSources(relativePaths, weakTesting = false)
-        }
-    }
-
-    @Test
-    fun testModifyTrackedJavaClassInLib() {
-        val project = Project("incrementalMultiproject", GRADLE_VERSION)
-        project.build("build") {
-            assertSuccessful()
-        }
-
-        val javaClassJava = project.projectDir.getFileByName("TrackedJavaClass.java")
-        javaClassJava.modify { it.replace("String getString", "Object getString") }
-
-        project.build("build") {
-            assertSuccessful()
-            val affectedSources = project.projectDir.getFilesByNames(
-                    "TrackedJavaClassChild.kt", "useTrackedJavaClass.kt", "useTrackedJavaClassSameModule.kt"
-            )
-            val relativePaths = project.relativize(affectedSources)
-            assertCompiledKotlinSources(relativePaths, weakTesting = false)
-        }
-    }
-
     @Test
     fun testCleanBuildLib() {
         val project = Project("incrementalMultiproject", GRADLE_VERSION)
@@ -232,3 +194,88 @@ open class A {
         }
     }
 }
+
+class IncrementalJavaChangeDefaultIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = null) {
+    @Test
+    override fun testModifySignatureTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeSignature,
+               expectedAffectedSources = listOf(
+                       "TrackedJavaClassChild.kt", "useTrackedJavaClass.kt", "useTrackedJavaClassFooMethodUsage.kt",
+                       "useTrackedJavaClassSameModule.kt"))
+    }
+
+    @Test
+    override fun testModifyBodyTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeBody,
+               expectedAffectedSources = listOf(
+                       "TrackedJavaClassChild.kt", "useTrackedJavaClass.kt", "useTrackedJavaClassFooMethodUsage.kt",
+                       "useTrackedJavaClassSameModule.kt"
+               ))
+    }
+}
+
+class IncrementalJavaChangePreciseIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = true) {
+    @Test
+    override fun testModifySignatureTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeSignature, expectedAffectedSources = listOf("TrackedJavaClassChild.kt", "useTrackedJavaClass.kt"))
+    }
+
+    @Test
+    override fun testModifyBodyTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeBody, expectedAffectedSources = listOf())
+    }
+}
+
+abstract class IncrementalCompilationJavaChangesBase(val usePreciseJavaTracking: Boolean?) : BaseGradleIT() {
+    companion object {
+        protected val GRADLE_VERSION = "2.10"
+    }
+
+    override fun defaultBuildOptions(): BuildOptions =
+            super.defaultBuildOptions().copy(withDaemon = true, incremental = true)
+
+    protected val trackedJavaClass = "TrackedJavaClass.java"
+    private val javaClass = "JavaClass.java"
+    protected val changeBody: (String)->String = { it.replace("Hello, World!", "Hello, World!!!!")  }
+    protected val changeSignature: (String)->String = { it.replace("String getString", "Object getString")  }
+
+    @Test
+    fun testModifySignatureJavaInLib() {
+        doTest(javaClass, changeBody,
+               expectedAffectedSources = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
+        )
+    }
+
+    @Test
+    fun testModifyBodyJavaInLib() {
+        doTest(javaClass, changeBody,
+               expectedAffectedSources = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
+        )
+    }
+
+    abstract fun testModifySignatureTrackedJavaInLib()
+    abstract fun testModifyBodyTrackedJavaInLib()
+
+    protected fun doTest(
+            fileToModify: String,
+            transformFile: (String)->String,
+            expectedAffectedSources: Collection<String>
+    ) {
+        val project = Project("incrementalMultiproject", GRADLE_VERSION)
+
+        val options = defaultBuildOptions().copy(usePreciseJavaTracking = usePreciseJavaTracking)
+        project.build("build", options = options) {
+            assertSuccessful()
+        }
+
+        val javaClassJava = project.projectDir.getFileByName(fileToModify)
+        javaClassJava.modify(transformFile)
+
+        project.build("build", options = options) {
+            assertSuccessful()
+            val affectedSources = project.projectDir.getFilesByNames(*expectedAffectedSources.toTypedArray())
+            val relativePaths = project.relativize(affectedSources)
+            assertCompiledKotlinSources(relativePaths, weakTesting = false)
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/incrementalMultiproject/lib/src/main/java/bar/useTrackedJavaClassSameModule.kt

@@ -1,5 +1,5 @@
 package bar
 
 private fun useTrackedJavaClass() {
-    TrackedJavaClass().getString()
+    TrackedJavaClass().foo()
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt

@@ -38,5 +38,6 @@ internal class GradleIncrementalCompilerEnvironment(
         val artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider? = null,
         val artifactFile: File? = null,
         val buildHistoryFile: File? = null,
-        val friendBuildHistoryFile: File? = null
+        val friendBuildHistoryFile: File? = null,
+        val usePreciseJavaTracking: Boolean = false
 ) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -257,7 +257,8 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                 compilerMode = CompilerMode.INCREMENTAL_COMPILER,
                 targetPlatform = targetPlatform,
                 resultDifferenceFile = environment.buildHistoryFile,
-                friendDifferenceFile = environment.friendBuildHistoryFile
+                friendDifferenceFile = environment.friendBuildHistoryFile,
+                usePreciseJavaTracking = environment.usePreciseJavaTracking
         )
 
         log.info("Options for KOTLIN DAEMON: $compilationOptions")

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt

@@ -22,7 +22,6 @@ import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
 import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import java.util.*
-import kotlin.reflect.KMutableProperty1
 
 fun mapKotlinTaskProperties(project: Project, task: AbstractKotlinCompile<*>) {
     PropertiesProvider(project).apply {
@@ -30,6 +29,9 @@ fun mapKotlinTaskProperties(project: Project, task: AbstractKotlinCompile<*>) {
 
         if (task is KotlinCompile) {
             incrementalJvm?.let { task.incremental = it }
+            usePreciseJavaTracking?.let {
+                task.usePreciseJavaTracking = it
+            }
         }
 
         if (task is Kotlin2JsCompile) {
@@ -62,6 +64,9 @@ internal class PropertiesProvider(private val project: Project) {
     val incrementalMultiplatform: Boolean?
         get() = booleanProperty("kotlin.incremental.multiplatform")
 
+    val usePreciseJavaTracking: Boolean?
+        get() = booleanProperty("kotlin.incremental.usePreciseJavaTracking")
+
     private fun booleanProperty(propName: String): Boolean? =
             property(propName)?.toBoolean()
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -18,7 +18,6 @@ package org.jetbrains.kotlin.gradle.tasks
 
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.logging.Logger
 import org.gradle.api.plugins.BasePluginConvention
 import org.gradle.api.tasks.Input
@@ -250,6 +249,13 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
     internal var artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider? = null
     internal var artifactFile: File? = null
 
+    @get:Input
+    var usePreciseJavaTracking: Boolean = false
+        set(value) {
+            field = value
+            logger.kotlinDebug { "Set $this.usePreciseJavaTracking=$value" }
+        }
+
     init {
         incremental = true
     }
@@ -314,7 +320,9 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
                         artifactDifferenceRegistryProvider,
                         artifactFile = artifactFile,
                         buildHistoryFile = buildHistoryFile,
-                        friendBuildHistoryFile = friendTask?.buildHistoryFile)
+                        friendBuildHistoryFile = friendTask?.buildHistoryFile,
+                        usePreciseJavaTracking = usePreciseJavaTracking
+                )
             }
         }