Add property for precise version of Java tracking in Gradle IC

The flag name is kotlin.incremental.usePreciseJavaTracking By default precise version is disabled #KT-17621 Fixed
2017-12-04 15:05:58 +03:00
parent a978c6be35
commit a079b92ea0
12 changed files with 231 additions and 63 deletions
@@ -61,7 +61,8 @@ class IncrementalCompilationOptions(
        /** @See [CompilationResultCategory]] */
        requestedCompilationResults: Array<Int>,
        val resultDifferenceFile: File? = null,
-        val friendDifferenceFile: File? = null
+        val friendDifferenceFile: File? = null,
+        val usePreciseJavaTracking: Boolean
 ) : CompilationOptions(compilerMode, targetPlatform, reportCategories, reportSeverity, requestedCompilationResults) {
    companion object {
        const val serialVersionUID: Long = 0
@@ -77,7 +78,8 @@ class IncrementalCompilationOptions(
               "customCacheVersionFileName='$customCacheVersionFileName', " +
               "customCacheVersion=$customCacheVersion, " +
               "resultDifferenceFile=$resultDifferenceFile, " +
-               "friendDifferenceFile=$friendDifferenceFile" +
+               "friendDifferenceFile=$friendDifferenceFile, " +
+               "usePreciseJavaTracking=$usePreciseJavaTracking" +
               ")"
    }
 }
@@ -86,4 +88,4 @@ enum class CompilerMode : Serializable {
    NON_INCREMENTAL_COMPILER,
    INCREMENTAL_COMPILER,
    JPS_COMPILER
-}
+}
@@ -511,7 +511,9 @@ class CompileServiceImpl(
                                                    reporter, annotationFileUpdater,
                                                    artifactChanges, changesRegistry,
                                                    buildHistoryFile = incrementalCompilationOptions.resultDifferenceFile,
-                                                    friendBuildHistoryFile = incrementalCompilationOptions.friendDifferenceFile)
+                                                    friendBuildHistoryFile = incrementalCompilationOptions.friendDifferenceFile,
+                                                    usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking
+        )
        return compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, { changedFiles })
    }

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import com.intellij.psi.PsiClass
+import com.intellij.psi.PsiFile
+import com.intellij.psi.PsiJavaFile
+import java.io.File
+import java.util.*
+
+internal class ChangedJavaFilesProcessor(
+        private val reporter: ICReporter,
+        private val psiFileFactory: (File) -> PsiFile?
+) {
+    private val allSymbols = HashSet<LookupSymbol>()
+
+    val allChangedSymbols: Collection<LookupSymbol>
+        get() = allSymbols
+
+    fun process(filesDiff: ChangedFiles.Known): ChangesEither {
+        val modifiedJava = filesDiff.modified.filter(File::isJavaFile)
+        val removedJava = filesDiff.removed.filter(File::isJavaFile)
+
+        if (removedJava.any()) {
+            reporter.report { "Some java files are removed: [${removedJava.joinToString()}]" }
+            return ChangesEither.Unknown()
+        }
+
+        val symbols = HashSet<LookupSymbol>()
+        for (javaFile in modifiedJava) {
+            assert(javaFile.extension.equals("java", ignoreCase = true))
+
+            val psiFile = psiFileFactory(javaFile)
+            if (psiFile !is PsiJavaFile) {
+                reporter.report { "Expected PsiJavaFile, got ${psiFile?.javaClass}" }
+                return ChangesEither.Unknown()
+            }
+
+            psiFile.classes.forEach { it.addLookupSymbols(symbols) }
+        }
+        allSymbols.addAll(symbols)
+        return ChangesEither.Known(lookupSymbols = symbols)
+    }
+
+    private fun PsiClass.addLookupSymbols(symbols: MutableSet<LookupSymbol>) {
+        val fqn = qualifiedName.orEmpty()
+
+        symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!)))
+        methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) }
+        fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) }
+        innerClasses.forEach { it.addLookupSymbols(symbols) }
+    }
+}
@@ -143,6 +143,9 @@ abstract class IncrementalCompilerRunner<
    protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List<GeneratedFile>): Iterable<File> =
            emptyList()

+    protected open fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
+            emptyList()
+
    protected open fun makeServices(
            args: Args,
            lookupTracker: LookupTracker,
@@ -245,6 +248,9 @@ abstract class IncrementalCompilerRunner<
        if (exitCode == ExitCode.OK) {
            BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
        }
+        if (exitCode == ExitCode.OK && compilationMode is CompilationMode.Incremental) {
+            buildDirtyLookupSymbols.addAll(additionalDirtyLookupSymbols())
+        }

        val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
        processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)
@@ -66,9 +66,13 @@ fun makeIncrementally(
    val kotlinFiles = sourceFiles.filter { it.extension.toLowerCase() in kotlinExtensions }

    withIC {
-        val compiler = IncrementalJvmCompilerRunner(cachesDir,
-                                                    sourceRoots.map { JvmSourceRoot(it, null) }.toSet(),
-                                                    versions, reporter)
+        val compiler = IncrementalJvmCompilerRunner(
+                cachesDir,
+                sourceRoots.map { JvmSourceRoot(it, null) }.toSet(),
+                versions, reporter,
+                // Use precise setting in case of non-Gradle build
+                usePreciseJavaTracking = true
+        )
        compiler.compile(kotlinFiles, args, messageCollector) {
            it.inputsCache.sourceSnapshotMap.compareAndUpdate(sourceFiles)
        }
@@ -101,7 +105,8 @@ class IncrementalJvmCompilerRunner(
        artifactChangesProvider: ArtifactChangesProvider? = null,
        changesRegistry: ChangesRegistry? = null,
        private val buildHistoryFile: File? = null,
-        private val friendBuildHistoryFile: File? = null
+        private val friendBuildHistoryFile: File? = null,
+        private val usePreciseJavaTracking: Boolean
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
        workingDir,
        "caches-jvm",
@@ -129,6 +134,12 @@ class IncrementalJvmCompilerRunner(

    private val changedUntrackedJavaClasses = mutableSetOf<ClassId>()

+    private var javaFilesProcessor =
+            if (!usePreciseJavaTracking)
+                ChangedJavaFilesProcessor(reporter) { it.psiFile() }
+            else
+                null
+
    override fun calculateSourcesToCompile(caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments): CompilationMode {
        val dirtyFiles = getDirtyFiles(changedFiles)

@@ -187,8 +198,18 @@ class IncrementalJvmCompilerRunner(
            return CompilationMode.Rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
        }

-        if (!processChangedJava(changedFiles, caches)) {
-            return CompilationMode.Rebuild { "Could not get changes for java files" }
+        if (!usePreciseJavaTracking) {
+            val javaFilesChanges = javaFilesProcessor!!.process(changedFiles)
+            val affectedJavaSymbols = when (javaFilesChanges) {
+                is ChangesEither.Known -> javaFilesChanges.lookupSymbols
+                is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" }
+            }
+            markDirtyBy(affectedJavaSymbols)
+        }
+        else {
+            if (!processChangedJava(changedFiles, caches)) {
+                return CompilationMode.Rebuild { "Could not get changes for java files" }
+            }
        }

        if ((changedFiles.modified + changedFiles.removed).any { it.extension.toLowerCase() == "xml" }) {
@@ -350,6 +371,9 @@ class IncrementalJvmCompilerRunner(
        return result
    }

+    override fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
+            javaFilesProcessor?.allChangedSymbols ?: emptyList()
+
    override fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
        super.processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)

@@ -378,9 +402,11 @@ class IncrementalJvmCompilerRunner(
            val targetToCache = mapOf(targetId to caches.platformCache)
            val incrementalComponents = IncrementalCompilationComponentsImpl(targetToCache)
            register(IncrementalCompilationComponents::class.java, incrementalComponents)
-            val changesTracker = JavaClassesTrackerImpl(caches.platformCache, changedUntrackedJavaClasses.toSet())
-            changedUntrackedJavaClasses.clear()
-            register(JavaClassesTracker::class.java, changesTracker)
+            if (usePreciseJavaTracking) {
+                val changesTracker = JavaClassesTrackerImpl(caches.platformCache, changedUntrackedJavaClasses.toSet())
+                changedUntrackedJavaClasses.clear()
+                register(JavaClassesTracker::class.java, changesTracker)
+            }
        }

    override fun runCompiler(
@@ -1,7 +1,7 @@
 package org.jetbrains.kotlin.gradle

-import org.gradle.api.logging.LogLevel
 import com.intellij.openapi.util.io.FileUtil
+import org.gradle.api.logging.LogLevel
 import org.jetbrains.kotlin.gradle.util.*
 import org.junit.After
 import org.junit.AfterClass
@@ -146,7 +146,9 @@ abstract class BaseGradleIT {
            val debug: Boolean = false,
            val freeCommandLineArgs: List<String> = emptyList(),
            val kotlinVersion: String = KOTLIN_VERSION,
-            val kotlinDaemonDebugPort: Int? = null)
+            val kotlinDaemonDebugPort: Int? = null,
+            val usePreciseJavaTracking: Boolean? = null
+    )

    open inner class Project(
            val projectName: String,
@@ -441,6 +443,7 @@ abstract class BaseGradleIT {

                add("-Pkotlin_version=" + options.kotlinVersion)
                options.incremental?.let { add("-Pkotlin.incremental=$it") }
+                options.usePreciseJavaTracking?.let { add("-Pkotlin.incremental.usePreciseJavaTracking=$it") }
                options.androidGradlePluginVersion?.let { add("-Pandroid_tools_version=$it")}
                if (options.debug) {
                    add("-Dorg.gradle.debug=true")
@@ -486,4 +489,4 @@ abstract class BaseGradleIT {
    }

    private fun String.normalizePath() = replace("\\", "/")
-}
+}
@@ -88,44 +88,6 @@ open class A {
        }
    }

-    @Test
-    fun testModifyJavaInLib() {
-        val project = Project("incrementalMultiproject", GRADLE_VERSION)
-        project.build("build") {
-            assertSuccessful()
-        }
-
-        val javaClassJava = project.projectDir.getFileByName("JavaClass.java")
-        javaClassJava.modify { it.replace("String getString", "Object getString") }
-
-        project.build("build") {
-            assertSuccessful()
-            val affectedSources = project.projectDir.getFilesByNames("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
-            val relativePaths = project.relativize(affectedSources)
-            assertCompiledKotlinSources(relativePaths, weakTesting = false)
-        }
-    }
-
-    @Test
-    fun testModifyTrackedJavaClassInLib() {
-        val project = Project("incrementalMultiproject", GRADLE_VERSION)
-        project.build("build") {
-            assertSuccessful()
-        }
-
-        val javaClassJava = project.projectDir.getFileByName("TrackedJavaClass.java")
-        javaClassJava.modify { it.replace("String getString", "Object getString") }
-
-        project.build("build") {
-            assertSuccessful()
-            val affectedSources = project.projectDir.getFilesByNames(
-                    "TrackedJavaClassChild.kt", "useTrackedJavaClass.kt", "useTrackedJavaClassSameModule.kt"
-            )
-            val relativePaths = project.relativize(affectedSources)
-            assertCompiledKotlinSources(relativePaths, weakTesting = false)
-        }
-    }
-
    @Test
    fun testCleanBuildLib() {
        val project = Project("incrementalMultiproject", GRADLE_VERSION)
@@ -231,4 +193,89 @@ open class A {
            assertCompiledKotlinSources(project.relativize(affectedSources), weakTesting = false)
        }
    }
-}
+}
+
+class IncrementalJavaChangeDefaultIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = null) {
+    @Test
+    override fun testModifySignatureTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeSignature,
+               expectedAffectedSources = listOf(
+                       "TrackedJavaClassChild.kt", "useTrackedJavaClass.kt", "useTrackedJavaClassFooMethodUsage.kt",
+                       "useTrackedJavaClassSameModule.kt"))
+    }
+
+    @Test
+    override fun testModifyBodyTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeBody,
+               expectedAffectedSources = listOf(
+                       "TrackedJavaClassChild.kt", "useTrackedJavaClass.kt", "useTrackedJavaClassFooMethodUsage.kt",
+                       "useTrackedJavaClassSameModule.kt"
+               ))
+    }
+}
+
+class IncrementalJavaChangePreciseIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = true) {
+    @Test
+    override fun testModifySignatureTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeSignature, expectedAffectedSources = listOf("TrackedJavaClassChild.kt", "useTrackedJavaClass.kt"))
+    }
+
+    @Test
+    override fun testModifyBodyTrackedJavaInLib() {
+        doTest(trackedJavaClass, changeBody, expectedAffectedSources = listOf())
+    }
+}
+
+abstract class IncrementalCompilationJavaChangesBase(val usePreciseJavaTracking: Boolean?) : BaseGradleIT() {
+    companion object {
+        protected val GRADLE_VERSION = "2.10"
+    }
+
+    override fun defaultBuildOptions(): BuildOptions =
+            super.defaultBuildOptions().copy(withDaemon = true, incremental = true)
+
+    protected val trackedJavaClass = "TrackedJavaClass.java"
+    private val javaClass = "JavaClass.java"
+    protected val changeBody: (String)->String = { it.replace("Hello, World!", "Hello, World!!!!")  }
+    protected val changeSignature: (String)->String = { it.replace("String getString", "Object getString")  }
+
+    @Test
+    fun testModifySignatureJavaInLib() {
+        doTest(javaClass, changeBody,
+               expectedAffectedSources = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
+        )
+    }
+
+    @Test
+    fun testModifyBodyJavaInLib() {
+        doTest(javaClass, changeBody,
+               expectedAffectedSources = listOf("JavaClassChild.kt", "useJavaClass.kt", "useJavaClassFooMethodUsage.kt")
+        )
+    }
+
+    abstract fun testModifySignatureTrackedJavaInLib()
+    abstract fun testModifyBodyTrackedJavaInLib()
+
+    protected fun doTest(
+            fileToModify: String,
+            transformFile: (String)->String,
+            expectedAffectedSources: Collection<String>
+    ) {
+        val project = Project("incrementalMultiproject", GRADLE_VERSION)
+
+        val options = defaultBuildOptions().copy(usePreciseJavaTracking = usePreciseJavaTracking)
+        project.build("build", options = options) {
+            assertSuccessful()
+        }
+
+        val javaClassJava = project.projectDir.getFileByName(fileToModify)
+        javaClassJava.modify(transformFile)
+
+        project.build("build", options = options) {
+            assertSuccessful()
+            val affectedSources = project.projectDir.getFilesByNames(*expectedAffectedSources.toTypedArray())
+            val relativePaths = project.relativize(affectedSources)
+            assertCompiledKotlinSources(relativePaths, weakTesting = false)
+        }
+    }
+}
@@ -1,5 +1,5 @@
 package bar

 private fun useTrackedJavaClass() {
-    TrackedJavaClass().getString()
+    TrackedJavaClass().foo()
 }
@@ -38,5 +38,6 @@ internal class GradleIncrementalCompilerEnvironment(
        val artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider? = null,
        val artifactFile: File? = null,
        val buildHistoryFile: File? = null,
-        val friendBuildHistoryFile: File? = null
-) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)
+        val friendBuildHistoryFile: File? = null,
+        val usePreciseJavaTracking: Boolean = false
+) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)
@@ -257,7 +257,8 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                compilerMode = CompilerMode.INCREMENTAL_COMPILER,
                targetPlatform = targetPlatform,
                resultDifferenceFile = environment.buildHistoryFile,
-                friendDifferenceFile = environment.friendBuildHistoryFile
+                friendDifferenceFile = environment.friendBuildHistoryFile,
+                usePreciseJavaTracking = environment.usePreciseJavaTracking
        )

        log.info("Options for KOTLIN DAEMON: $compilationOptions")
@@ -22,7 +22,6 @@ import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
 import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import java.util.*
-import kotlin.reflect.KMutableProperty1

 fun mapKotlinTaskProperties(project: Project, task: AbstractKotlinCompile<*>) {
    PropertiesProvider(project).apply {
@@ -30,6 +29,9 @@ fun mapKotlinTaskProperties(project: Project, task: AbstractKotlinCompile<*>) {

        if (task is KotlinCompile) {
            incrementalJvm?.let { task.incremental = it }
+            usePreciseJavaTracking?.let {
+                task.usePreciseJavaTracking = it
+            }
        }

        if (task is Kotlin2JsCompile) {
@@ -62,6 +64,9 @@ internal class PropertiesProvider(private val project: Project) {
    val incrementalMultiplatform: Boolean?
        get() = booleanProperty("kotlin.incremental.multiplatform")

+    val usePreciseJavaTracking: Boolean?
+        get() = booleanProperty("kotlin.incremental.usePreciseJavaTracking")
+
    private fun booleanProperty(propName: String): Boolean? =
            property(propName)?.toBoolean()

@@ -72,4 +77,4 @@ internal class PropertiesProvider(private val project: Project) {
            else {
                localProperties.getProperty(propName)
            }
-}
+}
@@ -18,7 +18,6 @@ package org.jetbrains.kotlin.gradle.tasks

 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.logging.Logger
 import org.gradle.api.plugins.BasePluginConvention
 import org.gradle.api.tasks.Input
@@ -250,6 +249,13 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
    internal var artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider? = null
    internal var artifactFile: File? = null

+    @get:Input
+    var usePreciseJavaTracking: Boolean = false
+        set(value) {
+            field = value
+            logger.kotlinDebug { "Set $this.usePreciseJavaTracking=$value" }
+        }
+
    init {
        incremental = true
    }
@@ -314,7 +320,9 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
                        artifactDifferenceRegistryProvider,
                        artifactFile = artifactFile,
                        buildHistoryFile = buildHistoryFile,
-                        friendBuildHistoryFile = friendTask?.buildHistoryFile)
+                        friendBuildHistoryFile = friendTask?.buildHistoryFile,
+                        usePreciseJavaTracking = usePreciseJavaTracking
+                )
            }
        }