Implement module detection for Gradle IC with Android

#KT-22431 fixed

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt

@@ -65,7 +65,7 @@ class IncrementalCompilationOptions(
          * Directories that should be cleared when IC decides to rebuild
          */
         val localStateDirs: List<File>,
-        val buildHistoryFile: File,
+        val multiModuleICSettings: MultiModuleICSettings,
         val modulesInfo: IncrementalModuleInfo
 ) : CompilationOptions(compilerMode, targetPlatform, reportCategories, reportSeverity, requestedCompilationResults) {
     companion object {
@@ -81,13 +81,22 @@ class IncrementalCompilationOptions(
                "workingDir=$workingDir, " +
                "customCacheVersionFileName='$customCacheVersionFileName', " +
                "customCacheVersion=$customCacheVersion, " +
-               "buildHistoryFile=$buildHistoryFile, " +
+               "multiModuleICSettings=$multiModuleICSettings, " +
                "usePreciseJavaTracking=$usePreciseJavaTracking" +
                "localStateDirs=$localStateDirs" +
                ")"
     }
 }
 
+data class MultiModuleICSettings(
+    val buildHistoryFile: File,
+    val useModuleDetection: Boolean
+) : Serializable {
+    companion object {
+        const val serialVersionUID: Long = 0
+    }
+}
+
 enum class CompilerMode : Serializable {
     NON_INCREMENTAL_COMPILER,
     INCREMENTAL_COMPILER,

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -48,6 +48,7 @@ import org.jetbrains.kotlin.daemon.report.RemoteICReporter
 import org.jetbrains.kotlin.incremental.*
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.parsing.classesFqNames
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryAndroid
 import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJvm
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.modules.Module
@@ -523,14 +524,20 @@ class CompileServiceImpl(
                                           workingDir,
                                           enabled = true)
 
-        val modulesApiHistory = ModulesApiHistoryJvm(incrementalCompilationOptions.modulesInfo)
+        val modulesApiHistory = incrementalCompilationOptions.run {
+            if (!multiModuleICSettings.useModuleDetection) {
+                ModulesApiHistoryJvm(modulesInfo)
+            } else {
+                ModulesApiHistoryAndroid(modulesInfo)
+            }
+        }
 
         val compiler = IncrementalJvmCompilerRunner(
             workingDir,
             javaSourceRoots,
             versions,
             reporter, annotationFileUpdater,
-            buildHistoryFile = incrementalCompilationOptions.buildHistoryFile,
+            buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile,
             localStateDirs = incrementalCompilationOptions.localStateDirs,
             usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
             modulesApiHistory = modulesApiHistory

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt

@@ -97,5 +97,43 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo
     }
 }
 
+class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryJvm(modulesInfo) {
+    override fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>> {
+        // Module detection is expensive, so we don't don it for jars outside of project dir
+        if (!projectRootPath.isParentOf(jar)) return Either.Error("Non-project jar is modified $jar")
+
+        val jarPath = Paths.get(jar.absolutePath)
+        val possibleModules = getPossibleModuleNamesFromJar(jarPath)
+            .flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() }
+
+        val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(jarPath) }
+        if (modules.isEmpty()) return Either.Error("Unknown module for $jar (candidates: ${possibleModules.joinToString()})")
+
+        val result = modules.mapTo(HashSet()) { it.buildHistoryFile }
+        return Either.Success(result)
+    }
+
+    private fun getPossibleModuleNamesFromJar(path: Path): Collection<String> {
+        val result = HashSet<String>()
+
+        try {
+            ZipFile(path.toFile()).use { zip ->
+                val entries = zip.entries()
+                while (entries.hasMoreElements()) {
+                    val entry = entries.nextElement()
+                    val name = entry.name
+                    if (name.endsWith(".kotlin_module", ignoreCase = true)) {
+                        result.add(File(name).nameWithoutExtension)
+                    }
+                }
+            }
+        } catch (t: Throwable) {
+            return emptyList()
+        }
+
+        return result
+    }
+}
+
 private fun Path.isParentOf(path: Path) = path.startsWith(this)
 private fun Path.isParentOf(file: File) = this.isParentOf(Paths.get(file.absolutePath))

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/AbstractKotlinAndroidGradleTests.kt

@@ -157,6 +157,25 @@ fun getSomething() = 10
         }
     }
 
+    @Test
+    fun testMultiModuleIC() {
+        val project = Project("AndroidProject", gradleVersion)
+        val options = defaultBuildOptions().copy(incremental = true)
+
+        project.build("assembleDebug", options = options) {
+            assertSuccessful()
+        }
+
+        val libUtilKt = project.projectDir.getFileByName("libUtil.kt")
+        libUtilKt.modify { it.replace("fun libUtil(): String", "fun libUtil(): CharSequence") }
+
+        project.build("assembleDebug", options = options) {
+            assertSuccessful()
+            val affectedSources = project.projectDir.getFilesByNames("libUtil.kt", "MainActivity2.kt")
+            assertCompiledKotlinSources(project.relativize(affectedSources), weakTesting = false)
+        }
+    }
+
     @Test
     fun testIncrementalBuildWithNoChanges() {
         val project = Project("AndroidIncrementalSingleModuleProject", gradleVersion)

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/IncrementalCompilationMultiProjectIT.kt

@@ -9,18 +9,6 @@ import org.junit.Test
 import java.io.File
 
 class IncrementalCompilationMultiProjectIT : BaseGradleIT() {
-    companion object {
-        private val ANDROID_GRADLE_PLUGIN_VERSION = "1.5.+"
-    }
-
-    private fun androidBuildOptions() =
-        BuildOptions(
-            withDaemon = true,
-            androidHome = KotlinTestUtils.findAndroidSdk(),
-            androidGradlePluginVersion = ANDROID_GRADLE_PLUGIN_VERSION,
-            incremental = true
-        )
-
     override fun defaultBuildOptions(): BuildOptions =
         super.defaultBuildOptions().copy(withDaemon = true, incremental = true)
 
@@ -201,25 +189,6 @@ open class A {
             assertCompiledKotlinSources(relativePaths, weakTesting = false)
         }
     }
-
-    @Test
-    fun testAndroid() {
-        val project = Project("AndroidProject", GradleVersionRequired.Exact("2.10"))
-        val options = androidBuildOptions()
-
-        project.build("assembleDebug", options = options) {
-            assertSuccessful()
-        }
-
-        val libUtilKt = project.projectDir.getFileByName("libUtil.kt")
-        libUtilKt.modify { it.replace("fun libUtil(): String", "fun libUtil(): None") }
-
-        project.build("assembleDebug", options = options) {
-            assertSuccessful()
-            val affectedSources = project.projectDir.getFilesByNames("libUtil.kt", "MainActivity2.kt")
-            assertCompiledKotlinSources(project.relativize(affectedSources), weakTesting = false)
-        }
-    }
 }
 
 class IncrementalJavaChangeDefaultIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = null) {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt

@@ -3,6 +3,7 @@ package org.jetbrains.kotlin.compilerRunner
 import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
 import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector
 import org.jetbrains.kotlin.gradle.tasks.findToolsJar
 import org.jetbrains.kotlin.incremental.ChangedFiles
@@ -31,8 +32,8 @@ internal class GradleIncrementalCompilerEnvironment(
         messageCollector: GradleMessageCollector,
         outputItemsCollector: OutputItemsCollector,
         compilerArgs: CommonCompilerArguments,
-        val buildHistoryFile: File,
         val kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
         val usePreciseJavaTracking: Boolean = false,
-        val localStateDirs: List<File> = emptyList()
+        val localStateDirs: List<File> = emptyList(),
+        val multiModuleICSettings: MultiModuleICSettings
 ) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -275,7 +275,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                 targetPlatform = targetPlatform,
                 usePreciseJavaTracking = environment.usePreciseJavaTracking,
                 localStateDirs = environment.localStateDirs,
-                buildHistoryFile = environment.buildHistoryFile,
+                multiModuleICSettings = environment.multiModuleICSettings,
                 modulesInfo = buildModulesInfo(project.gradle)
         )
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -35,6 +35,7 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.compilerRunner.*
+import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
 import org.jetbrains.kotlin.gradle.dsl.*
 import org.jetbrains.kotlin.gradle.internal.CompilerArgumentAwareWithInput
 import org.jetbrains.kotlin.gradle.internal.prepareCompilerArguments
@@ -99,6 +100,13 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
     @get:Internal
     internal val buildHistoryFile: File get() = File(taskBuildDirectory, "build-history.bin")
 
+    @get:Input
+    internal var useModuleDetection: Boolean = false
+
+    @get:Internal
+    protected val multiModuleICSettings: MultiModuleICSettings
+        get() = MultiModuleICSettings(buildHistoryFile, useModuleDetection)
+
     @get:Internal
     internal val pluginOptions = CompilerPluginOptions()
 
@@ -353,11 +361,10 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
                         computedCompilerClasspath,
                         if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
                         taskBuildDirectory,
-                        messageCollector, outputItemCollector, args,
+                        messageCollector, outputItemCollector, args, kaptAnnotationsFileUpdater,
-                        buildHistoryFile = buildHistoryFile,
-                        kaptAnnotationsFileUpdater = kaptAnnotationsFileUpdater,
                         usePreciseJavaTracking = usePreciseJavaTracking,
-                        localStateDirs = outputDirectories
+                        localStateDirs = outputDirectories,
+                        multiModuleICSettings = multiModuleICSettings
                 )
             }
         }
@@ -509,7 +516,7 @@ open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>(),
                         if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
                         taskBuildDirectory,
                         messageCollector, outputItemCollector, args,
-                        buildHistoryFile = buildHistoryFile
+                        multiModuleICSettings = multiModuleICSettings
                 )
             }
             else -> {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksProvider.kt

@@ -22,25 +22,29 @@ import org.jetbrains.kotlin.gradle.plugin.TaskToFriendTaskMapper
 import org.jetbrains.kotlin.gradle.plugin.mapKotlinTaskProperties
 
 internal open class KotlinTasksProvider {
-    fun createKotlinJVMTask(project: Project, name: String, sourceSetName: String): KotlinCompile =
+    open fun createKotlinJVMTask(project: Project, name: String, sourceSetName: String): KotlinCompile =
         project.tasks.create(name, KotlinCompile::class.java).apply {
-            configure(project, sourceSetName)
+            configure(this, project, sourceSetName)
         }
 
     fun createKotlinJSTask(project: Project, name: String, sourceSetName: String): Kotlin2JsCompile =
         project.tasks.create(name, Kotlin2JsCompile::class.java).apply {
-            configure(project, sourceSetName)
+            configure(this, project, sourceSetName)
         }
 
     fun createKotlinCommonTask(project: Project, name: String, sourceSetName: String): KotlinCompileCommon =
         project.tasks.create(name, KotlinCompileCommon::class.java).apply {
-            configure(project, sourceSetName)
+            configure(this, project, sourceSetName)
         }
 
-    private fun AbstractKotlinCompile<*>.configure(project: Project, sourceSetName: String) {
+    open fun configure(
-        this.sourceSetName = sourceSetName
+        kotlinTask: AbstractKotlinCompile<*>,
-        this.friendTaskName = taskToFriendTaskMapper[this]
+        project: Project,
-        mapKotlinTaskProperties(project, this)
+        sourceSetName: String
+    ) {
+        kotlinTask.sourceSetName = sourceSetName
+        kotlinTask.friendTaskName = taskToFriendTaskMapper[kotlinTask]
+        mapKotlinTaskProperties(project, kotlinTask)
     }
 
     protected open val taskToFriendTaskMapper: TaskToFriendTaskMapper =
@@ -60,4 +64,9 @@ internal class Kotlin2JsTasksProvider : KotlinTasksProvider() {
 internal class AndroidTasksProvider : KotlinTasksProvider() {
     override val taskToFriendTaskMapper: TaskToFriendTaskMapper =
         RegexTaskToFriendTaskMapper.Android()
+
+    override fun configure(kotlinTask: AbstractKotlinCompile<*>, project: Project, sourceSetName: String) {
+        super.configure(kotlinTask, project, sourceSetName)
+        kotlinTask.useModuleDetection = true
+    }
 }