Implement module detection for Gradle IC with Android

#KT-22431 fixed
2018-05-17 05:34:07 +03:00
parent e5cdc64f4f
commit 53266300d4
9 changed files with 110 additions and 51 deletions
@@ -65,7 +65,7 @@ class IncrementalCompilationOptions(
         * Directories that should be cleared when IC decides to rebuild
         */
        val localStateDirs: List<File>,
-        val buildHistoryFile: File,
+        val multiModuleICSettings: MultiModuleICSettings,
        val modulesInfo: IncrementalModuleInfo
 ) : CompilationOptions(compilerMode, targetPlatform, reportCategories, reportSeverity, requestedCompilationResults) {
    companion object {
@@ -81,13 +81,22 @@ class IncrementalCompilationOptions(
               "workingDir=$workingDir, " +
               "customCacheVersionFileName='$customCacheVersionFileName', " +
               "customCacheVersion=$customCacheVersion, " +
-               "buildHistoryFile=$buildHistoryFile, " +
+               "multiModuleICSettings=$multiModuleICSettings, " +
               "usePreciseJavaTracking=$usePreciseJavaTracking" +
               "localStateDirs=$localStateDirs" +
               ")"
    }
 }

+data class MultiModuleICSettings(
+    val buildHistoryFile: File,
+    val useModuleDetection: Boolean
+) : Serializable {
+    companion object {
+        const val serialVersionUID: Long = 0
+    }
+}
+
 enum class CompilerMode : Serializable {
    NON_INCREMENTAL_COMPILER,
    INCREMENTAL_COMPILER,
@@ -48,6 +48,7 @@ import org.jetbrains.kotlin.daemon.report.RemoteICReporter
 import org.jetbrains.kotlin.incremental.*
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.parsing.classesFqNames
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryAndroid
 import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJvm
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.modules.Module
@@ -523,14 +524,20 @@ class CompileServiceImpl(
                                          workingDir,
                                          enabled = true)

-        val modulesApiHistory = ModulesApiHistoryJvm(incrementalCompilationOptions.modulesInfo)
+        val modulesApiHistory = incrementalCompilationOptions.run {
+            if (!multiModuleICSettings.useModuleDetection) {
+                ModulesApiHistoryJvm(modulesInfo)
+            } else {
+                ModulesApiHistoryAndroid(modulesInfo)
+            }
+        }

        val compiler = IncrementalJvmCompilerRunner(
            workingDir,
            javaSourceRoots,
            versions,
            reporter, annotationFileUpdater,
-            buildHistoryFile = incrementalCompilationOptions.buildHistoryFile,
+            buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile,
            localStateDirs = incrementalCompilationOptions.localStateDirs,
            usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
            modulesApiHistory = modulesApiHistory
@@ -97,5 +97,43 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo
    }
 }

+class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryJvm(modulesInfo) {
+    override fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>> {
+        // Module detection is expensive, so we don't don it for jars outside of project dir
+        if (!projectRootPath.isParentOf(jar)) return Either.Error("Non-project jar is modified $jar")
+
+        val jarPath = Paths.get(jar.absolutePath)
+        val possibleModules = getPossibleModuleNamesFromJar(jarPath)
+            .flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() }
+
+        val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(jarPath) }
+        if (modules.isEmpty()) return Either.Error("Unknown module for $jar (candidates: ${possibleModules.joinToString()})")
+
+        val result = modules.mapTo(HashSet()) { it.buildHistoryFile }
+        return Either.Success(result)
+    }
+
+    private fun getPossibleModuleNamesFromJar(path: Path): Collection<String> {
+        val result = HashSet<String>()
+
+        try {
+            ZipFile(path.toFile()).use { zip ->
+                val entries = zip.entries()
+                while (entries.hasMoreElements()) {
+                    val entry = entries.nextElement()
+                    val name = entry.name
+                    if (name.endsWith(".kotlin_module", ignoreCase = true)) {
+                        result.add(File(name).nameWithoutExtension)
+                    }
+                }
+            }
+        } catch (t: Throwable) {
+            return emptyList()
+        }
+
+        return result
+    }
+}
+
 private fun Path.isParentOf(path: Path) = path.startsWith(this)
 private fun Path.isParentOf(file: File) = this.isParentOf(Paths.get(file.absolutePath))
@@ -157,6 +157,25 @@ fun getSomething() = 10
        }
    }

+    @Test
+    fun testMultiModuleIC() {
+        val project = Project("AndroidProject", gradleVersion)
+        val options = defaultBuildOptions().copy(incremental = true)
+
+        project.build("assembleDebug", options = options) {
+            assertSuccessful()
+        }
+
+        val libUtilKt = project.projectDir.getFileByName("libUtil.kt")
+        libUtilKt.modify { it.replace("fun libUtil(): String", "fun libUtil(): CharSequence") }
+
+        project.build("assembleDebug", options = options) {
+            assertSuccessful()
+            val affectedSources = project.projectDir.getFilesByNames("libUtil.kt", "MainActivity2.kt")
+            assertCompiledKotlinSources(project.relativize(affectedSources), weakTesting = false)
+        }
+    }
+
    @Test
    fun testIncrementalBuildWithNoChanges() {
        val project = Project("AndroidIncrementalSingleModuleProject", gradleVersion)
@@ -9,18 +9,6 @@ import org.junit.Test
 import java.io.File

 class IncrementalCompilationMultiProjectIT : BaseGradleIT() {
-    companion object {
-        private val ANDROID_GRADLE_PLUGIN_VERSION = "1.5.+"
-    }
-
-    private fun androidBuildOptions() =
-        BuildOptions(
-            withDaemon = true,
-            androidHome = KotlinTestUtils.findAndroidSdk(),
-            androidGradlePluginVersion = ANDROID_GRADLE_PLUGIN_VERSION,
-            incremental = true
-        )
-
    override fun defaultBuildOptions(): BuildOptions =
        super.defaultBuildOptions().copy(withDaemon = true, incremental = true)

@@ -201,25 +189,6 @@ open class A {
            assertCompiledKotlinSources(relativePaths, weakTesting = false)
        }
    }
-
-    @Test
-    fun testAndroid() {
-        val project = Project("AndroidProject", GradleVersionRequired.Exact("2.10"))
-        val options = androidBuildOptions()
-
-        project.build("assembleDebug", options = options) {
-            assertSuccessful()
-        }
-
-        val libUtilKt = project.projectDir.getFileByName("libUtil.kt")
-        libUtilKt.modify { it.replace("fun libUtil(): String", "fun libUtil(): None") }
-
-        project.build("assembleDebug", options = options) {
-            assertSuccessful()
-            val affectedSources = project.projectDir.getFilesByNames("libUtil.kt", "MainActivity2.kt")
-            assertCompiledKotlinSources(project.relativize(affectedSources), weakTesting = false)
-        }
-    }
 }

 class IncrementalJavaChangeDefaultIT : IncrementalCompilationJavaChangesBase(usePreciseJavaTracking = null) {
@@ -3,6 +3,7 @@ package org.jetbrains.kotlin.compilerRunner
 import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
 import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector
 import org.jetbrains.kotlin.gradle.tasks.findToolsJar
 import org.jetbrains.kotlin.incremental.ChangedFiles
@@ -31,8 +32,8 @@ internal class GradleIncrementalCompilerEnvironment(
        messageCollector: GradleMessageCollector,
        outputItemsCollector: OutputItemsCollector,
        compilerArgs: CommonCompilerArguments,
-        val buildHistoryFile: File,
        val kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
        val usePreciseJavaTracking: Boolean = false,
-        val localStateDirs: List<File> = emptyList()
+        val localStateDirs: List<File> = emptyList(),
+        val multiModuleICSettings: MultiModuleICSettings
 ) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)
@@ -275,7 +275,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                targetPlatform = targetPlatform,
                usePreciseJavaTracking = environment.usePreciseJavaTracking,
                localStateDirs = environment.localStateDirs,
-                buildHistoryFile = environment.buildHistoryFile,
+                multiModuleICSettings = environment.multiModuleICSettings,
                modulesInfo = buildModulesInfo(project.gradle)
        )

@@ -35,6 +35,7 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.compilerRunner.*
+import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
 import org.jetbrains.kotlin.gradle.dsl.*
 import org.jetbrains.kotlin.gradle.internal.CompilerArgumentAwareWithInput
 import org.jetbrains.kotlin.gradle.internal.prepareCompilerArguments
@@ -99,6 +100,13 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
    @get:Internal
    internal val buildHistoryFile: File get() = File(taskBuildDirectory, "build-history.bin")

+    @get:Input
+    internal var useModuleDetection: Boolean = false
+
+    @get:Internal
+    protected val multiModuleICSettings: MultiModuleICSettings
+        get() = MultiModuleICSettings(buildHistoryFile, useModuleDetection)
+
    @get:Internal
    internal val pluginOptions = CompilerPluginOptions()

@@ -353,11 +361,10 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
                        computedCompilerClasspath,
                        if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
                        taskBuildDirectory,
-                        messageCollector, outputItemCollector, args,
-                        buildHistoryFile = buildHistoryFile,
-                        kaptAnnotationsFileUpdater = kaptAnnotationsFileUpdater,
+                        messageCollector, outputItemCollector, args, kaptAnnotationsFileUpdater,
                        usePreciseJavaTracking = usePreciseJavaTracking,
-                        localStateDirs = outputDirectories
+                        localStateDirs = outputDirectories,
+                        multiModuleICSettings = multiModuleICSettings
                )
            }
        }
@@ -509,7 +516,7 @@ open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>(),
                        if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
                        taskBuildDirectory,
                        messageCollector, outputItemCollector, args,
-                        buildHistoryFile = buildHistoryFile
+                        multiModuleICSettings = multiModuleICSettings
                )
            }
            else -> {
@@ -22,25 +22,29 @@ import org.jetbrains.kotlin.gradle.plugin.TaskToFriendTaskMapper
 import org.jetbrains.kotlin.gradle.plugin.mapKotlinTaskProperties

 internal open class KotlinTasksProvider {
-    fun createKotlinJVMTask(project: Project, name: String, sourceSetName: String): KotlinCompile =
+    open fun createKotlinJVMTask(project: Project, name: String, sourceSetName: String): KotlinCompile =
        project.tasks.create(name, KotlinCompile::class.java).apply {
-            configure(project, sourceSetName)
+            configure(this, project, sourceSetName)
        }

    fun createKotlinJSTask(project: Project, name: String, sourceSetName: String): Kotlin2JsCompile =
        project.tasks.create(name, Kotlin2JsCompile::class.java).apply {
-            configure(project, sourceSetName)
+            configure(this, project, sourceSetName)
        }

    fun createKotlinCommonTask(project: Project, name: String, sourceSetName: String): KotlinCompileCommon =
        project.tasks.create(name, KotlinCompileCommon::class.java).apply {
-            configure(project, sourceSetName)
+            configure(this, project, sourceSetName)
        }

-    private fun AbstractKotlinCompile<*>.configure(project: Project, sourceSetName: String) {
-        this.sourceSetName = sourceSetName
-        this.friendTaskName = taskToFriendTaskMapper[this]
-        mapKotlinTaskProperties(project, this)
+    open fun configure(
+        kotlinTask: AbstractKotlinCompile<*>,
+        project: Project,
+        sourceSetName: String
+    ) {
+        kotlinTask.sourceSetName = sourceSetName
+        kotlinTask.friendTaskName = taskToFriendTaskMapper[kotlinTask]
+        mapKotlinTaskProperties(project, kotlinTask)
    }

    protected open val taskToFriendTaskMapper: TaskToFriendTaskMapper =
@@ -60,4 +64,9 @@ internal class Kotlin2JsTasksProvider : KotlinTasksProvider() {
 internal class AndroidTasksProvider : KotlinTasksProvider() {
    override val taskToFriendTaskMapper: TaskToFriendTaskMapper =
        RegexTaskToFriendTaskMapper.Android()
+
+    override fun configure(kotlinTask: AbstractKotlinCompile<*>, project: Project, sourceSetName: String) {
+        super.configure(kotlinTask, project, sourceSetName)
+        kotlinTask.useModuleDetection = true
+    }
 }