Support fetching build history files from directories for Android

Android Gradle plugin will start publishing clases in directories.
This commit adds support to find build history files when changed
files come from directories. It is similar to existing implementation
for jars i.e. it looks for .kotlin_module file under META-INF.

Test: ModulesApiHistoryAndroidTest

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.incremental.multiproject
 
+import org.jetbrains.kotlin.daemon.common.IncrementalModuleEntry
 import org.jetbrains.kotlin.daemon.common.IncrementalModuleInfo
 import org.jetbrains.kotlin.incremental.util.Either
 import java.io.File
@@ -23,7 +24,7 @@ object EmptyModulesApiHistory : ModulesApiHistory {
 
 open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo) : ModulesApiHistory {
     protected val projectRootPath: Path = Paths.get(modulesInfo.projectRoot.absolutePath)
-    private val dirToHistoryFileCache = HashMap<File, File?>()
+    protected val dirToHistoryFileCache = HashMap<File, Set<File>>()
 
     override fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>> {
         val result = HashSet<File>()
@@ -53,28 +54,37 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo
 
         val classFileDirs = classFiles.groupBy { it.parentFile }
         for ((dir, files) in classFileDirs) {
-            val buildHistory = getBuildHistoryForDir(dir)
+            val historyEither = getBuildHistoryForDir(dir)
-                    ?: return Either.Error("Could not get build history for class files: ${files.joinToString()}")
+            when (historyEither) {
-            result.add(buildHistory)
+                is Either.Success<Set<File>> -> result.addAll(historyEither.value)
+                is Either.Error -> return historyEither
+            }
         }
 
         return Either.Success(result)
     }
 
-    private fun getBuildHistoryForDir(file: File): File? =
+    protected open fun getBuildHistoryForDir(file: File): Either<Set<File>> {
-        dirToHistoryFileCache.getOrPut(file) {
+        val history = dirToHistoryFileCache.getOrPut(file) {
             val module = modulesInfo.dirToModule[file]
             val parent = file.parentFile
 
             when {
                 module != null ->
-                    module.buildHistoryFile
+                    setOf(module.buildHistoryFile)
-                parent != null && projectRootPath.isParentOf(parent) ->
+                parent != null && projectRootPath.isParentOf(parent) -> {
-                    getBuildHistoryForDir(parent)
+                    val parentHistory = getBuildHistoryForDir(parent)
+                    when (parentHistory) {
+                        is Either.Success<Set<File>> -> parentHistory.value
+                        is Either.Error -> return parentHistory
+                    }
+                }
                 else ->
-                    null
+                    setOf()
             }
         }
+        return Either.Success(history)
+    }
 
     protected open fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>> {
         val classListFile = modulesInfo.jarToClassListFile[jar] ?: return Either.Error("Unknown jar: $jar")
@@ -89,9 +99,11 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo
         val classFileDirs = classFiles.groupBy { it.parentFile }
         val result = HashSet<File>()
         for ((dir, files) in classFileDirs) {
-            val buildHistory = getBuildHistoryForDir(dir)
+            val historyEither = getBuildHistoryForDir(dir)
-                    ?: return Either.Error("Could not get build history for class files: ${files.joinToString()}")
+            when (historyEither) {
-            result.add(buildHistory)
+                is Either.Success<Set<File>> -> result.addAll(historyEither.value)
+                is Either.Error -> return historyEither
+            }
         }
         return Either.Success(result)
     }
@@ -103,14 +115,24 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH
         if (!projectRootPath.isParentOf(jar)) return Either.Error("Non-project jar is modified $jar")
 
         val jarPath = Paths.get(jar.absolutePath)
-        val possibleModules = getPossibleModuleNamesFromJar(jarPath)
+        return getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath))
-            .flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() }
+    }
 
-        val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(jarPath) }
+    override fun getBuildHistoryForDir(file: File): Either<Set<File>> {
-        if (modules.isEmpty()) return Either.Error("Unknown module for $jar (candidates: ${possibleModules.joinToString()})")
+        if (!projectRootPath.isParentOf(file)) return Either.Error("Non-project file while looking for history $file")
 
-        val result = modules.mapTo(HashSet()) { it.buildHistoryFile }
+        // check both meta-inf and META-INF directories
-        return Either.Success(result)
+        val moduleNames =
+            getPossibleModuleNamesForDir(file.resolve("meta-inf")) + getPossibleModuleNamesForDir(file.resolve("META-INF"))
+        if (moduleNames.isEmpty()) {
+            return if (file.parentFile == null) {
+                Either.Error("Unable to find history for $file")
+            } else {
+                getBuildHistoryForDir(file.parentFile)
+            }
+        }
+
+        return getHistoryForModuleNames(file.toPath(), moduleNames)
     }
 
     private fun getPossibleModuleNamesFromJar(path: Path): Collection<String> {
@@ -133,6 +155,22 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH
 
         return result
     }
+
+    private fun getPossibleModuleNamesForDir(path: File): List<String> {
+        if (!path.isDirectory) return listOf()
+
+        return path.listFiles().filter { it.name.endsWith(".kotlin_module", ignoreCase = true) }.map { it.nameWithoutExtension }
+    }
+
+    private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable<String>): Either<Set<File>> {
+        val possibleModules =
+            moduleNames.flatMapTo(HashSet<IncrementalModuleEntry>()) { modulesInfo.nameToModules[it] ?: emptySet() }
+        val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(path) }
+        if (modules.isEmpty()) return Either.Error("Unknown module for $path (candidates: ${possibleModules.joinToString()})")
+
+        val result = modules.mapTo(HashSet()) { it.buildHistoryFile }
+        return Either.Success(result)
+    }
 }
 
 private fun Path.isParentOf(path: Path) = path.startsWith(this)

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.multiproject
+
+import org.jetbrains.kotlin.daemon.common.IncrementalModuleEntry
+import org.jetbrains.kotlin.daemon.common.IncrementalModuleInfo
+import org.jetbrains.kotlin.incremental.util.Either
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import java.io.File
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class ModulesApiHistoryAndroidTest {
+    @JvmField
+    @Rule
+    val tmpFolder = TemporaryFolder()
+
+    private lateinit var appRoot: File
+    private lateinit var appHistory: File
+    private lateinit var libRoot: File
+    private lateinit var libHistory: File
+
+    private lateinit var androidHistory: ModulesApiHistoryAndroid
+
+    @Before
+    fun setUp() {
+        val projectRoot = tmpFolder.newFolder()
+
+        appRoot = projectRoot.resolve("app")
+        appHistory = appRoot.resolve("build/tmp/kotlin/app_history.bin")
+        val appEntry = IncrementalModuleEntry(":app", "app", appRoot.resolve("build"), appHistory)
+        appRoot.resolve("build/intermediates/classes/meta-inf/").apply {
+            mkdirs();
+            resolve("app.kotlin_module").createNewFile()
+        }
+
+        libRoot = projectRoot.resolve("lib")
+        libHistory = libRoot.resolve("lib/build/tmp/kotlin/lib_history.bin")
+        val libEntry = IncrementalModuleEntry(":lib", "lib", libRoot.resolve("build"), libHistory)
+        libRoot.resolve("build/intermediates/classes/meta-inf/").apply {
+            mkdirs();
+            resolve("lib.kotlin_module").createNewFile()
+        }
+
+        val info = IncrementalModuleInfo(
+            projectRoot = projectRoot,
+            dirToModule = mapOf(appRoot to appEntry, libRoot to libEntry),
+            nameToModules = mapOf("app" to setOf(appEntry), "lib" to setOf(libEntry)),
+            jarToClassListFile = mapOf()
+        )
+
+        androidHistory = ModulesApiHistoryAndroid(info)
+    }
+
+    @Test
+    fun testClassChangeInAppTopLevel() {
+        val changed = appRoot.resolve("build/intermediates/classes/Changed.class").let {
+            it.mkdirs()
+            it
+        }
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changed))
+        changedFiles as Either.Success<Set<File>>
+        assertEquals(setOf(appHistory), changedFiles.value)
+    }
+
+    @Test
+    fun testClassChangeInAppInPackage() {
+        val changed = appRoot.resolve("build/intermediates/classes/com/exampleChanged.class").let {
+            it.mkdirs()
+            it
+        }
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changed))
+        changedFiles as Either.Success<Set<File>>
+        assertEquals(setOf(appHistory), changedFiles.value)
+    }
+
+    @Test
+    fun testClassMultipleChanges() {
+
+        val classesRoot = appRoot.resolve("build/intermediates/classes/com/example/")
+        classesRoot.mkdirs()
+        val changed = 1.until(10).map { classesRoot.resolve("MyClass_$it.class") }.toSet()
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(changed)
+        changedFiles as Either.Success<Set<File>>
+        assertEquals(setOf(appHistory), changedFiles.value)
+    }
+
+    @Test
+    fun testClassChangeInAppOutsideClasses() {
+        val changed = appRoot.resolve("build/Changed.class").let {
+            it.mkdirs()
+            it
+        }
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changed))
+        assertTrue(changedFiles is Either.Error, "Fetching history should fail for file outside classes dir.")
+    }
+
+    @Test
+    fun testClassChangeInAppAndLib() {
+        val changedApp = appRoot.resolve("build/intermediates/classes/com/exampleChanged.class").let {
+            it.mkdirs()
+            it
+        }
+        val changedLib = libRoot.resolve("build/intermediates/classes/com/exampleChanged.class").let {
+            it.mkdirs()
+            it
+        }
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changedApp, changedLib))
+        changedFiles as Either.Success<Set<File>>
+        assertEquals(setOf(appHistory, libHistory), changedFiles.value)
+    }
+
+    @Test
+    fun testJarChangeInApp() {
+        val jarFile = appRoot.resolve("build/intermediates/classes.jar")
+        jarFile.parentFile.mkdirs()
+
+        ZipOutputStream(jarFile.outputStream()).use {
+            it.putNextEntry(ZipEntry("meta-inf/app.kotlin_module"))
+            it.closeEntry()
+        }
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(jarFile))
+        changedFiles as Either.Success<Set<File>>
+        assertEquals(setOf(appHistory), changedFiles.value)
+    }
+
+    @Test
+    fun testJarChangesInAppAndLib() {
+        val appJar = appRoot.resolve("build/intermediates/classes.jar")
+        appJar.parentFile.mkdirs()
+        ZipOutputStream(appJar.outputStream()).use {
+            it.putNextEntry(ZipEntry("meta-inf/app.kotlin_module"))
+            it.closeEntry()
+        }
+
+        val libJar = libRoot.resolve("build/intermediates/classes.jar")
+        libJar.parentFile.mkdirs()
+        ZipOutputStream(libJar.outputStream()).use {
+            it.putNextEntry(ZipEntry("META-INF/lib.kotlin_module"))
+            it.closeEntry()
+        }
+
+        val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(appJar, libJar))
+        changedFiles as Either.Success<Set<File>>
+        assertEquals(setOf(appHistory, libHistory), changedFiles.value)
+    }
+}