From 6bb54fd9a338f42188dd55704b40167f41e34ebd Mon Sep 17 00:00:00 2001 From: Ivan Gavrilovic Date: Tue, 5 Jun 2018 15:44:33 +0100 Subject: [PATCH] Support fetching build history files from directories for Android Android Gradle plugin will start publishing clases in directories. This commit adds support to find build history files when changed files come from directories. It is similar to existing implementation for jars i.e. it looks for .kotlin_module file under META-INF. Test: ModulesApiHistoryAndroidTest --- .../multiproject/ModulesApiHistory.kt | 76 ++++++--- .../ModulesApiHistoryAndroidTest.kt | 161 ++++++++++++++++++ 2 files changed, 218 insertions(+), 19 deletions(-) create mode 100644 compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt index 794982c6e56..2089b1f5350 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistory.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.incremental.multiproject +import org.jetbrains.kotlin.daemon.common.IncrementalModuleEntry import org.jetbrains.kotlin.daemon.common.IncrementalModuleInfo import org.jetbrains.kotlin.incremental.util.Either import java.io.File @@ -23,7 +24,7 @@ object EmptyModulesApiHistory : ModulesApiHistory { open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo) : ModulesApiHistory { protected val projectRootPath: Path = Paths.get(modulesInfo.projectRoot.absolutePath) - private val dirToHistoryFileCache = HashMap() + protected val dirToHistoryFileCache = HashMap>() override fun historyFilesForChangedFiles(changedFiles: Set): Either> { val result = HashSet() @@ -53,28 +54,37 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo val classFileDirs = classFiles.groupBy { it.parentFile } for ((dir, files) in classFileDirs) { - val buildHistory = getBuildHistoryForDir(dir) - ?: return Either.Error("Could not get build history for class files: ${files.joinToString()}") - result.add(buildHistory) + val historyEither = getBuildHistoryForDir(dir) + when (historyEither) { + is Either.Success> -> result.addAll(historyEither.value) + is Either.Error -> return historyEither + } } return Either.Success(result) } - private fun getBuildHistoryForDir(file: File): File? = - dirToHistoryFileCache.getOrPut(file) { + protected open fun getBuildHistoryForDir(file: File): Either> { + val history = dirToHistoryFileCache.getOrPut(file) { val module = modulesInfo.dirToModule[file] val parent = file.parentFile when { module != null -> - module.buildHistoryFile - parent != null && projectRootPath.isParentOf(parent) -> - getBuildHistoryForDir(parent) + setOf(module.buildHistoryFile) + parent != null && projectRootPath.isParentOf(parent) -> { + val parentHistory = getBuildHistoryForDir(parent) + when (parentHistory) { + is Either.Success> -> parentHistory.value + is Either.Error -> return parentHistory + } + } else -> - null + setOf() } } + return Either.Success(history) + } protected open fun getBuildHistoryFilesForJar(jar: File): Either> { val classListFile = modulesInfo.jarToClassListFile[jar] ?: return Either.Error("Unknown jar: $jar") @@ -89,9 +99,11 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo val classFileDirs = classFiles.groupBy { it.parentFile } val result = HashSet() for ((dir, files) in classFileDirs) { - val buildHistory = getBuildHistoryForDir(dir) - ?: return Either.Error("Could not get build history for class files: ${files.joinToString()}") - result.add(buildHistory) + val historyEither = getBuildHistoryForDir(dir) + when (historyEither) { + is Either.Success> -> result.addAll(historyEither.value) + is Either.Error -> return historyEither + } } return Either.Success(result) } @@ -103,14 +115,24 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH if (!projectRootPath.isParentOf(jar)) return Either.Error("Non-project jar is modified $jar") val jarPath = Paths.get(jar.absolutePath) - val possibleModules = getPossibleModuleNamesFromJar(jarPath) - .flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() } + return getHistoryForModuleNames(jarPath, getPossibleModuleNamesFromJar(jarPath)) + } - val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(jarPath) } - if (modules.isEmpty()) return Either.Error("Unknown module for $jar (candidates: ${possibleModules.joinToString()})") + override fun getBuildHistoryForDir(file: File): Either> { + if (!projectRootPath.isParentOf(file)) return Either.Error("Non-project file while looking for history $file") - val result = modules.mapTo(HashSet()) { it.buildHistoryFile } - return Either.Success(result) + // check both meta-inf and META-INF directories + val moduleNames = + getPossibleModuleNamesForDir(file.resolve("meta-inf")) + getPossibleModuleNamesForDir(file.resolve("META-INF")) + if (moduleNames.isEmpty()) { + return if (file.parentFile == null) { + Either.Error("Unable to find history for $file") + } else { + getBuildHistoryForDir(file.parentFile) + } + } + + return getHistoryForModuleNames(file.toPath(), moduleNames) } private fun getPossibleModuleNamesFromJar(path: Path): Collection { @@ -133,6 +155,22 @@ class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiH return result } + + private fun getPossibleModuleNamesForDir(path: File): List { + if (!path.isDirectory) return listOf() + + return path.listFiles().filter { it.name.endsWith(".kotlin_module", ignoreCase = true) }.map { it.nameWithoutExtension } + } + + private fun getHistoryForModuleNames(path: Path, moduleNames: Iterable): Either> { + val possibleModules = + moduleNames.flatMapTo(HashSet()) { modulesInfo.nameToModules[it] ?: emptySet() } + val modules = possibleModules.filter { Paths.get(it.buildDir.absolutePath).isParentOf(path) } + if (modules.isEmpty()) return Either.Error("Unknown module for $path (candidates: ${possibleModules.joinToString()})") + + val result = modules.mapTo(HashSet()) { it.buildHistoryFile } + return Either.Success(result) + } } private fun Path.isParentOf(path: Path) = path.startsWith(this) diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt new file mode 100644 index 00000000000..c292ef2e643 --- /dev/null +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/multiproject/ModulesApiHistoryAndroidTest.kt @@ -0,0 +1,161 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.multiproject + +import org.jetbrains.kotlin.daemon.common.IncrementalModuleEntry +import org.jetbrains.kotlin.daemon.common.IncrementalModuleInfo +import org.jetbrains.kotlin.incremental.util.Either +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class ModulesApiHistoryAndroidTest { + @JvmField + @Rule + val tmpFolder = TemporaryFolder() + + private lateinit var appRoot: File + private lateinit var appHistory: File + private lateinit var libRoot: File + private lateinit var libHistory: File + + private lateinit var androidHistory: ModulesApiHistoryAndroid + + @Before + fun setUp() { + val projectRoot = tmpFolder.newFolder() + + appRoot = projectRoot.resolve("app") + appHistory = appRoot.resolve("build/tmp/kotlin/app_history.bin") + val appEntry = IncrementalModuleEntry(":app", "app", appRoot.resolve("build"), appHistory) + appRoot.resolve("build/intermediates/classes/meta-inf/").apply { + mkdirs(); + resolve("app.kotlin_module").createNewFile() + } + + libRoot = projectRoot.resolve("lib") + libHistory = libRoot.resolve("lib/build/tmp/kotlin/lib_history.bin") + val libEntry = IncrementalModuleEntry(":lib", "lib", libRoot.resolve("build"), libHistory) + libRoot.resolve("build/intermediates/classes/meta-inf/").apply { + mkdirs(); + resolve("lib.kotlin_module").createNewFile() + } + + val info = IncrementalModuleInfo( + projectRoot = projectRoot, + dirToModule = mapOf(appRoot to appEntry, libRoot to libEntry), + nameToModules = mapOf("app" to setOf(appEntry), "lib" to setOf(libEntry)), + jarToClassListFile = mapOf() + ) + + androidHistory = ModulesApiHistoryAndroid(info) + } + + @Test + fun testClassChangeInAppTopLevel() { + val changed = appRoot.resolve("build/intermediates/classes/Changed.class").let { + it.mkdirs() + it + } + + val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changed)) + changedFiles as Either.Success> + assertEquals(setOf(appHistory), changedFiles.value) + } + + @Test + fun testClassChangeInAppInPackage() { + val changed = appRoot.resolve("build/intermediates/classes/com/exampleChanged.class").let { + it.mkdirs() + it + } + + val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changed)) + changedFiles as Either.Success> + assertEquals(setOf(appHistory), changedFiles.value) + } + + @Test + fun testClassMultipleChanges() { + + val classesRoot = appRoot.resolve("build/intermediates/classes/com/example/") + classesRoot.mkdirs() + val changed = 1.until(10).map { classesRoot.resolve("MyClass_$it.class") }.toSet() + + val changedFiles = androidHistory.historyFilesForChangedFiles(changed) + changedFiles as Either.Success> + assertEquals(setOf(appHistory), changedFiles.value) + } + + @Test + fun testClassChangeInAppOutsideClasses() { + val changed = appRoot.resolve("build/Changed.class").let { + it.mkdirs() + it + } + + val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changed)) + assertTrue(changedFiles is Either.Error, "Fetching history should fail for file outside classes dir.") + } + + @Test + fun testClassChangeInAppAndLib() { + val changedApp = appRoot.resolve("build/intermediates/classes/com/exampleChanged.class").let { + it.mkdirs() + it + } + val changedLib = libRoot.resolve("build/intermediates/classes/com/exampleChanged.class").let { + it.mkdirs() + it + } + + val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(changedApp, changedLib)) + changedFiles as Either.Success> + assertEquals(setOf(appHistory, libHistory), changedFiles.value) + } + + @Test + fun testJarChangeInApp() { + val jarFile = appRoot.resolve("build/intermediates/classes.jar") + jarFile.parentFile.mkdirs() + + ZipOutputStream(jarFile.outputStream()).use { + it.putNextEntry(ZipEntry("meta-inf/app.kotlin_module")) + it.closeEntry() + } + + val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(jarFile)) + changedFiles as Either.Success> + assertEquals(setOf(appHistory), changedFiles.value) + } + + @Test + fun testJarChangesInAppAndLib() { + val appJar = appRoot.resolve("build/intermediates/classes.jar") + appJar.parentFile.mkdirs() + ZipOutputStream(appJar.outputStream()).use { + it.putNextEntry(ZipEntry("meta-inf/app.kotlin_module")) + it.closeEntry() + } + + val libJar = libRoot.resolve("build/intermediates/classes.jar") + libJar.parentFile.mkdirs() + ZipOutputStream(libJar.outputStream()).use { + it.putNextEntry(ZipEntry("META-INF/lib.kotlin_module")) + it.closeEntry() + } + + val changedFiles = androidHistory.historyFilesForChangedFiles(setOf(appJar, libJar)) + changedFiles as Either.Success> + assertEquals(setOf(appHistory, libHistory), changedFiles.value) + } +}