azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Incremental KAPT - pass changed classpath entries

Browse Source 
Pass computed list of changed classpath names to KAPT instead
of relying on the history files to be computed by stub generation.
Also, stop generating classpath history changes during the stub generation.

This commit does not compute the actual changed classpath entries,
and that will be done in the following commits.

 #KT-23880
This commit is contained in:
Ivan Gavrilovic
2019-03-26 17:31:17 +00:00
committed by Alexey Tsvetkov
parent 0c09f56118
commit c85e21d43b
26 changed files with 136 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt
-2
View File
@@ -67,7 +67,6 @@ class IncrementalCompilationOptions(
val outputFiles: List<File>,
val multiModuleICSettings: MultiModuleICSettings,
val modulesInfo: IncrementalModuleInfo,
val classpathFqNamesHistory: File? = null,
kotlinScriptExtensions: Array<String>? = null
) : CompilationOptions(
compilerMode,
@@ -91,7 +90,6 @@ class IncrementalCompilationOptions(
"multiModuleICSettings=$multiModuleICSettings, " +
"usePreciseJavaTracking=$usePreciseJavaTracking" +
"outputFiles=$outputFiles" +
"classpathFqNamesHistory=$classpathFqNamesHistory" +
")"
}
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt
+2 -6
View File
@@ -566,18 +566,14 @@ class CompileServiceImpl(
}
}
val outputFiles = incrementalCompilationOptions.outputFiles.toMutableList()
incrementalCompilationOptions.classpathFqNamesHistory?.let { outputFiles.add(it) }
val compiler = IncrementalJvmCompilerRunner(
workingDir,
reporter,
buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile,
outputFiles = outputFiles,
outputFiles = incrementalCompilationOptions.outputFiles,
usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
modulesApiHistory = modulesApiHistory,
kotlinSourceFilesExtensions = allKotlinExtensions,
classpathFqNamesHistory = incrementalCompilationOptions.classpathFqNamesHistory
kotlinSourceFilesExtensions = allKotlinExtensions
)
return try {
compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles)
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt
+1 -1
View File
@@ -313,7 +313,7 @@ abstract class IncrementalCompilerRunner<
open fun runWithNoDirtyKotlinSources(caches: CacheManager): Boolean = false
protected open fun processChangesAfterBuild(
private fun processChangesAfterBuild(
compilationMode: CompilationMode,
currentBuildInfo: BuildInfo,
dirtyData: DirtyData
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt
+1 -24
View File
@@ -109,8 +109,7 @@ class IncrementalJvmCompilerRunner(
buildHistoryFile: File,
outputFiles: Collection<File>,
private val modulesApiHistory: ModulesApiHistory,
override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
private val classpathFqNamesHistory: File? = null
override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
workingDir,
"caches-jvm",
@@ -257,28 +256,6 @@ class IncrementalJvmCompilerRunner(
}
}
override fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
super.processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)
classpathFqNamesHistory ?: return
classpathFqNamesHistory.mkdirs()
val historyFiles = classpathFqNamesHistory.listFiles()
if (dirtyClasspathChanges.isEmpty() && historyFiles.isNotEmpty()) {
// Don't write an empty file. We check there is at least one file so that downstream task can mark what it has processed.
return
}
if (historyFiles.size > 10) {
historyFiles.minBy { it.lastModified() }!!.delete()
}
val newHistoryFile = classpathFqNamesHistory.resolve(System.currentTimeMillis().toString())
ObjectOutputStream(newHistoryFile.outputStream().buffered()).use {
val listOfNames = dirtyClasspathChanges.map { it.toString() }.toList()
it.writeObject(listOfNames)
}
}
override fun postCompilationHook(exitCode: ExitCode) {}
override fun updateCaches(
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithAggregatingApt.kt
-12
View File
@@ -35,26 +35,14 @@ class KaptIncrementalWithAggregatingApt : KaptIncrementalIT() {
fun testIncrementalChanges() {
val project = getProject()
var aptTimestamp = 0L
project.build("clean", "build") {
assertSuccessful()
val classpathHistory =
fileInWorkingDir("build/kotlin/kaptGenerateStubsKotlin/classpath-fq-history").listFiles().asList().single()
val stubsTimestamp = classpathHistory.name.toLong()
aptTimestamp = fileInWorkingDir("build/tmp/kapt3/incApCache/main/last-build-ts.bin").readText().toLong()
assertTrue(stubsTimestamp < aptTimestamp)
}
project.projectFile("useB.kt").modify { current -> "$current\nfun otherFunction() {}" }
project.build("build") {
assertSuccessful()
val newAptTimestamp = fileInWorkingDir("build/tmp/kapt3/incApCache/main/last-build-ts.bin").readText().toLong()
assertTrue(aptTimestamp < newAptTimestamp)
assertEquals(
setOf(
fileInWorkingDir("build/tmp/kapt3/stubs/main/bar/UseBKt.java").absolutePath,
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithIsolatingApt.kt
-12
View File
@@ -38,26 +38,14 @@ class KaptIncrementalWithIsolatingApt : KaptIncrementalIT() {
fun testIncrementalChanges() {
val project = getProject()
var aptTimestamp = 0L
project.build("clean", "build") {
assertSuccessful()
val classpathHistory =
fileInWorkingDir("build/kotlin/kaptGenerateStubsKotlin/classpath-fq-history").listFiles().asList().single()
val stubsTimestamp = classpathHistory.name.toLong()
aptTimestamp = fileInWorkingDir("build/tmp/kapt3/incApCache/main/last-build-ts.bin").readText().toLong()
assertTrue(stubsTimestamp < aptTimestamp)
}
project.projectFile("useB.kt").modify { current -> "$current\nfun otherFunction() {}" }
project.build("build") {
assertSuccessful()
val newAptTimestamp = fileInWorkingDir("build/tmp/kapt3/incApCache/main/last-build-ts.bin").readText().toLong()
assertTrue(aptTimestamp < newAptTimestamp)
assertEquals(setOf(fileInWorkingDir("build/tmp/kapt3/stubs/main/bar/UseBKt.java").absolutePath), getProcessedSources(output))
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt
-1
View File
@@ -268,7 +268,6 @@ internal class GradleKotlinCompilerWork @Inject constructor(
outputFiles = outputFiles,
multiModuleICSettings = icEnv.multiModuleICSettings,
modulesInfo = incrementalModuleInfo!!,
classpathFqNamesHistory = icEnv.classpathFqNamesHistory,
kotlinScriptExtensions = kotlinScriptExtensions
)
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt
+1 -2
View File
@@ -15,8 +15,7 @@ internal class IncrementalCompilationEnvironment(
val workingDir: File,
val usePreciseJavaTracking: Boolean = false,
val disableMultiModuleIC: Boolean = false,
val multiModuleICSettings: MultiModuleICSettings,
val classpathFqNamesHistory: File? = null
val multiModuleICSettings: MultiModuleICSettings
) : Serializable {
companion object {
const val serialVersionUID: Long = 0
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt
+2 -9
View File
@@ -234,9 +234,7 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
)
val kaptGenerateStubsTask = context.createKaptGenerateStubsTask()
val kaptTask = context.createKaptKotlinTask(
useWorkerApi = project.isUseWorkerApi(),
classpathHistoryDir = kaptGenerateStubsTask.getClasspathFqNamesHistoryDir())
val kaptTask = context.createKaptKotlinTask(useWorkerApi = project.isUseWorkerApi())
kaptGenerateStubsTask.source(*kaptConfigurations.toTypedArray())
@@ -374,7 +372,7 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
}
}
private fun Kapt3SubpluginContext.createKaptKotlinTask(useWorkerApi: Boolean, classpathHistoryDir: File? = null): KaptTask {
private fun Kapt3SubpluginContext.createKaptKotlinTask(useWorkerApi: Boolean): KaptTask {
val taskClass = if (useWorkerApi) KaptWithoutKotlincTask::class.java else KaptWithKotlincTask::class.java
val kaptTask = project.tasks.create(getKaptTaskName("kapt"), taskClass)
@@ -391,7 +389,6 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
kaptTask.isIncremental = project.isIncrementalKapt()
if (kaptTask.isIncremental) {
kaptTask.incAptCache = getKaptIncrementalAnnotationProcessingCache()
kaptTask.classpathDirtyFqNamesHistoryDir = project.files(classpathHistoryDir)
}
kotlinCompilation?.run {
@@ -422,10 +419,6 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
kaptTask.pluginOptions.addPluginArgument(
getCompilerPluginId(),
SubpluginOption("incrementalCache", kaptTask.incAptCache!!.absolutePath))
kaptTask.pluginOptions.addPluginArgument(
getCompilerPluginId(),
SubpluginOption("classpathFqNamesHistory", kaptTask.classpathDirtyFqNamesHistoryDir.singleFile!!.absolutePath))
}
buildAndAddOptionsTo(kaptTask, kaptTask.pluginOptions, aptMode = "apt")
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptGenerateStubsTask.kt
-5
View File
@@ -74,11 +74,6 @@ open class KaptGenerateStubsTask : KotlinCompile() {
super.setSource(sourceRootsContainer.set(sources))
}
@Internal
override fun getClasspathFqNamesHistoryDir(): File? {
return taskBuildDirectory.resolve("classpath-fq-history")
}
private fun isSourceRootAllowed(source: File): Boolean =
!destinationDir.isParentOf(source) &&
!stubsDir.isParentOf(source) &&
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt
+9 -6
View File
@@ -67,9 +67,6 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState {
@get:Input
internal var includeCompileClasspath: Boolean = true
@get:InputFiles
internal var classpathDirtyFqNamesHistoryDir: FileCollection = project.files()
@get:Input
internal var isIncremental = true
@@ -160,15 +157,21 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState {
protected fun getCompiledSources() = listOfNotNull(kotlinCompileTask.destinationDir, kotlinCompileTask.javaOutputDir)
protected fun getChangedFiles(inputs: IncrementalTaskInputs): List<File> {
return if (!isIncremental || !inputs.isIncremental || !getCompiledSources().all { it.exists() }) {
if (!isIncremental || !inputs.isIncremental || !getCompiledSources().all { it.exists() }) {
clearLocalState()
emptyList()
return emptyList()
} else {
with(mutableSetOf<File>()) {
val changes = with(mutableSetOf<File>()) {
inputs.outOfDate { this.add(it.file) }
inputs.removed { this.add(it.file) }
return@with this.toList()
}
return if (changes.all { it.extension == "java" }) {
changes
} else {
emptyList()
}
}
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt
+19 -8
View File
@@ -16,17 +16,13 @@ import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment
import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
import org.jetbrains.kotlin.compilerRunner.IncrementalCompilationEnvironment
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
import org.jetbrains.kotlin.gradle.incremental.ChangedFiles
import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector
import org.jetbrains.kotlin.gradle.plugin.PLUGIN_CLASSPATH_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.tasks.CompilerPluginOptions
import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector
import org.jetbrains.kotlin.gradle.tasks.clearLocalState
import org.jetbrains.kotlin.gradle.utils.toSortedPathsArray
import org.jetbrains.kotlin.incremental.ChangedFiles
import java.io.File
open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JVMCompilerArguments> {
@@ -54,7 +50,10 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
val pluginOptionsWithKapt: CompilerPluginOptions = pluginOptions.withWrappedKaptOptions(
withApClasspath = kaptClasspath,
changedFiles = changedFiles,
compiledSourcesDir = getCompiledSources())
classpathChanges = classpathChanges,
compiledSourcesDir = getCompiledSources(),
processIncrementally = processIncrementally
)
args.pluginOptions = (pluginOptionsWithKapt.arguments + args.pluginOptions!!).toTypedArray()
@@ -66,13 +65,25 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
* in the task action.
*/
private var changedFiles: List<File> = emptyList()
private var classpathChanges: List<String> = emptyList()
private var processIncrementally = false
@TaskAction
fun compile(inputs: IncrementalTaskInputs) {
logger.debug("Running kapt annotation processing using the Kotlin compiler")
checkAnnotationProcessorClasspath()
changedFiles = getChangedFiles(inputs)
val incrementalChanges = getChangedFiles(inputs)
when {
incrementalChanges.isNotEmpty() -> {
changedFiles = incrementalChanges
classpathChanges = emptyList()
processIncrementally = true
}
else -> {
// do nothing
}
}
val args = prepareCompilerArguments()
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt
+11 -8
View File
@@ -12,10 +12,8 @@ import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.gradle.workers.IsolationMode
import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.gradle.incremental.ChangedFiles
import org.jetbrains.kotlin.gradle.internal.Kapt3KotlinGradleSubplugin.Companion.KAPT_WORKER_DEPENDENCIES_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.plugin.KotlinAndroidPluginWrapper
import org.jetbrains.kotlin.gradle.tasks.clearLocalState
import org.jetbrains.kotlin.gradle.tasks.findKotlinStdlibClasspath
import org.jetbrains.kotlin.gradle.tasks.findToolsJar
import org.jetbrains.kotlin.utils.PathUtil
@@ -52,6 +50,8 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor
logger.info("Running kapt annotation processing using the Gradle Worker API")
checkAnnotationProcessorClasspath()
val incrementalChanges = getChangedFiles(inputs)
val compileClasspath = classpath.files.toMutableList()
if (project.plugins.none { it is KotlinAndroidPluginWrapper }) {
compileClasspath.addAll(0, PathUtil.getJdkClassesRootsFromCurrentJre())
@@ -68,10 +68,11 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor
compileClasspath,
javaSourceRoots.toList(),
getChangedFiles(inputs),
incrementalChanges,
getCompiledSources(),
incAptCache,
classpathDirtyFqNamesHistoryDir.singleOrNull(),
emptyList(),
incrementalChanges.isNotEmpty(),
destinationDir,
classesDir,
@@ -94,7 +95,7 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor
workerExecutor.submit(KaptExecution::class.java) { config ->
val isolationModeStr = project.findProperty("kapt.workers.isolation") as String? ?: "none"
config.isolationMode = when(isolationModeStr.toLowerCase()) {
config.isolationMode = when (isolationModeStr.toLowerCase()) {
"process" -> IsolationMode.PROCESS
"none" -> IsolationMode.NONE
else -> IsolationMode.NONE
@@ -148,7 +149,7 @@ private class KaptExecution @Inject constructor(
}
}
private fun createKaptOptions(classLoader: ClassLoader) = with (optionsForWorker) {
private fun createKaptOptions(classLoader: ClassLoader) = with(optionsForWorker) {
val flags = kaptClass(classLoader).declaredMethods.single { it.name == "kaptFlags" }.invoke(null, flags)
val mode = Class.forName("org.jetbrains.kotlin.base.kapt3.AptMode", true, classLoader)
@@ -165,7 +166,8 @@ private class KaptExecution @Inject constructor(
changedFiles,
compiledSources,
incAptCache,
classpathFqNamesHistory,
classpathChanges,
processIncrementally,
sourcesOutputDir,
classesOutputDir,
@@ -201,7 +203,8 @@ private data class KaptOptionsForWorker(
val changedFiles: List<File>,
val compiledSources: List<File>,
val incAptCache: File?,
val classpathFqNamesHistory: File?,
val classpathChanges: List<String>,
val processIncrementally: Boolean,
val sourcesOutputDir: File,
val classesOutputDir: File,
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/subpluginUtils.kt
+13 -2
View File
@@ -44,13 +44,22 @@ fun encodePluginOptions(options: Map<String, List<String>>): String {
}
internal fun CompilerPluginOptions.withWrappedKaptOptions(
withApClasspath: Iterable<File>, changedFiles: List<File> = emptyList(), compiledSourcesDir: List<File> = emptyList()
withApClasspath: Iterable<File>,
changedFiles: List<File> = emptyList(),
classpathChanges: List<String> = emptyList(),
compiledSourcesDir: List<File> = emptyList(),
processIncrementally: Boolean = false
): CompilerPluginOptions {
val resultOptionsByPluginId: MutableMap<String, List<SubpluginOption>> =
subpluginOptionsByPluginId.toMutableMap()
resultOptionsByPluginId.compute(Kapt3KotlinGradleSubplugin.KAPT_SUBPLUGIN_ID) { _, kaptOptions ->
val changedFilesOption = FilesSubpluginOption("changedFile", changedFiles).takeIf { changedFiles.isNotEmpty() }
val classpathChangesOption = SubpluginOption(
"classpathChange",
classpathChanges.joinToString(separator = File.pathSeparator)
).takeIf { classpathChanges.isNotEmpty() }
val processIncrementallyOption = SubpluginOption("processIncrementally", processIncrementally.toString())
val compiledSourcesOption =
FilesSubpluginOption("compiledSourcesDir", compiledSourcesDir).takeIf { compiledSourcesDir.isNotEmpty() }
@@ -58,7 +67,9 @@ internal fun CompilerPluginOptions.withWrappedKaptOptions(
kaptOptions.orEmpty() +
withApClasspath.map { FilesSubpluginOption("apclasspath", listOf(it)) } +
changedFilesOption +
compiledSourcesOption
classpathChangesOption +
compiledSourcesOption +
processIncrementallyOption
wrapPluginOptions(kaptOptionsWithClasspath.filterNotNull(), "configuration")
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt
+1 -6
View File
@@ -435,8 +435,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
taskBuildDirectory,
usePreciseJavaTracking = usePreciseJavaTracking,
disableMultiModuleIC = disableMultiModuleIC(),
multiModuleICSettings = multiModuleICSettings,
classpathFqNamesHistory = getClasspathFqNamesHistoryDir()
multiModuleICSettings = multiModuleICSettings
)
} else null
@@ -479,10 +478,6 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
return false
}
@Optional
@Internal
internal open fun getClasspathFqNamesHistoryDir(): File? = null
// override setSource to track source directory sets and files (for generated android folders)
override fun setSource(sources: Any?) {
sourceRootsContainer.set(sources)
plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptContext.kt
+10 -5
View File
@@ -57,11 +57,16 @@ open class KaptContext(val options: KaptOptions, val withJdk: Boolean, val logge
KaptJavaCompiler.preRegister(context)
cacheManager = options.incrementalCache?.let {
JavaClassCacheManager(it, options.classpathFqNamesHistory!!)
JavaClassCacheManager(it)
}
if (options.processIncrementally) {
sourcesToReprocess =
cacheManager?.invalidateAndGetDirtyFiles(
options.changedFiles, options.classpathChanges
) ?: SourcesToReprocess.FullRebuild
} else {
sourcesToReprocess = SourcesToReprocess.FullRebuild
}
sourcesToReprocess = cacheManager?.invalidateAndGetDirtyFiles(
options.changedFiles.filter { it.extension == "java" }
) ?: SourcesToReprocess.FullRebuild
javacOptions = Options.instance(context).apply {
for ((key, value) in options.processingOptions) {
@@ -88,7 +93,7 @@ open class KaptContext(val options: KaptOptions, val withJdk: Boolean, val logge
put("accessInternalAPI", "true")
}
val compileClasspath = if (sourcesToReprocess is SourcesToReprocess.FullRebuild || options.changedFiles.isEmpty()) {
val compileClasspath = if (sourcesToReprocess is SourcesToReprocess.FullRebuild) {
options.compileClasspath
} else {
options.compileClasspath + options.compiledSources
plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt
+7 -4
View File
@@ -17,7 +17,8 @@ class KaptOptions(
val changedFiles: List<File>,
val compiledSources: List<File>,
val incrementalCache: File?,
val classpathFqNamesHistory: File?,
val classpathChanges: List<String>,
val processIncrementally: Boolean,
val sourcesOutputDir: File,
val classesOutputDir: File,
@@ -45,12 +46,13 @@ class KaptOptions(
val changedFiles: MutableList<File> = mutableListOf()
val compiledSources: MutableList<File> = mutableListOf()
var incrementalCache: File? = null
var classpathFqNamesHistory: File? = null
val classpathChanges: MutableList<String> = mutableListOf()
var sourcesOutputDir: File? = null
var classesOutputDir: File? = null
var stubsOutputDir: File? = null
var incrementalDataOutputDir: File? = null
var processIncrementally: Boolean = false
val processingClasspath: MutableList<File> = mutableListOf()
val processors: MutableList<String> = mutableListOf()
@@ -73,7 +75,7 @@ class KaptOptions(
return KaptOptions(
projectBaseDir, compileClasspath, javaSourceRoots,
changedFiles, compiledSources, incrementalCache, classpathFqNamesHistory,
changedFiles, compiledSources, incrementalCache, classpathChanges, processIncrementally,
sourcesOutputDir, classesOutputDir, stubsOutputDir, incrementalDataOutputDir,
processingClasspath, processors, processingOptions, javacOptions, KaptFlags.fromSet(flags),
mode, detectMemoryLeaks
@@ -169,5 +171,6 @@ fun KaptOptions.logString(additionalInfo: String = "") = buildString {
appendln("[incremental apt] Changed files: $changedFiles")
appendln("[incremental apt] Compiled sources directories: ${compiledSources.joinToString()}")
appendln("[incremental apt] Cache directory for incremental compilation: $incrementalCache")
appendln("[incremental apt] Classpath fq names history dir: $classpathFqNamesHistory")
appendln("[incremental apt] Changes classpath names: ${classpathChanges.joinToString()}")
appendln("[incremental apt] If processing incrementally: $processIncrementally")
}
plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/incremental/cache.kt
+23 -62
View File
@@ -8,7 +8,7 @@ package org.jetbrains.kotlin.kapt3.base.incremental
import java.io.*
// TODO(gavra): switch away from Java serialization
class JavaClassCacheManager(val file: File, private val classpathFqNamesHistory: File) : Closeable {
class JavaClassCacheManager(val file: File) : Closeable {
private val javaCacheFile = file.resolve("java-cache.bin")
internal val javaCache = maybeGetJavaCacheFromFile()
@@ -16,38 +16,8 @@ class JavaClassCacheManager(val file: File, private val classpathFqNamesHistory:
private val aptCacheFile = file.resolve("apt-cache.bin")
private val aptCache = maybeGetAptCacheFromFile()
private val lastBuildTimestamp = file.resolve("last-build-ts.bin")
private var closed = false
private fun getDirtyFqNamesFromClasspath(): ClasspathChanged {
if (!lastBuildTimestamp.exists()) return ClasspathChanged.FullRebuild
val lastTimestamp = lastBuildTimestamp.readText()
val (before, after) = classpathFqNamesHistory.listFiles().partition { it.name < lastTimestamp }
if (before.isEmpty()) {
return ClasspathChanged.FullRebuild
}
val dirtyFqNames = mutableSetOf<String>()
after.forEach { file ->
ObjectInputStream(file.inputStream().buffered()).use {
@Suppress("UNCHECKED_CAST")
dirtyFqNames.addAll(it.readObject() as Collection<String>)
}
}
return if (dirtyFqNames.isNotEmpty()) {
// TODO(gavra): We need to handle constants from classpath that might change between runs being incremental. One solution
// would be to fetch the changed symbols alongside changed fqNames, and to check if the symbol is a constant using ASM.
ClasspathChanged.FullRebuild
} else {
ClasspathChanged.Incremental(dirtyFqNames)
}
}
fun updateCache(processors: List<IncrementalProcessor>) {
if (!aptCache.updateCache(processors)) {
javaCache.invalidateAll()
@@ -58,39 +28,38 @@ class JavaClassCacheManager(val file: File, private val classpathFqNamesHistory:
* From set of changed sources, get list of files to recompile using structural information and dependency information from
* annotation processing.
*/
fun invalidateAndGetDirtyFiles(changedSources: Collection<File>): SourcesToReprocess {
fun invalidateAndGetDirtyFiles(changedSources: Collection<File>, dirtyClasspathJvmNames: Collection<String>): SourcesToReprocess {
if (!aptCache.isIncremental) {
return SourcesToReprocess.FullRebuild
}
val dirtyFqNamesFromClasspath = getDirtyFqNamesFromClasspath()
return when (dirtyFqNamesFromClasspath) {
is ClasspathChanged.FullRebuild -> SourcesToReprocess.FullRebuild
is ClasspathChanged.Incremental -> {
val changes = Changes(changedSources, dirtyFqNamesFromClasspath.dirtyFqNames)
val filesToReprocess = javaCache.invalidateEntriesForChangedFiles(changes)
val dirtyClasspathFqNames = HashSet<String>(dirtyClasspathJvmNames.size)
dirtyClasspathJvmNames.forEach {
dirtyClasspathFqNames.add(it.replace("$", ".").replace("/", "."))
}
when (filesToReprocess) {
is SourcesToReprocess.FullRebuild -> SourcesToReprocess.FullRebuild
is SourcesToReprocess.Incremental -> {
val toReprocess = filesToReprocess.toReprocess.toMutableSet()
val changes = Changes(changedSources, dirtyClasspathFqNames.toSet())
val filesToReprocess = javaCache.invalidateEntriesForChangedFiles(changes)
val isolatingGenerated = aptCache.invalidateIsolatingGenerated(toReprocess)
val generatedDirtyTypes = javaCache.invalidateGeneratedTypes(isolatingGenerated).toMutableSet()
return when (filesToReprocess) {
is SourcesToReprocess.FullRebuild -> SourcesToReprocess.FullRebuild
is SourcesToReprocess.Incremental -> {
val toReprocess = filesToReprocess.toReprocess.toMutableSet()
if (!toReprocess.isEmpty()) {
// only if there are some files to reprocess we should invalidate the aggregating ones
val aggregatingGenerated = aptCache.invalidateAggregating()
generatedDirtyTypes.addAll(javaCache.invalidateGeneratedTypes(aggregatingGenerated))
val isolatingGenerated = aptCache.invalidateIsolatingGenerated(toReprocess)
val generatedDirtyTypes = javaCache.invalidateGeneratedTypes(isolatingGenerated).toMutableSet()
toReprocess.addAll(
javaCache.invalidateEntriesAnnotatedWith(aptCache.getAggregatingClaimedAnnotations())
)
}
if (!toReprocess.isEmpty()) {
// only if there are some files to reprocess we should invalidate the aggregating ones
val aggregatingGenerated = aptCache.invalidateAggregating()
generatedDirtyTypes.addAll(javaCache.invalidateGeneratedTypes(aggregatingGenerated))
SourcesToReprocess.Incremental(toReprocess.toList(), generatedDirtyTypes)
}
toReprocess.addAll(
javaCache.invalidateEntriesAnnotatedWith(aptCache.getAggregatingClaimedAnnotations())
)
}
SourcesToReprocess.Incremental(toReprocess.toList(), generatedDirtyTypes)
}
}
}
@@ -144,9 +113,6 @@ class JavaClassCacheManager(val file: File, private val classpathFqNamesHistory:
}
}
with(lastBuildTimestamp) {
writeText(System.currentTimeMillis().toString())
}
closed = true
}
}
@@ -154,9 +120,4 @@ class JavaClassCacheManager(val file: File, private val classpathFqNamesHistory:
sealed class SourcesToReprocess {
class Incremental(val toReprocess: List<File>, val dirtyTypes: Set<String>) : SourcesToReprocess()
object FullRebuild : SourcesToReprocess()
}
sealed class ClasspathChanged {
class Incremental(val dirtyFqNames: Set<String>) : ClasspathChanged()
object FullRebuild : ClasspathChanged()
}
plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/IncrementalKaptTest.kt
+8 -17
View File
@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.base.kapt3.KaptOptions
import org.jetbrains.kotlin.base.kapt3.collectJavaSourceFiles
import org.jetbrains.kotlin.kapt3.base.KaptContext
import org.jetbrains.kotlin.kapt3.base.doAnnotationProcessing
import org.jetbrains.kotlin.kapt3.base.incremental.SourcesToReprocess
import org.jetbrains.kotlin.kapt3.base.util.WriterBackedKaptLogger
import org.junit.Assert.*
import org.junit.Rule
@@ -32,9 +33,6 @@ class IncrementalKaptTest {
val outputDir = tmp.newFolder()
val incrementalCacheDir = tmp.newFolder()
val classpathHistory = tmp.newFolder().also {
it.resolve("0").createNewFile()
}
val options = KaptOptions.Builder().apply {
projectBaseDir = tmp.newFolder()
javaSourceRoots.add(sourcesDir)
@@ -45,14 +43,12 @@ class IncrementalKaptTest {
incrementalDataOutputDir = outputDir
incrementalCache = incrementalCacheDir
classpathFqNamesHistory = classpathHistory
}.build()
val logger = WriterBackedKaptLogger(isVerbose = true)
KaptContext(options, true, logger).use {
val toReprocess = it.cacheManager!!.invalidateAndGetDirtyFiles(options.changedFiles)
it.doAnnotationProcessing(
options.collectJavaSourceFiles(toReprocess), listOf(SimpleProcessor().toIsolating())
options.collectJavaSourceFiles(SourcesToReprocess.FullRebuild), listOf(SimpleProcessor().toIsolating())
)
}
@@ -72,14 +68,14 @@ class IncrementalKaptTest {
incrementalDataOutputDir = outputDir
incrementalCache = incrementalCacheDir
classpathFqNamesHistory = classpathHistory
compiledSources.add(classesOutput)
changedFiles.add(sourcesDir.resolve("User.java"))
processIncrementally = true
}.build()
KaptContext(optionsForSecondRun, true, logger).use {
val sourcesToReprocess =
it.cacheManager!!.invalidateAndGetDirtyFiles(optionsForSecondRun.changedFiles)
it.cacheManager!!.invalidateAndGetDirtyFiles(optionsForSecondRun.changedFiles, emptyList())
assertFalse(outputDir.resolve("test/UserGenerated.java").exists())
it.doAnnotationProcessing(
@@ -92,7 +88,7 @@ class IncrementalKaptTest {
sourcesDir.resolve("User.java").delete()
KaptContext(optionsForSecondRun, true, logger).use {
val sourcesToReprocess = it.cacheManager!!.invalidateAndGetDirtyFiles(optionsForSecondRun.changedFiles)
val sourcesToReprocess = it.cacheManager!!.invalidateAndGetDirtyFiles(optionsForSecondRun.changedFiles, emptyList())
it.doAnnotationProcessing(
optionsForSecondRun.collectJavaSourceFiles(sourcesToReprocess), listOf(SimpleProcessor().toIsolating())
@@ -114,9 +110,6 @@ class IncrementalKaptTest {
val outputDir = tmp.newFolder()
val incrementalCacheDir = tmp.newFolder()
val classpathHistory = tmp.newFolder().also {
it.resolve("0").createNewFile()
}
val options = KaptOptions.Builder().apply {
projectBaseDir = tmp.newFolder()
javaSourceRoots.add(sourcesDir)
@@ -127,14 +120,12 @@ class IncrementalKaptTest {
incrementalDataOutputDir = outputDir
incrementalCache = incrementalCacheDir
classpathFqNamesHistory = classpathHistory
}.build()
val logger = WriterBackedKaptLogger(isVerbose = true)
KaptContext(options, true, logger).use {
val toReprocess = it.cacheManager!!.invalidateAndGetDirtyFiles(options.changedFiles)
it.doAnnotationProcessing(
options.collectJavaSourceFiles(toReprocess), listOf(SimpleGeneratingIfTypeDoesNotExist().toIsolating())
options.collectJavaSourceFiles(SourcesToReprocess.FullRebuild), listOf(SimpleGeneratingIfTypeDoesNotExist().toIsolating())
)
}
@@ -151,14 +142,14 @@ class IncrementalKaptTest {
incrementalDataOutputDir = outputDir
incrementalCache = incrementalCacheDir
classpathFqNamesHistory = classpathHistory
compiledSources.add(classesOutput)
changedFiles.add(sourcesDir.resolve("User.java"))
processIncrementally = true
}.build()
KaptContext(optionsForSecondRun, true, logger).use {
val sourcesToReprocess =
it.cacheManager!!.invalidateAndGetDirtyFiles(optionsForSecondRun.changedFiles)
it.cacheManager!!.invalidateAndGetDirtyFiles(optionsForSecondRun.changedFiles, emptyList())
assertFalse(outputDir.resolve("test/UserGenerated.java").exists())
it.doAnnotationProcessing(
plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/InheritedAnnotationTest.kt
+1 -1
View File
@@ -30,7 +30,7 @@ class TestInheritedAnnotation {
@BeforeClass
fun setUp() {
val classpathHistory = tmp.newFolder()
cache = JavaClassCacheManager(tmp.newFolder(), classpathHistory)
cache = JavaClassCacheManager(tmp.newFolder())
generatedSources = tmp.newFolder()
cache.close()
classpathHistory.resolve("0").createNewFile()
plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/JavaClassCacheManagerTest.kt
+9 -45
View File
@@ -22,13 +22,11 @@ class JavaClassCacheManagerTest {
private lateinit var cache: JavaClassCacheManager
private lateinit var cacheDir: File
private lateinit var classpathHistory: File
@Before
fun setUp() {
cacheDir = tmp.newFolder()
classpathHistory = tmp.newFolder()
cache = JavaClassCacheManager(cacheDir, classpathHistory)
cache = JavaClassCacheManager(cacheDir)
}
@@ -38,7 +36,6 @@ class JavaClassCacheManagerTest {
assertTrue(cacheDir.resolve("java-cache.bin").exists())
assertTrue(cacheDir.resolve("apt-cache.bin").exists())
assertTrue(cacheDir.resolve("last-build-ts.bin").exists())
}
@Test
@@ -59,7 +56,7 @@ class JavaClassCacheManagerTest {
}
prepareForIncremental()
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("Mentioned.java"))) as SourcesToReprocess.Incremental
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("Mentioned.java")), emptyList()) as SourcesToReprocess.Incremental
assertEquals(
listOf(
File("Mentioned.java").absoluteFile,
@@ -87,7 +84,7 @@ class JavaClassCacheManagerTest {
}
prepareForIncremental()
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("Mentioned.java"))) as SourcesToReprocess.Incremental
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("Mentioned.java")), emptyList()) as SourcesToReprocess.Incremental
assertEquals(
listOf(
File("Mentioned.java").absoluteFile,
@@ -115,7 +112,7 @@ class JavaClassCacheManagerTest {
}
prepareForIncremental()
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("TwoTypes.java"))) as SourcesToReprocess.Incremental
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("TwoTypes.java")), emptyList()) as SourcesToReprocess.Incremental
assertEquals(
listOf(
File("TwoTypes.java").absoluteFile,
@@ -126,48 +123,16 @@ class JavaClassCacheManagerTest {
}
@Test
fun testNoClasspathHistory() {
SourceFileStructure(File("Src.java").toURI()).also {
it.addDeclaredType("test.Src")
cache.javaCache.addSourceStructure(it)
}
cache.close()
classpathHistory.resolve(Long.MAX_VALUE.toString()).createNewFile()
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf())
assertEquals(SourcesToReprocess.FullRebuild, dirtyFiles)
}
@Test
fun testWithClasspathHistoryButNoNewChanges() {
fun testWithClasspathChanges() {
SourceFileStructure(File("Src.java").toURI()).also {
it.addDeclaredType("test.Src")
it.addMentionedType("test.Mentioned")
cache.javaCache.addSourceStructure(it)
}
prepareForIncremental()
ObjectOutputStream(classpathHistory.resolve("0").outputStream()).use {
it.writeObject(listOf("test.Mentioned"))
}
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf()) as SourcesToReprocess.Incremental
assertEquals(emptyList<File>(), dirtyFiles.toReprocess)
}
@Test
fun testWithClasspathHistoryWithChanges() {
SourceFileStructure(File("Src.java").toURI()).also {
it.addDeclaredType("test.Src")
it.addMentionedType("test.Mentioned")
cache.javaCache.addSourceStructure(it)
}
prepareForIncremental()
ObjectOutputStream(classpathHistory.resolve(Long.MAX_VALUE.toString()).outputStream()).use {
it.writeObject(listOf("test.Mentioned"))
}
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf())
assertEquals(SourcesToReprocess.FullRebuild, dirtyFiles)
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(), listOf("test/Mentioned")) as SourcesToReprocess.Incremental
assertEquals(listOf(File("Src.java").absoluteFile), dirtyFiles.toReprocess)
}
@Test
@@ -187,7 +152,7 @@ class JavaClassCacheManagerTest {
}
prepareForIncremental()
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("Constants.java")))
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(File("Constants.java")), emptyList())
assertEquals(SourcesToReprocess.FullRebuild, dirtyFiles)
}
@@ -217,7 +182,6 @@ class JavaClassCacheManagerTest {
private fun prepareForIncremental() {
cache.close()
classpathHistory.resolve("0").createNewFile()
cache = JavaClassCacheManager(cacheDir, classpathHistory)
cache = JavaClassCacheManager(cacheDir)
}
}
plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/ReferencedContantsTest.kt
+1 -3
View File
@@ -32,11 +32,9 @@ class ReferencedConstantsTest {
val compiledClasses = tmp.newFolder()
compileSources(listOf(MY_TEST_DIR.resolve("CKlass.java")), compiledClasses)
val classpathHistory = tmp.newFolder()
cache = JavaClassCacheManager(tmp.newFolder(), classpathHistory)
cache = JavaClassCacheManager(tmp.newFolder())
generatedSources = tmp.newFolder()
cache.close()
classpathHistory.resolve("0").createNewFile()
val processor = SimpleProcessor().toAggregating()
val srcFiles = listOf(
"A.java",
plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/TestComplexIncrementalAptCache.kt
+1 -3
View File
@@ -29,11 +29,9 @@ class TestComplexIncrementalAptCache {
@JvmStatic
@BeforeClass
fun setUp() {
val classpathHistory = tmp.newFolder()
cache = JavaClassCacheManager(tmp.newFolder(), classpathHistory)
cache = JavaClassCacheManager(tmp.newFolder())
generatedSources = tmp.newFolder()
cache.close()
classpathHistory.resolve("0").createNewFile()
val processor = SimpleProcessor().toAggregating()
val srcFiles = listOf(
"MyEnum.java",
plugins/kapt3/kapt3-base/test/org/jetbrains/kotlin/kapt3/base/incremental/TestSimpleIncrementalAptCache.kt
+4 -6
View File
@@ -27,18 +27,16 @@ class TestSimpleIncrementalAptCache {
@Before
fun setUp() {
val classpathHistory = tmp.newFolder()
cache = JavaClassCacheManager(tmp.newFolder(), classpathHistory)
cache = JavaClassCacheManager(tmp.newFolder())
generatedSources = tmp.newFolder()
cache.close()
classpathHistory.resolve("0").createNewFile()
}
@Test
fun testAggregatingAnnotations() {
runProcessor(SimpleProcessor().toAggregating())
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(TEST_DATA_DIR.resolve("User.java"))) as SourcesToReprocess.Incremental
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(TEST_DATA_DIR.resolve("User.java")), emptyList()) as SourcesToReprocess.Incremental
assertEquals(
listOf(TEST_DATA_DIR.resolve("User.java").absoluteFile, TEST_DATA_DIR.resolve("Address.java").absoluteFile),
dirtyFiles.toReprocess
@@ -51,7 +49,7 @@ class TestSimpleIncrementalAptCache {
fun testIsolatingAnnotations() {
runProcessor(SimpleProcessor().toIsolating())
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(TEST_DATA_DIR.resolve("User.java"))) as SourcesToReprocess.Incremental
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(TEST_DATA_DIR.resolve("User.java")), emptyList()) as SourcesToReprocess.Incremental
assertFalse(generatedSources.resolve("test/UserGenerated.java").exists())
assertEquals(
listOf(TEST_DATA_DIR.resolve("User.java").absoluteFile),
@@ -63,7 +61,7 @@ class TestSimpleIncrementalAptCache {
fun testNonIncremental() {
runProcessor(SimpleProcessor().toNonIncremental())
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(TEST_DATA_DIR.resolve("User.java")))
val dirtyFiles = cache.invalidateAndGetDirtyFiles(listOf(TEST_DATA_DIR.resolve("User.java")), emptyList())
assertTrue(dirtyFiles is SourcesToReprocess.FullRebuild)
}
plugins/kapt3/kapt3-cli/src/KaptCliOption.kt
+10 -4
View File
@@ -90,10 +90,16 @@ enum class KaptCliOption(
"Use only in apt mode. Output directory for cache necessary to support incremental annotation processing."
),
CLASSPATH_FQ_NAMES_HISTORY(
"classpathFqNamesHistory",
"<path>",
"Use only in apt mode. Directory containing history of classpath fq name changes."
CLASSPATH_CHANGES(
"classpathChange",
"<jvmInternalName,[jvmInternalName,...]>",
"Use only in apt mode. Classpath jvm internal names that changed."
),
PROCESS_INCREMENTALLY(
"processIncrementally",
"boolean",
"Use only in apt mode. Enables incremental apt processing"
),
ANNOTATION_PROCESSOR_CLASSPATH_OPTION(
plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt
+2 -1
View File
@@ -100,7 +100,8 @@ class Kapt3CommandLineProcessor : CommandLineProcessor {
CHANGED_FILES -> changedFiles.addAll(value.split(File.pathSeparator).map { File(it) })
COMPILED_SOURCES_DIR -> compiledSources.addAll(value.split(File.pathSeparator).map { File(it) })
INCREMENTAL_CACHE -> incrementalCache = File(value)
CLASSPATH_FQ_NAMES_HISTORY -> classpathFqNamesHistory = File(value)
CLASSPATH_CHANGES -> classpathChanges.addAll(value.split(File.pathSeparator).map { it })
PROCESS_INCREMENTALLY -> processIncrementally = value.toBoolean()
ANNOTATION_PROCESSOR_CLASSPATH_OPTION -> processingClasspath += File(value)
ANNOTATION_PROCESSORS_OPTION -> processors.addAll(value.split(',').map { it.trim() }.filter { it.isNotEmpty() })