Implement multiproject IC in Gradle

#KT-13528 fixed
This commit is contained in:
Alexey Tsvetkov
2016-08-11 16:05:54 +03:00
parent f967edfaeb
commit 4e8deb9566
36 changed files with 764 additions and 97 deletions
@@ -175,8 +175,8 @@ fun<Target> OutputItemsCollectorImpl.generatedFiles(
}
data class DirtyData(
val dirtyLookupSymbols: Iterable<LookupSymbol>,
val dirtyClassesFqNames: Iterable<FqName>
val dirtyLookupSymbols: Collection<LookupSymbol> = emptyList(),
val dirtyClassesFqNames: Collection<FqName> = emptyList()
)
fun <Target> CompilationResult.getDirtyData(
@@ -261,7 +261,7 @@ private fun File.isJavaFile() = extension.equals(JavaFileType.INSTANCE.defaultEx
private fun findSrcDirRoot(file: File, roots: Iterable<File>): File? =
roots.firstOrNull { FileUtil.isAncestor(it, file, false) }
private fun <Target> withSubtypes(
fun <Target> withSubtypes(
typeFqName: FqName,
caches: Iterable<IncrementalCacheImpl<Target>>
): Set<FqName> {
@@ -0,0 +1,12 @@
package org.jetbrains.kotlin.gradle.tasks
import org.jetbrains.kotlin.incremental.DirtyData
import java.io.File
interface ArtifactDifferenceRegistry {
operator fun get(artifact: File): Iterable<ArtifactDifference>?
fun add(artifact: File, difference: ArtifactDifference)
fun remove(artifact: File)
}
class ArtifactDifference(val buildTS: Long, val dirtyData: DirtyData)
@@ -0,0 +1,23 @@
package org.jetbrains.kotlin.gradle.tasks
import java.io.*
internal data class BuildInfo(val startTS: Long) : Serializable {
companion object {
fun read(file: File): BuildInfo? =
try {
ObjectInputStream(FileInputStream(file)).use {
it.readObject() as BuildInfo
}
}
catch (e: Exception) {
null
}
fun write(buildInfo: BuildInfo, file: File) {
ObjectOutputStream(FileOutputStream(file)).use {
it.writeObject(buildInfo)
}
}
}
}
@@ -0,0 +1,8 @@
package org.jetbrains.kotlin.gradle.tasks
import org.jetbrains.kotlin.incremental.LookupSymbol
internal sealed class ChangesEither {
internal class Known(val lookupSymbols: Set<LookupSymbol>) : ChangesEither()
internal class Unknown : ChangesEither()
}
@@ -3,6 +3,7 @@ package org.jetbrains.kotlin.gradle.tasks
import org.apache.commons.io.FilenameUtils
import org.codehaus.groovy.runtime.MethodClosure
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.file.SourceDirectorySet
import org.gradle.api.logging.Logger
import org.gradle.api.logging.Logging
@@ -42,6 +43,7 @@ import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
import org.jetbrains.kotlin.utils.LibraryUtils
import java.io.File
@@ -52,6 +54,7 @@ const val ANNOTATIONS_PLUGIN_NAME = "org.jetbrains.kotlin.kapt"
const val KOTLIN_BUILD_DIR_NAME = "kotlin"
const val CACHES_DIR_NAME = "caches"
const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
const val USING_EXPERIMENTAL_INCREMENTAL_MESSAGE = "Using experimental kotlin incremental compilation"
abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractCompile() {
@@ -154,6 +157,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
private val taskBuildDirectory: File by lazy { File(File(project.buildDir, KOTLIN_BUILD_DIR_NAME), name) }
private val cacheDirectory: File by lazy { File(taskBuildDirectory, CACHES_DIR_NAME) }
private val dirtySourcesSinceLastTimeFile: File by lazy { File(taskBuildDirectory, DIRTY_SOURCES_FILE_NAME) }
private val lastBuildInfoFile: File by lazy { File(taskBuildDirectory, LAST_BUILD_INFO_FILE_NAME) }
private val cacheVersions by lazy {
listOf(normalCacheVersion(taskBuildDirectory),
experimentalCacheVersion(taskBuildDirectory),
@@ -172,6 +176,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
val kaptOptions = KaptOptions()
val pluginOptions = CompilerPluginOptions()
var artifactDifferenceRegistry: ArtifactDifferenceRegistry? = null
override fun populateTargetSpecificArgs(args: K2JVMCompilerArguments) {
logger.kotlinDebug("args.freeArgs = ${args.freeArgs}")
@@ -232,15 +237,11 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
}
override fun callCompiler(args: K2JVMCompilerArguments, sources: List<File>, isIncrementalRequested: Boolean, modified: List<File>, removed: List<File>) {
val rootProjectDir = project.rootProject.projectDir
fun projectRelativePath(f: File) = f.toRelativeString(project.projectDir)
if (incremental) {
// TODO: consider other ways to pass incremental flag to compiler/builder
System.setProperty("kotlin.incremental.compilation", "true")
// TODO: experimental should be removed as soon as it becomes standard
System.setProperty("kotlin.incremental.compilation.experimental", "true")
}
fun projectRelativePath(f: File) = f.toRelativeString(rootProjectDir)
fun filesToString(files: Iterable<File>) =
"[" + files.map(::projectRelativePath).sorted().joinToString(separator = ", \n") + "]"
val targetType = "java-production"
val moduleName = args.moduleName
@@ -252,6 +253,8 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
var currentRemoved = removed.filter { it.isKotlinFile() }
val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
val logAction = { logStr: String -> logger.kotlinInfo(logStr) }
val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
logger.kotlinDebug { "Last Kotlin Build info for task $path -- $lastBuildInfo" }
fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl {
val cacheDir = File(cacheDirectory, "increCache.${target.name}")
@@ -261,68 +264,117 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
fun getIncrementalCache(it: TargetId) = caches.getOrPut(it, { getOrCreateIncrementalCache(it) })
fun PsiClass.findLookupSymbols(): Iterable<LookupSymbol> {
fun PsiClass.addLookupSymbols(symbols: MutableSet<LookupSymbol>) {
val fqn = qualifiedName.orEmpty()
return listOf(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!))) +
methods.map { LookupSymbol(it.name, fqn) } +
fields.map { LookupSymbol(it.name.orEmpty(), fqn) } +
innerClasses.flatMap { it.findLookupSymbols() }
symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!)))
methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) }
fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) }
innerClasses.forEach { it.addLookupSymbols(symbols) }
}
fun dirtyLookupSymbolsFromModifiedJavaFiles(): List<LookupSymbol> {
val dirtyJavaLookupSymbols: Lazy<Collection<LookupSymbol>> = lazy {
val symbols = HashSet<LookupSymbol>()
val modifiedJavaFiles = modified.filter { it.isJavaFile() }
return (if (modifiedJavaFiles.any()) {
val rootDisposable = Disposer.newDisposable()
val configuration = CompilerConfiguration()
val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
val project = environment.project
val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
modifiedJavaFiles.flatMap {
val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText())
if (javaFile is PsiJavaFile)
javaFile.classes.flatMap { it.findLookupSymbols() }
else listOf()
if (modifiedJavaFiles.isEmpty()) return@lazy symbols
val rootDisposable = Disposer.newDisposable()
val configuration = CompilerConfiguration()
val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
val project = environment.project
val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
modifiedJavaFiles.forEach {
val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText())
if (javaFile is PsiJavaFile) {
javaFile.classes.forEach { it.addLookupSymbols(symbols) }
}
} else listOf())
}
symbols
}
fun dirtyKotlinSourcesFromGradle(): MutableSet<File> {
// TODO: handle classpath changes similarly - compare with cashed version (likely a big change, may be costly, some heuristics could be considered)
val modifiedKotlinFiles = modified.filter { it.isKotlinFile() }.toMutableSet()
val lookupSymbols = dirtyLookupSymbolsFromModifiedJavaFiles()
// TODO: add dirty lookups from modified kotlin files to reduce number of steps needed
if (lookupSymbols.any()) {
val kotlinModifiedFilesSet = modifiedKotlinFiles.toHashSet()
val dirtyFilesFromLookups = mapLookupSymbolsToFiles(lookupStorage, lookupSymbols, logAction, ::projectRelativePath, excludes = kotlinModifiedFilesSet)
modifiedKotlinFiles.addAll(dirtyFilesFromLookups)
fun getClasspathChanges(modifiedClasspath: List<File>): DirtyData? {
if (modifiedClasspath.isEmpty()) {
logger.kotlinDebug { "No classpath changes" }
return DirtyData()
}
if (artifactDifferenceRegistry == null) {
logger.kotlinDebug { "No artifact history provider" }
return null
}
return modifiedKotlinFiles
val lastBuildTS = lastBuildInfo?.startTS
if (lastBuildTS == null) {
logger.kotlinDebug { "Could not determine last build timestamp" }
return null
}
val symbols = HashSet<LookupSymbol>()
val fqNames = HashSet<FqName>()
for (file in modifiedClasspath) {
val diffs = artifactDifferenceRegistry!![file]
if (diffs == null) {
logger.kotlinDebug { "Could not get changes for file: $file" }
return null
}
val (beforeLastBuild, afterLastBuild) = diffs.partition { it.buildTS < lastBuildTS }
if (beforeLastBuild.isEmpty()) {
logger.kotlinDebug { "No known build preceding timestamp $lastBuildTS for file $file" }
return null
}
afterLastBuild.forEach {
symbols.addAll(it.dirtyData.dirtyLookupSymbols)
fqNames.addAll(it.dirtyData.dirtyClassesFqNames)
}
}
return DirtyData(symbols, fqNames)
}
fun isClassPathChanged(): Boolean =
modified.any { classpath.contains(it) }
fun calculateSourcesToCompile(): Pair<Set<File>, Boolean> {
if (!incremental
|| !isIncrementalRequested
// TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes
|| isClassPathChanged()
// so far considering it not incremental TODO: store java files in the cache and extract removed symbols from it here
|| removed.any { it.isJavaFile() || it.hasClassFileExtension() }
|| modified.any { it.hasClassFileExtension() }
) {
logger.kotlinInfo(if (!isIncrementalRequested) "clean caches on rebuild" else "classpath changed, rebuilding all kotlin files")
fun rebuild(reason: String): Pair<Set<File>, Boolean> {
logger.kotlinInfo("Non-incremental compilation will be performed: $reason")
targets.forEach { getIncrementalCache(it).clean() }
lookupStorage.clean()
dirtySourcesSinceLastTimeFile.delete()
return Pair(sources.toSet(), false)
}
val dirtyFiles = dirtyKotlinSourcesFromGradle()
if (!incremental) return rebuild("incremental compilation is not enabled")
if (!isIncrementalRequested) return rebuild("inputs' changes are unknown (first or clean build)")
// TODO: store java files in the cache and extract removed symbols from it here
val illegalRemovedFiles = removed.filter { it.isJavaFile() || it.hasClassFileExtension() }
if (illegalRemovedFiles.any()) return rebuild("unsupported removed files: ${filesToString(illegalRemovedFiles)}")
val illegalModifiedFiles = modified.filter(File::hasClassFileExtension)
if (illegalModifiedFiles.any()) return rebuild("unsupported modified files: ${filesToString(illegalModifiedFiles)}")
val modifiedClasspathEntries = modified.filter { it in classpath }
val classpathChanges = getClasspathChanges(modifiedClasspathEntries)
?: return rebuild("could not get changes from modified classpath entries: ${filesToString(modifiedClasspathEntries)}")
val dirtyFiles = modified.filter { it.isKotlinFile() }.toMutableSet()
val lookupSymbols = HashSet<LookupSymbol>()
lookupSymbols.addAll(dirtyJavaLookupSymbols.value)
lookupSymbols.addAll(classpathChanges.dirtyLookupSymbols)
if (lookupSymbols.any()) {
val dirtyFilesFromLookups = mapLookupSymbolsToFiles(lookupStorage, lookupSymbols, logAction, ::projectRelativePath)
dirtyFiles.addAll(dirtyFilesFromLookups)
}
val allCaches = targets.map(::getIncrementalCache)
val dirtyClassesFqNames = classpathChanges.dirtyClassesFqNames.flatMap { withSubtypes(it, allCaches) }
if (dirtyClassesFqNames.any()) {
val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(allCaches, dirtyClassesFqNames, logAction, ::projectRelativePath)
dirtyFiles.addAll(dirtyFilesFromFqNames)
}
if (dirtySourcesSinceLastTimeFile.exists()) {
val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter { it.exists() }
val files = dirtySourcesSinceLastTimeFile.readLines().map(::File).filter(File::exists)
if (files.isNotEmpty()) {
logger.kotlinDebug { "Source files added since last compilation: $files" }
}
@@ -334,9 +386,8 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
}
fun cleanupOnError() {
val filesToDelete = allGeneratedFiles.map { it.outputFile }
logger.kotlinInfo("deleting output on error: ${filesToDelete.joinToString()}")
filesToDelete.forEach { it.delete() }
logger.kotlinInfo("deleting $destinationDir on error")
destinationDir.deleteRecursively()
}
fun processCompilerExitCode(exitCode: ExitCode) {
@@ -379,6 +430,13 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
kaptAnnotationsFileUpdater = null
}
val artifactFile = project.tryGetSingleArtifact()
logger.kotlinDebug { "Artifact to register difference for task $path: $artifactFile" }
val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
val buildDirtyFqNames = HashSet<FqName>()
var exitCode = ExitCode.OK
while (sourcesToCompile.any() || currentRemoved.any()) {
val removedAndModified = (sourcesToCompile + currentRemoved).toList()
@@ -419,18 +477,38 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved)
if (!isIncrementalDecided) break
if (!isIncrementalDecided) {
artifactFile?.let { artifactDifferenceRegistry?.remove(it) }
break
}
val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(caches.values, logAction)
sourcesToCompile = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile) +
mapClassesFqNamesToFiles(caches.values, dirtyClassFqNames, logAction, ::projectRelativePath, excludes = sourcesToCompile)
buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
buildDirtyFqNames.addAll(dirtyClassFqNames)
if (currentRemoved.any()) {
anyClassesCompiled = true
currentRemoved = listOf()
}
}
if (exitCode == ExitCode.OK) {
buildDirtyLookupSymbols.addAll(dirtyJavaLookupSymbols.value)
}
if (artifactFile != null && artifactDifferenceRegistry != null) {
val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
val artifactDifference = ArtifactDifference(currentBuildInfo.startTS, dirtyData)
artifactDifferenceRegistry!!.add(artifactFile, artifactDifference)
logger.kotlinDebug {
val dirtySymbolsSorted = buildDirtyLookupSymbols.map { it.scope + "#" + it.name }.sorted()
"Added artifact difference for $artifactFile (ts: ${currentBuildInfo.startTS}): " +
"[\n\t${dirtySymbolsSorted.joinToString(",\n\t")}]"
}
}
anyClassesCompiled = anyClassesCompiled || allGeneratedFiles.isNotEmpty()
processCompilerExitCode(exitCode)
}
@@ -719,6 +797,22 @@ class GradleMessageCollector(val logger: Logger, val outputCollector: OutputItem
}
}
fun Project.tryGetSingleArtifact(): File? {
val log = logger
log.kotlinDebug { "Trying to determine single artifact for project $path" }
val archives = configurations.findByName("archives")
if (archives == null) {
log.kotlinDebug { "Could not find 'archives' configuration for project $path" }
return null
}
val artifacts = archives.artifacts.files.files
log.kotlinDebug { "All artifacts for project $path: [${artifacts.joinToString()}]" }
return if (artifacts.size == 1) artifacts.first() else null
}
internal fun Logger.kotlinInfo(message: String) {
this.info("[KOTLIN] $message")
}
@@ -26,6 +26,9 @@ import org.jetbrains.kotlin.com.intellij.openapi.util.io.ZipFileCache
import org.jetbrains.kotlin.com.intellij.openapi.vfs.impl.ZipHandler
import org.jetbrains.kotlin.com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
import org.jetbrains.kotlin.gradle.tasks.ArtifactDifferenceRegistry
import org.jetbrains.kotlin.gradle.tasks.incremental.BuildCacheStorage
import java.io.File
private fun comparableVersionStr(version: String) =
"(\\d+)\\.(\\d+).*"
@@ -37,31 +40,44 @@ private fun comparableVersionStr(version: String) =
?.let { if (it.all { (it?.value?.length ?: 0).let { it > 0 && it < 4 }}) it else null }
?.joinToString(".", transform = { it!!.value.padStart(3, '0') })
class KotlinGradleBuildServices private constructor(): BuildAdapter() {
class KotlinGradleBuildServices private constructor(gradle: Gradle): BuildAdapter() {
companion object {
private val CLASS_NAME = KotlinGradleBuildServices::class.java.simpleName
const val FORCE_SYSTEM_GC_MESSAGE = "Forcing System.gc()"
const val INIT_MESSAGE = "Initialized KotlinGradleBuildServices"
const val DISPOSE_MESSAGE = "Disposed KotlinGradleBuildServices"
private var initialized = false
val INIT_MESSAGE = "Initialized $CLASS_NAME"
val DISPOSE_MESSAGE = "Disposed $CLASS_NAME"
val ALREADY_INITIALIZED_MESSAGE = "$CLASS_NAME is already initialized"
@field:Volatile
private var instance: KotlinGradleBuildServices? = null
@JvmStatic
@Synchronized
fun init(gradle: Gradle) {
if (initialized) return
val listener = KotlinGradleBuildServices()
gradle.addBuildListener(listener)
initialized = true
fun getInstance(gradle: Gradle): KotlinGradleBuildServices {
val log = Logging.getLogger(KotlinGradleBuildServices::class.java)
if (instance != null) {
log.kotlinDebug(ALREADY_INITIALIZED_MESSAGE)
return instance!!
}
val services = KotlinGradleBuildServices(gradle)
gradle.addBuildListener(services)
instance = services
log.kotlinDebug(INIT_MESSAGE)
listener.buildStarted()
services.buildStarted()
return services
}
}
private val log = Logging.getLogger(this.javaClass)
private val cleanup = CompilerServicesCleanup()
private var startMemory: Long? = null
private val workingDir = File(gradle.rootProject.buildDir, "kotlin-build").apply { mkdirs() }
private val buildCacheStorage = BuildCacheStorage(workingDir)
internal val artifactDifferenceRegistry: ArtifactDifferenceRegistry
get() = buildCacheStorage.artifactDifferenceRegistry
// There is function with the same name in BuildAdapter,
// but it is called before any plugin can attach build listener
@@ -93,8 +109,11 @@ class KotlinGradleBuildServices private constructor(): BuildAdapter() {
}
}
buildCacheStorage.flush(memoryCachesOnly = false)
buildCacheStorage.close()
gradle.removeListener(this)
initialized = false
instance = null
log.kotlinDebug(DISPOSE_MESSAGE)
}
@@ -25,10 +25,8 @@ import org.jetbrains.kotlin.gradle.internal.Kapt2GradleSubplugin
import org.jetbrains.kotlin.gradle.internal.Kapt2KotlinGradleSubplugin
import org.jetbrains.kotlin.gradle.internal.initKapt
import org.jetbrains.kotlin.gradle.plugin.android.AndroidGradleWrapper
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.tasks.KotlinTasksProvider
import org.jetbrains.kotlin.gradle.tasks.SyncOutputTask
import org.jetbrains.kotlin.gradle.tasks.*
import org.jetbrains.kotlin.gradle.tasks.incremental.configureMultiProjectIncrementalCompilation
import java.io.File
import java.net.URL
import java.util.*
@@ -98,18 +96,14 @@ class Kotlin2JvmSourceSetProcessor(
sourceSet: SourceSet,
tasksProvider: KotlinTasksProvider,
kotlinSourceSetProvider: KotlinSourceSetProvider,
private val kotlinPluginVersion: String
private val kotlinPluginVersion: String,
private val artifactDifferenceRegistry: ArtifactDifferenceRegistry
) : KotlinSourceSetProcessor<KotlinCompile>(
project, javaBasePlugin, sourceSet, tasksProvider, kotlinSourceSetProvider,
dslExtensionName = KOTLIN_DSL_NAME,
compileTaskNameSuffix = "kotlin",
taskDescription = "Compiles the $sourceSet.kotlin."
) {
private companion object {
private var cachedKotlinAnnotationProcessingDep: String? = null
}
override val defaultKotlinDestinationDir: File
get() = File(project.buildDir, "kotlin-classes/$sourceSetName")
@@ -147,6 +141,7 @@ class Kotlin2JvmSourceSetProcessor(
kotlinAfterJavaTask?.let { it.source(kotlinSourceSet.kotlin) }
configureJavaTask(kotlinTask, javaTask, logger)
createSyncOutputTask(project, kotlinTask, javaTask, kotlinAfterJavaTask, sourceSetName)
configureMultiProjectIncrementalCompilation(project, kotlinTask, javaTask, kotlinAfterJavaTask, artifactDifferenceRegistry)
}
}
}
@@ -222,10 +217,11 @@ abstract class AbstractKotlinPlugin(val tasksProvider: KotlinTasksProvider, val
open class KotlinPlugin(
tasksProvider: KotlinTasksProvider,
kotlinSourceSetProvider: KotlinSourceSetProvider,
kotlinPluginVersion: String
kotlinPluginVersion: String,
private val artifactDifferenceRegistry: ArtifactDifferenceRegistry
) : AbstractKotlinPlugin(tasksProvider, kotlinSourceSetProvider, kotlinPluginVersion) {
override fun buildSourceSetProcessor(project: Project, javaBasePlugin: JavaBasePlugin, sourceSet: SourceSet, kotlinPluginVersion: String) =
Kotlin2JvmSourceSetProcessor(project, javaBasePlugin, sourceSet, tasksProvider, kotlinSourceSetProvider, kotlinPluginVersion)
Kotlin2JvmSourceSetProcessor(project, javaBasePlugin, sourceSet, tasksProvider, kotlinSourceSetProvider, kotlinPluginVersion, artifactDifferenceRegistry)
override fun apply(project: Project) {
project.createKaptExtension()
@@ -19,28 +19,28 @@ abstract class KotlinBasePluginWrapper(protected val fileResolver: FileResolver)
override fun apply(project: Project) {
// TODO: consider only set if if daemon or parallel compilation are enabled, though this way it should be safe too
System.setProperty(org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true")
val kotlinGradleBuildServices = KotlinGradleBuildServices.getInstance(project.gradle)
val plugin = getPlugin()
val plugin = getPlugin(kotlinGradleBuildServices)
plugin.apply(project)
KotlinGradleBuildServices.init(project.gradle)
}
protected abstract fun getPlugin(): Plugin<Project>
protected abstract fun getPlugin(kotlinGradleBuildServices: KotlinGradleBuildServices): Plugin<Project>
}
open class KotlinPluginWrapper @Inject constructor(fileResolver: FileResolver): KotlinBasePluginWrapper(fileResolver) {
override fun getPlugin() =
KotlinPlugin(KotlinTasksProvider(), KotlinSourceSetProviderImpl(fileResolver), kotlinPluginVersion)
override fun getPlugin(kotlinGradleBuildServices: KotlinGradleBuildServices) =
KotlinPlugin(KotlinTasksProvider(), KotlinSourceSetProviderImpl(fileResolver), kotlinPluginVersion,
kotlinGradleBuildServices.artifactDifferenceRegistry)
}
open class KotlinAndroidPluginWrapper @Inject constructor(fileResolver: FileResolver): KotlinBasePluginWrapper(fileResolver) {
override fun getPlugin() =
override fun getPlugin(kotlinGradleBuildServices: KotlinGradleBuildServices) =
KotlinAndroidPlugin(AndroidTasksProvider(), KotlinSourceSetProviderImpl(fileResolver), kotlinPluginVersion)
}
open class Kotlin2JsPluginWrapper @Inject constructor(fileResolver: FileResolver): KotlinBasePluginWrapper(fileResolver) {
override fun getPlugin() =
override fun getPlugin(kotlinGradleBuildServices: KotlinGradleBuildServices) =
Kotlin2JsPlugin(KotlinTasksProvider(), KotlinSourceSetProviderImpl(fileResolver), kotlinPluginVersion)
}
@@ -0,0 +1,102 @@
package org.jetbrains.kotlin.gradle.tasks.incremental
import org.jetbrains.kotlin.com.intellij.util.io.DataExternalizer
import org.jetbrains.kotlin.gradle.tasks.ArtifactDifference
import org.jetbrains.kotlin.gradle.tasks.ArtifactDifferenceRegistry
import org.jetbrains.kotlin.incremental.DirtyData
import org.jetbrains.kotlin.incremental.LookupSymbol
import org.jetbrains.kotlin.incremental.storage.BasicStringMap
import org.jetbrains.kotlin.name.FqName
import java.io.DataInput
import java.io.DataOutput
import java.io.File
import java.util.*
internal class ArtifactDifferenceRegistryImpl(
storageFile: File
) : ArtifactDifferenceRegistry, BasicStringMap<Collection<ArtifactDifference>>(
storageFile,
ArtifactDifferenceCollectionExternalizer
) {
companion object {
private val MAX_BUILDS_PER_ARTIFACT = 10
}
override fun get(artifact: File): Iterable<ArtifactDifference>? =
storage[artifact.canonicalPath]
override fun add(artifact: File, difference: ArtifactDifference) {
val key = artifact.canonicalPath
val oldVal = storage[key] ?: emptyList()
val newVal = ArrayList(oldVal)
newVal.add(difference)
newVal.sortBy { it.buildTS }
storage[key] = newVal.takeLast(MAX_BUILDS_PER_ARTIFACT)
}
override fun remove(artifact: File) {
storage.remove(artifact.canonicalPath)
}
override fun dumpValue(value: Collection<ArtifactDifference>): String =
value.sortedBy { it.buildTS }.joinToString(separator = ",\n\t") { diff ->
"{ " +
"timestamp: ${diff.buildTS}, " +
"lookup symbols: [${diff.dirtyData.dirtyLookupSymbols.dumpLookupSymbols()}], " +
"fq names: [${diff.dirtyData.dirtyClassesFqNames.dumpFqNames()}]"
"}"
}
}
private fun <T> Collection<T>.takeLast(n: Int): Collection<T> =
drop(Math.max(size - n, 0))
private fun Collection<LookupSymbol>.dumpLookupSymbols(): String =
map { "${it.scope}#${it.name}" }.sorted().joinToString()
private fun Collection<FqName>.dumpFqNames(): String =
map(FqName::asString).sorted().joinToString()
private object ArtifactDifferenceCollectionExternalizer : DataExternalizer<Collection<ArtifactDifference>> {
override fun read(input: DataInput): Collection<ArtifactDifference> =
input.readCollectionTo(ArrayList()) {
val buildTS = readLong()
val dirtyLookupSymbols = readCollectionTo(HashSet()) {
val scope = readUTF()
val name = readUTF()
LookupSymbol(name, scope)
}
val dirtyFqNames = readCollectionTo(HashSet()) {
FqName(readUTF())
}
ArtifactDifference(buildTS, DirtyData(dirtyLookupSymbols, dirtyFqNames))
}
override fun save(output: DataOutput, value: Collection<ArtifactDifference>) {
output.writeCollection(value) { diff ->
writeLong(diff.buildTS)
writeCollection(diff.dirtyData.dirtyLookupSymbols) {
writeUTF(it.scope)
writeUTF(it.name)
}
writeCollection(diff.dirtyData.dirtyClassesFqNames) {
writeUTF(it.asString())
}
}
}
}
private inline fun <T> DataInput.readCollectionTo(col: MutableCollection<T>, readT: DataInput.()->T): Collection<T> {
val size = readInt()
repeat(size) {
col.add(readT())
}
return col
}
private inline fun <T> DataOutput.writeCollection(col: Collection<T>, writeT: DataOutput.(T)->Unit) {
writeInt(col.size)
col.forEach { writeT(it) }
}
@@ -0,0 +1,59 @@
package org.jetbrains.kotlin.gradle.tasks.incremental
import org.gradle.api.logging.Logging
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.tasks.ArtifactDifferenceRegistry
import org.jetbrains.kotlin.incremental.CacheVersion
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import java.io.File
/**
* "Global" cache holder. Should be created once per root project.
*/
internal class BuildCacheStorage(workingDir: File) : BasicMapsOwner() {
companion object {
private val OWN_VERSION = 0
private val ARTIFACT_DIFFERENCE = "artifact-difference"
}
private val log = Logging.getLogger(this.javaClass)
private val cachesDir: File = File(workingDir, "caches").apply { mkdirs() }
private val versionFile = File(cachesDir, "version.txt")
private val version = CacheVersion(
OWN_VERSION,
versionFile,
whenVersionChanged = CacheVersion.Action.REBUILD_ALL_KOTLIN,
whenTurnedOff = CacheVersion.Action.REBUILD_ALL_KOTLIN,
whenTurnedOn = CacheVersion.Action.REBUILD_ALL_KOTLIN,
// assume it's always enabled for simplicity (if IC is not enabled, just don't write to cache)
isEnabled = { true })
internal val artifactDifferenceRegistry: ArtifactDifferenceRegistry
private val String.storageFile: File
get() = File(cachesDir, this + "." + CACHE_EXTENSION)
init {
if (version.checkVersion() != CacheVersion.Action.DO_NOTHING) {
log.kotlinDebug { "Cache version is not up-to-date. Removing $cachesDir" }
cachesDir.deleteRecursively()
cachesDir.mkdirs()
}
artifactDifferenceRegistry = registerMap(ArtifactDifferenceRegistryImpl(ARTIFACT_DIFFERENCE.storageFile))
}
override fun clean() {
super.clean()
versionFile.delete()
}
override fun close() {
super.close()
version.saveIfNeeded()
}
override fun flush(memoryCachesOnly: Boolean) {
super.flush(memoryCachesOnly)
}
}
@@ -0,0 +1,51 @@
package org.jetbrains.kotlin.gradle.tasks.incremental
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.compile.AbstractCompile
import org.gradle.api.tasks.compile.JavaCompile
import org.jetbrains.kotlin.com.intellij.openapi.util.io.FileUtil
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.tasks.ArtifactDifferenceRegistry
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.tasks.tryGetSingleArtifact
import java.io.File
fun configureMultiProjectIncrementalCompilation(
project: Project,
kotlinTask: KotlinCompile,
javaTask: JavaCompile,
kotlinAfterJavaTask: KotlinCompile?,
artifactDifferenceRegistry: ArtifactDifferenceRegistry
) {
val log = kotlinTask.logger
log.kotlinDebug { "Configuring multi-project incremental compilation for project ${project.path}" }
fun cannotPerformMultiProjectIC(artifact: File, reason: String) {
log.kotlinDebug {
"Multi-project kotlin incremental compilation cannot be performed for project ${project.path}: $reason"
}
artifactDifferenceRegistry.remove(artifact)
}
fun isUnknownTaskOutputtingToJavaDestination(task: Task): Boolean {
return task !is JavaCompile &&
task !is KotlinCompile &&
task is AbstractCompile &&
FileUtil.isAncestor(javaTask.destinationDir, task.destinationDir, /* strict = */ false)
}
val artifact = project.tryGetSingleArtifact() ?: return
if (!kotlinTask.incremental) {
return cannotPerformMultiProjectIC(artifact, reason = "incremental compilation is not enabled")
}
val illegalTask = project.tasks.find(::isUnknownTaskOutputtingToJavaDestination)
if (illegalTask != null) {
return cannotPerformMultiProjectIC(artifact,
reason = "unknown task outputs to java destination dir ${illegalTask.path} $(${illegalTask.javaClass})")
}
val kotlinCompile = kotlinAfterJavaTask ?: kotlinTask
kotlinCompile.artifactDifferenceRegistry = artifactDifferenceRegistry
}
@@ -92,7 +92,7 @@ fun getSomething() = 10
project.build("build", options = options) {
assertSuccessful()
assertCompiledKotlinSources(listOf("src/main/kotlin/foo/KotlinActivity1.kt", "src/main/kotlin/foo/getSomething.kt"))
assertCompiledKotlinSources(listOf("app/src/main/kotlin/foo/KotlinActivity1.kt", "app/src/main/kotlin/foo/getSomething.kt"))
assertCompiledJavaSources(listOf("app/src/main/java/foo/JavaActivity.java"), weakTesting = true)
}
}
@@ -165,7 +165,7 @@ fun getSomething() = 10
project.build(":app:assembleDebug", options = options) {
assertSuccessful()
assertCompiledKotlinSources(project.relativizeToSubproject("app",
assertCompiledKotlinSources(project.relativize(
androidModuleKt,
baseApplicationKt,
useBuildConfigJavaKt,
@@ -95,11 +95,6 @@ abstract class BaseGradleIT {
fun relativize(vararg files: File): List<String> =
files.map { it.relativeTo(projectDir).path }
fun relativizeToSubproject(subproject: String, vararg files: File): List<String> {
val subprojectSir = File(projectDir, subproject)
return files.map { it.relativeTo(subprojectSir).path }
}
fun performModifications() {
for (file in projectDir.walk()) {
if (!file.isFile) continue
@@ -0,0 +1,188 @@
package org.jetbrains.kotlin.gradle
import org.jetbrains.kotlin.gradle.util.allKotlinFiles
import org.jetbrains.kotlin.gradle.util.getFileByName
import org.jetbrains.kotlin.gradle.util.getFilesByNames
import org.jetbrains.kotlin.gradle.util.modify
import org.junit.Test
import java.io.File
class IncrementalCompilationMultiProjectIT : BaseGradleIT() {
companion object {
private val GRADLE_VERSION = "2.10"
}
override fun defaultBuildOptions(): BuildOptions =
super.defaultBuildOptions().copy(withDaemon = true, incremental = true)
@Test
fun testMoveFunctionFromLib() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.build("build") {
assertSuccessful()
}
val barUseABKt = project.projectDir.getFileByName("barUseAB.kt")
val barInApp = File(project.projectDir, "app/src/main/java/bar").apply { mkdirs() }
barUseABKt.copyTo(File(barInApp, barUseABKt.name))
barUseABKt.delete()
project.build("build") {
assertSuccessful()
val affectedSources = project.projectDir.getFilesByNames("fooCallUseAB.kt", "barUseAB.kt")
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
@Test
fun testAddNewMethodToLib() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.build("build") {
assertSuccessful()
}
val aKt = project.projectDir.getFileByName("A.kt")
aKt.writeText("""
package bar
open class A {
fun a() {}
fun newA() {}
}
""")
project.build("build") {
assertSuccessful()
val affectedSources = project.projectDir.getFilesByNames("A.kt", "B.kt", "AA.kt", "BB.kt")
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
@Test
fun testLibClassBecameFinal() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.build("build") {
assertSuccessful()
}
val bKt = project.projectDir.getFileByName("B.kt")
bKt.modify { it.replace("open class", "class") }
project.build("build") {
assertFailed()
val affectedSources = project.projectDir.getFilesByNames(
"B.kt", "barUseAB.kt", "barUseB.kt",
"BB.kt", "fooCallUseAB.kt", "fooUseB.kt")
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
@Test
fun testModifyJavaInLib() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.build("build") {
assertSuccessful()
}
val javaClassJava = project.projectDir.getFileByName("JavaClass.java")
javaClassJava.modify { it.replace("String getString", "Object getString") }
project.build("build") {
assertSuccessful()
val affectedSources = project.projectDir.getFilesByNames("JavaClassChild.kt", "useJavaClass.kt")
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
@Test
fun testCleanBuildLib() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.build("build") {
assertSuccessful()
}
project.build(":lib:clean", ":lib:build") {
assertSuccessful()
val affectedSources = File(project.projectDir, "lib").allKotlinFiles()
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
project.build("build") {
assertSuccessful()
val affectedSources = File(project.projectDir, "app").allKotlinFiles()
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
@Test
fun testCompileErrorInLib() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.build("build") {
assertSuccessful()
}
val bKt = project.projectDir.getFileByName("B.kt")
bKt.delete()
project.build("build") {
assertFailed()
}
project.projectDir.getFileByName("barUseB.kt").delete()
project.projectDir.getFileByName("barUseAB.kt").delete()
project.build("build") {
assertFailed()
val affectedSources = project.projectDir.allKotlinFiles()
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
// checks that multi-project ic is disabled when there is a task that outputs to javaDestination dir
// that is not JavaCompile or KotlinCompile
@Test
fun testCompileLibWithGroovy() {
val project = Project("incrementalMultiproject", GRADLE_VERSION)
project.setupWorkingDir()
val lib = File(project.projectDir, "lib")
val libBuildGradle = File(lib, "build.gradle")
libBuildGradle.modify {
"""
apply plugin: 'groovy'
apply plugin: 'kotlin'
dependencies {
compile 'org.codehaus.groovy:groovy-all:2.4.7'
}
""".trimIndent()
}
val libGroovySrcBar = File(lib, "src/main/groovy/bar").apply { mkdirs() }
val groovyClass = File(libGroovySrcBar, "GroovyClass.groovy")
groovyClass.writeText("""
package bar
class GroovyClass {}
""")
project.build("build") {
assertSuccessful()
}
project.projectDir.getFileByName("barUseB.kt").delete()
project.build("build") {
assertSuccessful()
val affectedSources = File(project.projectDir, "app").allKotlinFiles()
val relativePaths = project.relativize(affectedSources)
assertCompiledKotlinSources(relativePaths, weakTesting = false)
}
}
}
@@ -5,6 +5,9 @@ import java.io.File
fun File.getFileByName(name: String): File =
findFileByName(name) ?: throw AssertionError("Could not find file with name '$name' in $this")
fun File.getFilesByNames(vararg names: String): List<File> =
names.map { getFileByName(it) }
fun File.findFileByName(name: String): File? =
walk().filter { it.isFile && it.name.equals(name, ignoreCase = true) }.firstOrNull()
@@ -0,0 +1,5 @@
apply plugin: 'kotlin'
dependencies {
compile project(':lib')
}
@@ -0,0 +1,7 @@
package foo
import bar.*
class AA : A() {
fun aa() {}
}
@@ -0,0 +1,7 @@
package foo
import bar.*
class BB : B() {
fun bb() {}
}
@@ -0,0 +1,5 @@
package foo
import bar.*
class JavaClassChild : JavaClass() {}
@@ -0,0 +1,8 @@
package foo
import bar.*
fun callUseAB() {
val b = B()
useAB(b)
}
@@ -0,0 +1,7 @@
package foo
import bar.*
private fun fooUseA(a: A) {
a.a()
}
@@ -0,0 +1,5 @@
package foo
private fun fooUseAA(aa: AA) {
aa.aa()
}
@@ -0,0 +1,7 @@
package foo
import bar.*
private fun fooUseB(b: B) {
b.b()
}
@@ -0,0 +1,5 @@
package foo
private fun fooUseBB(bb: BB) {
bb.bb()
}
@@ -0,0 +1,7 @@
package foo
import bar.*
fun useJavaClass(jc: JavaClass) {
jc.getString()
}
@@ -0,0 +1,16 @@
buildscript {
ext.kotlin_version = '0.1-SNAPSHOT'
repositories {
maven { url 'file://' + pathToKotlinPlugin }
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
}
}
allprojects {
// for test with groovy
repositories {
mavenCentral()
}
}
@@ -0,0 +1,5 @@
package bar
open class A {
fun a() {}
}
@@ -0,0 +1,5 @@
package bar
open class B : A() {
fun b() {}
}
@@ -0,0 +1,5 @@
package bar;
public class JavaClass {
public String getString() { return "Hello, World!"; }
}
@@ -0,0 +1,5 @@
package bar
private fun barUseA(a: A) {
a.a()
}
@@ -0,0 +1,5 @@
package bar
private fun barUseB(b: B) {
b.b()
}