azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding extended gradle incremental cache implementation with map from sources to classfiles and some exposed functionality from parent class, using it to clean classfiles and find dirty lookups from removed files, refactoring dirty lookup calculation

Browse Source 
KT-8487
This commit is contained in:
Ilya Chernikov
2016-01-27 19:19:08 +01:00
committed by Alexey Tsvetkov
parent 4be395dcee
commit 819735e073
4 changed files with 135 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt
+4 -1
View File
@@ -139,7 +139,7 @@ open class IncrementalCacheImpl<Target>(
return CompilationResult.NO_CHANGES
}
fun saveFileToCache(generatedClass: GeneratedJvmClass<Target>): CompilationResult {
open fun saveFileToCache(generatedClass: GeneratedJvmClass<Target>): CompilationResult {
val sourceFiles: Collection<File> = generatedClass.sourceFiles
val kotlinClass: LocalFileKotlinClass = generatedClass.outputClass
val className = kotlinClass.className
@@ -347,6 +347,9 @@ open class IncrementalCacheImpl<Target>(
experimentalMaps.forEach { it.clean() }
}
fun classesBySources(sources: Iterable<File>): Iterable<JvmClassName> =
sources.flatMap { sourceToClassesMap[it] }
private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): CompilationResult {
libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt
+73
View File
@@ -0,0 +1,73 @@
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.gradle.tasks
import org.gradle.api.logging.Logging
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.incremental.CompilationResult
import org.jetbrains.kotlin.incremental.IncrementalCacheImpl
import org.jetbrains.kotlin.incremental.dumpCollection
import org.jetbrains.kotlin.incremental.storage.BasicStringMap
import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer
import org.jetbrains.kotlin.modules.TargetId
import java.io.File
class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl<TargetId>(targetDataRoot, targetOutputDir, target) {
companion object {
private val SOURCES_TO_CLASSFILES = "sources-to-classfiles"
}
private val loggerInstance = Logging.getLogger(this.javaClass)
fun getLogger() = loggerInstance
private val sourceToClassfilesMap = registerMap(SourceToClassfilesMap(SOURCES_TO_CLASSFILES.storageFile))
fun classfilesBySources(sources: Iterable<File>): Iterable<File> =
sources.flatMap { sourceToClassfilesMap[it] }
fun removeClassfilesBySources(sources: Iterable<File>): Unit =
sources.forEach { sourceToClassfilesMap.remove(it) }
override fun saveFileToCache(generatedClass: GeneratedJvmClass<TargetId>): CompilationResult {
generatedClass.sourceFiles.forEach { sourceToClassfilesMap.add(it, generatedClass.outputFile) }
return super.saveFileToCache(generatedClass)
}
inner class SourceToClassfilesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
fun add(sourceFile: File, classFile: File) {
storage.append(sourceFile.absolutePath, classFile.absolutePath)
}
operator fun get(sourceFile: File): Collection<File> =
storage[sourceFile.absolutePath].orEmpty().map { File(it) }
override fun dumpValue(value: Collection<String>) = value.dumpCollection()
fun remove(file: File) {
// TODO: do it in the code that uses cache, since cache should not generally delete anything outside of it!
// but for a moment it is an easiest solution to implement
get(file).forEach {
getLogger().debug("[KOTLIN] Deleting $it on clearing cache for $file")
it.delete()
}
storage.remove(file.absolutePath)
}
}
}
libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt
+49 -25
View File
@@ -229,17 +229,17 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
val moduleName = args.moduleName
val targets = listOf(TargetId(moduleName, targetType))
val outputDir = File(args.destination)
val caches = hashMapOf<TargetId, IncrementalCacheImpl<TargetId>>()
val caches = hashMapOf<TargetId, GradleIncrementalCacheImpl>()
val lookupStorage = LookupStorage(File(cachesBaseDir, "lookups"))
val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
var currentRemoved = removed
val allGeneratedFiles = hashSetOf<GeneratedFile<TargetId>>()
val compiledSourcesSet = hashSetOf<File>()
fun getOrCreateIncrementalCache(target: TargetId): IncrementalCacheImpl<TargetId> {
fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl {
val cacheDir = File(cachesBaseDir, "increCache.${target.name}")
cacheDir.mkdirs()
return IncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target)
return GradleIncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target)
}
fun getIncrementalCache(it: TargetId) = caches.getOrPut(it, { getOrCreateIncrementalCache(it) })
@@ -261,35 +261,48 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
files
}
fun dirtyKotlinSourcesFromGradle(): List<File> {
// TODO: take into account deleted files (take their symbols from caches
// TODO: handle classpath changes similarly - compare with cashed version (likely a big change, may be costly, some heuristics could be considered)
val kotlinFiles = modified.filter { it.isKotlinFile() }
val javaFiles = modified.filter { it.isJavaFile() }
if (javaFiles.any()) {
fun dirtyLookupSymbolsFromRemovedKotlinFiles(): List<LookupSymbol> {
val removedKotlinFiles = removed.filter { it.isKotlinFile() }
return if (removedKotlinFiles.isNotEmpty())
targets.flatMap { getIncrementalCache(it).classesBySources(removedKotlinFiles).map { LookupSymbol(it.fqNameForClassNameWithoutDollars.shortName().toString(), it.packageFqName.toString()) } }
else listOf()
}
fun dirtyLookupSymbolsFromModifiedJavaFiles(): List<LookupSymbol> {
val modifiedJavaFiles = modified.filter { it.isJavaFile() }
return (if (modifiedJavaFiles.any()) {
val rootDisposable = Disposer.newDisposable()
val configuration = CompilerConfiguration()
val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
val project = environment.project
val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
val lookupSymbols = javaFiles.flatMap {
modifiedJavaFiles.flatMap {
val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText())
if (javaFile is PsiJavaFile)
javaFile.classes.flatMap { it.findLookupSymbols() }
else listOf()
}
// logger.kotlinDebug("changed java symbols: ${lookupSymbols.joinToString { "${it.name}:${it.scope}" }}")
if (lookupSymbols.any()) {
val kotlinFilesSet = kotlinFiles.toHashSet()
return kotlinFiles +
lookupSymbols.files(
filesFilter = { it !in kotlinFilesSet },
logAction = { lookup, files ->
logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}")
})
}
} else listOf())
}
fun dirtyKotlinSourcesFromGradle(): List<File> {
// TODO: handle classpath changes similarly - compare with cashed version (likely a big change, may be costly, some heuristics could be considered)
val modifiedKotlinFiles = modified.filter { it.isKotlinFile() }
val lookupSymbols =
dirtyLookupSymbolsFromModifiedJavaFiles() +
dirtyLookupSymbolsFromRemovedKotlinFiles()
// TODO: add dirty lookups from modified kotlin files to reduce number of steps needed
if (lookupSymbols.any()) {
val kotlinModifiedFilesSet = modifiedKotlinFiles.toHashSet()
return modifiedKotlinFiles +
lookupSymbols.files(
filesFilter = { it !in kotlinModifiedFilesSet },
logAction = { lookup, files ->
logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}")
})
}
return kotlinFiles
return modifiedKotlinFiles
}
fun isClassPathChanged(): Boolean {
@@ -303,8 +316,12 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
fun calculateSourcesToCompile(): Pair<List<File>, Boolean> {
// TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes
if (!isIncrementalRequested || isClassPathChanged()) {
if (!isIncrementalRequested ||
// TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes
isClassPathChanged() ||
// so far considering it not incremental TODO: store java files in the cache and extract removed symbols from it here
removed.any { it.isJavaFile() }
) {
logger.kotlinInfo(if (!isIncrementalRequested) "clean caches on rebuild" else "classpath changed, rebuilding all kotlin files")
targets.forEach { getIncrementalCache(it).clean() }
lookupStorage.clean()
@@ -346,7 +363,14 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
}
}
}
return Pair(dirtyKotlinSourcesFromGradle().distinct(), true)
val dirtyFiles = dirtyKotlinSourcesFromGradle().distinct()
// first dirty files should be found and only then caches cleared
val removedKotlinFiles = removed.filter { it.isKotlinFile() }
targets.forEach { getIncrementalCache(it).let {
it.markOutputClassesDirty(removedKotlinFiles)
it.removeClassfilesBySources(removedKotlinFiles)
}}
return Pair(dirtyFiles, true)
}
fun cleanupOnError() {
@@ -465,7 +489,7 @@ open class KotlinCompile() : AbstractKotlinCompile<K2JVMCompilerArguments>() {
sourcesToCompile: List<File>,
outputDir: File,
args: K2JVMCompilerArguments,
getIncrementalCache: (TargetId) -> IncrementalCacheImpl<TargetId>,
getIncrementalCache: (TargetId) -> GradleIncrementalCacheImpl,
lookupTracker: LookupTracker)
: CompileChangedResults
{
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPlugin.kt
+9 -7
View File
@@ -136,14 +136,16 @@ class Kotlin2JvmSourceSetProcessor(
// Since we cannot update classpath statically, java not able to detect changes in the classpath after kotlin compiler.
// Therefore this (probably inefficient since java cannot decide "uptodateness" by the list of changed class files, but told
// explicitly being out of date whenever any kotlin files are compiled
kotlinTask.property("anyClassesCompiled")?.let {
kotlinTask.setProperty("anyClassesCompiled", false)
javaTask.outputs.upToDateWhen { task ->
val kotlinClassesCompiled = kotlinTask.property("anyClassesCompiled") as? Boolean ?: false
if (kotlinClassesCompiled) {
logger.info("Marking $task out of date, because kotlin classes are changed")
if (kotlinTask.hasProperty("anyClassesCompiled")) {
kotlinTask.property("anyClassesCompiled")?.let {
kotlinTask.setProperty("anyClassesCompiled", false)
javaTask.outputs.upToDateWhen { task ->
val kotlinClassesCompiled = kotlinTask.property("anyClassesCompiled") as? Boolean ?: false
if (kotlinClassesCompiled) {
logger.info("Marking $task out of date, because kotlin classes are changed")
}
!kotlinClassesCompiled
}
!kotlinClassesCompiled
}
}