diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt index 390bf2544ca..d8e0bed463c 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt @@ -139,7 +139,7 @@ open class IncrementalCacheImpl( return CompilationResult.NO_CHANGES } - fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult { + open fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult { val sourceFiles: Collection = generatedClass.sourceFiles val kotlinClass: LocalFileKotlinClass = generatedClass.outputClass val className = kotlinClass.className @@ -347,6 +347,9 @@ open class IncrementalCacheImpl( experimentalMaps.forEach { it.clean() } } + fun classesBySources(sources: Iterable): Iterable = + sources.flatMap { sourceToClassesMap[it] } + private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): CompilationResult { diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt new file mode 100644 index 00000000000..c3abd7ab4f9 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.gradle.tasks + +import org.gradle.api.logging.Logging +import org.jetbrains.kotlin.build.GeneratedJvmClass +import org.jetbrains.kotlin.incremental.CompilationResult +import org.jetbrains.kotlin.incremental.IncrementalCacheImpl +import org.jetbrains.kotlin.incremental.dumpCollection +import org.jetbrains.kotlin.incremental.storage.BasicStringMap +import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor +import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer +import org.jetbrains.kotlin.modules.TargetId +import java.io.File + +class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl(targetDataRoot, targetOutputDir, target) { + + companion object { + private val SOURCES_TO_CLASSFILES = "sources-to-classfiles" + } + + private val loggerInstance = Logging.getLogger(this.javaClass) + fun getLogger() = loggerInstance + + private val sourceToClassfilesMap = registerMap(SourceToClassfilesMap(SOURCES_TO_CLASSFILES.storageFile)) + + fun classfilesBySources(sources: Iterable): Iterable = + sources.flatMap { sourceToClassfilesMap[it] } + + fun removeClassfilesBySources(sources: Iterable): Unit = + sources.forEach { sourceToClassfilesMap.remove(it) } + + override fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult { + generatedClass.sourceFiles.forEach { sourceToClassfilesMap.add(it, generatedClass.outputFile) } + return super.saveFileToCache(generatedClass) + } + + inner class SourceToClassfilesMap(storageFile: File) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer) { + fun add(sourceFile: File, classFile: File) { + storage.append(sourceFile.absolutePath, classFile.absolutePath) + } + + operator fun get(sourceFile: File): Collection = + storage[sourceFile.absolutePath].orEmpty().map { File(it) } + + override fun dumpValue(value: Collection) = value.dumpCollection() + + fun remove(file: File) { + // TODO: do it in the code that uses cache, since cache should not generally delete anything outside of it! + // but for a moment it is an easiest solution to implement + get(file).forEach { + getLogger().debug("[KOTLIN] Deleting $it on clearing cache for $file") + it.delete() + } + storage.remove(file.absolutePath) + } + } +} + diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index 69fbd133d06..f7c4e8223e6 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -229,17 +229,17 @@ open class KotlinCompile() : AbstractKotlinCompile() { val moduleName = args.moduleName val targets = listOf(TargetId(moduleName, targetType)) val outputDir = File(args.destination) - val caches = hashMapOf>() + val caches = hashMapOf() val lookupStorage = LookupStorage(File(cachesBaseDir, "lookups")) val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) var currentRemoved = removed val allGeneratedFiles = hashSetOf>() val compiledSourcesSet = hashSetOf() - fun getOrCreateIncrementalCache(target: TargetId): IncrementalCacheImpl { + fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl { val cacheDir = File(cachesBaseDir, "increCache.${target.name}") cacheDir.mkdirs() - return IncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target) + return GradleIncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target) } fun getIncrementalCache(it: TargetId) = caches.getOrPut(it, { getOrCreateIncrementalCache(it) }) @@ -261,35 +261,48 @@ open class KotlinCompile() : AbstractKotlinCompile() { files } - fun dirtyKotlinSourcesFromGradle(): List { - // TODO: take into account deleted files (take their symbols from caches - // TODO: handle classpath changes similarly - compare with cashed version (likely a big change, may be costly, some heuristics could be considered) - val kotlinFiles = modified.filter { it.isKotlinFile() } - val javaFiles = modified.filter { it.isJavaFile() } - if (javaFiles.any()) { + fun dirtyLookupSymbolsFromRemovedKotlinFiles(): List { + val removedKotlinFiles = removed.filter { it.isKotlinFile() } + return if (removedKotlinFiles.isNotEmpty()) + targets.flatMap { getIncrementalCache(it).classesBySources(removedKotlinFiles).map { LookupSymbol(it.fqNameForClassNameWithoutDollars.shortName().toString(), it.packageFqName.toString()) } } + else listOf() + } + + fun dirtyLookupSymbolsFromModifiedJavaFiles(): List { + val modifiedJavaFiles = modified.filter { it.isJavaFile() } + return (if (modifiedJavaFiles.any()) { val rootDisposable = Disposer.newDisposable() val configuration = CompilerConfiguration() val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) val project = environment.project val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl - val lookupSymbols = javaFiles.flatMap { + modifiedJavaFiles.flatMap { val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText()) if (javaFile is PsiJavaFile) javaFile.classes.flatMap { it.findLookupSymbols() } else listOf() } -// logger.kotlinDebug("changed java symbols: ${lookupSymbols.joinToString { "${it.name}:${it.scope}" }}") - if (lookupSymbols.any()) { - val kotlinFilesSet = kotlinFiles.toHashSet() - return kotlinFiles + - lookupSymbols.files( - filesFilter = { it !in kotlinFilesSet }, - logAction = { lookup, files -> - logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}") - }) - } + } else listOf()) + } + + fun dirtyKotlinSourcesFromGradle(): List { + // TODO: handle classpath changes similarly - compare with cashed version (likely a big change, may be costly, some heuristics could be considered) + val modifiedKotlinFiles = modified.filter { it.isKotlinFile() } + val lookupSymbols = + dirtyLookupSymbolsFromModifiedJavaFiles() + + dirtyLookupSymbolsFromRemovedKotlinFiles() + // TODO: add dirty lookups from modified kotlin files to reduce number of steps needed + + if (lookupSymbols.any()) { + val kotlinModifiedFilesSet = modifiedKotlinFiles.toHashSet() + return modifiedKotlinFiles + + lookupSymbols.files( + filesFilter = { it !in kotlinModifiedFilesSet }, + logAction = { lookup, files -> + logger.kotlinInfo("changes in ${lookup.name} (${lookup.scope}) causes recompilation of ${files.joinToString { projectRelativePath(it) }}") + }) } - return kotlinFiles + return modifiedKotlinFiles } fun isClassPathChanged(): Boolean { @@ -303,8 +316,12 @@ open class KotlinCompile() : AbstractKotlinCompile() { fun calculateSourcesToCompile(): Pair, Boolean> { - // TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes - if (!isIncrementalRequested || isClassPathChanged()) { + if (!isIncrementalRequested || + // TODO: more precise will be not to rebuild unconditionally on classpath changes, but retrieve lookup info and try to find out which sources are affected by cp changes + isClassPathChanged() || + // so far considering it not incremental TODO: store java files in the cache and extract removed symbols from it here + removed.any { it.isJavaFile() } + ) { logger.kotlinInfo(if (!isIncrementalRequested) "clean caches on rebuild" else "classpath changed, rebuilding all kotlin files") targets.forEach { getIncrementalCache(it).clean() } lookupStorage.clean() @@ -346,7 +363,14 @@ open class KotlinCompile() : AbstractKotlinCompile() { } } } - return Pair(dirtyKotlinSourcesFromGradle().distinct(), true) + val dirtyFiles = dirtyKotlinSourcesFromGradle().distinct() + // first dirty files should be found and only then caches cleared + val removedKotlinFiles = removed.filter { it.isKotlinFile() } + targets.forEach { getIncrementalCache(it).let { + it.markOutputClassesDirty(removedKotlinFiles) + it.removeClassfilesBySources(removedKotlinFiles) + }} + return Pair(dirtyFiles, true) } fun cleanupOnError() { @@ -465,7 +489,7 @@ open class KotlinCompile() : AbstractKotlinCompile() { sourcesToCompile: List, outputDir: File, args: K2JVMCompilerArguments, - getIncrementalCache: (TargetId) -> IncrementalCacheImpl, + getIncrementalCache: (TargetId) -> GradleIncrementalCacheImpl, lookupTracker: LookupTracker) : CompileChangedResults { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPlugin.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPlugin.kt index 718356ebf37..ee418ff8088 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPlugin.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPlugin.kt @@ -136,14 +136,16 @@ class Kotlin2JvmSourceSetProcessor( // Since we cannot update classpath statically, java not able to detect changes in the classpath after kotlin compiler. // Therefore this (probably inefficient since java cannot decide "uptodateness" by the list of changed class files, but told // explicitly being out of date whenever any kotlin files are compiled - kotlinTask.property("anyClassesCompiled")?.let { - kotlinTask.setProperty("anyClassesCompiled", false) - javaTask.outputs.upToDateWhen { task -> - val kotlinClassesCompiled = kotlinTask.property("anyClassesCompiled") as? Boolean ?: false - if (kotlinClassesCompiled) { - logger.info("Marking $task out of date, because kotlin classes are changed") + if (kotlinTask.hasProperty("anyClassesCompiled")) { + kotlinTask.property("anyClassesCompiled")?.let { + kotlinTask.setProperty("anyClassesCompiled", false) + javaTask.outputs.upToDateWhen { task -> + val kotlinClassesCompiled = kotlinTask.property("anyClassesCompiled") as? Boolean ?: false + if (kotlinClassesCompiled) { + logger.info("Marking $task out of date, because kotlin classes are changed") + } + !kotlinClassesCompiled } - !kotlinClassesCompiled } }