diff --git a/libraries/tools/kotlin-gradle-plugin-core/pom.xml b/libraries/tools/kotlin-gradle-plugin-core/pom.xml index 6c420290b55..a6a3786cf1a 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/pom.xml +++ b/libraries/tools/kotlin-gradle-plugin-core/pom.xml @@ -62,6 +62,7 @@ ${project.basedir}/src/main/kotlin + ${project.basedir}/src/test/kotlin @@ -89,6 +90,16 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 2.19.1 + + + src/test/resources/** + + + diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/ChangedJavaFilesProcessor.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/ChangedJavaFilesProcessor.kt new file mode 100644 index 00000000000..22b502ef050 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/ChangedJavaFilesProcessor.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.gradle.tasks + +import org.gradle.api.logging.Logging +import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.com.intellij.lang.java.JavaLanguage +import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer +import org.jetbrains.kotlin.com.intellij.psi.PsiClass +import org.jetbrains.kotlin.com.intellij.psi.PsiFile +import org.jetbrains.kotlin.com.intellij.psi.PsiFileFactory +import org.jetbrains.kotlin.com.intellij.psi.PsiJavaFile +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.incremental.LookupSymbol +import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff +import java.io.File +import java.util.* + +internal class ChangedJavaFilesProcessor { + private val log = Logging.getLogger(this.javaClass.simpleName) + private val allSymbols = HashSet() + private val javaLang = JavaLanguage.INSTANCE + private val psiFileFactory: PsiFileFactory by lazy { + val rootDisposable = Disposer.newDisposable() + val configuration = CompilerConfiguration() + val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) + val project = environment.project + PsiFileFactory.getInstance(project) + } + + val allChangedSymbols: Collection + get() = allSymbols + + fun process(filesDiff: FileCollectionDiff): ChangesEither { + if (filesDiff.removed.any()) { + log.kotlinDebug { "Some java files are removed: [${filesDiff.removed.joinToString()}]" } + return ChangesEither.Unknown() + } + + val symbols = HashSet() + for (javaFile in filesDiff.newOrModified) { + assert(javaFile.extension.equals("java", ignoreCase = true)) + + val psiFile = javaFile.psiFile() + if (psiFile !is PsiJavaFile) { + log.kotlinDebug { "Expected PsiJavaFile, got ${psiFile?.javaClass}" } + return ChangesEither.Unknown() + } + + psiFile.classes.forEach { it.addLookupSymbols(symbols) } + } + allSymbols.addAll(symbols) + return ChangesEither.Known(lookupSymbols = symbols) + } + + private fun PsiClass.addLookupSymbols(symbols: MutableSet) { + val fqn = qualifiedName.orEmpty() + + symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!))) + methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) } + fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) } + innerClasses.forEach { it.addLookupSymbols(symbols) } + } + + private fun File.psiFile(): PsiFile? = + psiFileFactory.createFileFromText(nameWithoutExtension, javaLang, readText()) +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt index be5dac7f27e..f6b7c9fa520 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleIncrementalCacheImpl.kt @@ -21,22 +21,28 @@ import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.incremental.CompilationResult import org.jetbrains.kotlin.incremental.IncrementalCacheImpl import org.jetbrains.kotlin.incremental.dumpCollection +import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff +import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap +import org.jetbrains.kotlin.incremental.snapshots.SimpleFileSnapshotProviderImpl import org.jetbrains.kotlin.incremental.storage.BasicStringMap import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer import org.jetbrains.kotlin.modules.TargetId import java.io.File -class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl(targetDataRoot, targetOutputDir, target) { - +internal class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl(targetDataRoot, targetOutputDir, target) { companion object { private val SOURCES_TO_CLASSFILES = "sources-to-classfiles" + private val FILE_SNAPSHOT = "file-snapshot" } - private val loggerInstance = Logging.getLogger(this.javaClass) - fun getLogger() = loggerInstance + private val log = Logging.getLogger(this.javaClass) private val sourceToClassfilesMap = registerMap(SourceToClassfilesMap(SOURCES_TO_CLASSFILES.storageFile)) + private val fileSnapshotMap = registerMap(FileSnapshotMap(FILE_SNAPSHOT.storageFile)) + + fun compareAndUpdateFileSnapshots(files: Iterable): FileCollectionDiff = + fileSnapshotMap.compareAndUpdate(files, SimpleFileSnapshotProviderImpl()) fun removeClassfilesBySources(sources: Iterable): Unit = sources.forEach { sourceToClassfilesMap.remove(it) } @@ -60,11 +66,10 @@ class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, t // TODO: do it in the code that uses cache, since cache should not generally delete anything outside of it! // but for a moment it is an easiest solution to implement get(file).forEach { - getLogger().debug("[KOTLIN] Deleting $it on clearing cache for $file") + log.kotlinDebug { "Deleting $it on clearing cache for $file" } it.delete() } storage.remove(file.absolutePath) } } -} - +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/IncrementalCachesManager.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/IncrementalCachesManager.kt new file mode 100644 index 00000000000..e2cddce5388 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/IncrementalCachesManager.kt @@ -0,0 +1,49 @@ +package org.jetbrains.kotlin.gradle.tasks + +import org.jetbrains.kotlin.incremental.LookupStorage +import org.jetbrains.kotlin.modules.TargetId +import java.io.File + +internal class IncrementalCachesManager ( + private val targetId: TargetId, + private val cacheDirectory: File, + private val outputDir: File +) { + private val incrementalCacheDir = File(cacheDirectory, "increCache.${targetId.name}") + private val lookupCacheDir = File(cacheDirectory, "lookups") + private var incrementalCacheOpen = false + private var lookupCacheOpen = false + + val incrementalCache: GradleIncrementalCacheImpl by lazy { + incrementalCacheOpen = true + GradleIncrementalCacheImpl(targetDataRoot = incrementalCacheDir.apply { mkdirs() }, targetOutputDir = outputDir, target = targetId) + } + + val lookupCache: LookupStorage by lazy { + lookupCacheOpen = true + LookupStorage(lookupCacheDir.apply { mkdirs() }) + } + + fun clean() { + close(flush = false) + cacheDirectory.deleteRecursively() + } + + fun close(flush: Boolean = false) { + if (incrementalCacheOpen) { + if (flush) { + incrementalCache.flush(false) + } + incrementalCache.close() + incrementalCacheOpen = false + } + + if (lookupCacheOpen) { + if (flush) { + lookupCache.flush(false) + } + lookupCache.close() + lookupCacheOpen = false + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index 5e52acaa763..fdd3931e02d 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -43,6 +43,7 @@ import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.SourceRetentionAnnotationHandler +import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff import org.jetbrains.kotlin.modules.TargetId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.CompilationCanceledStatus @@ -155,7 +156,7 @@ open class KotlinCompile() : AbstractKotlinCompile() { private val sourceRoots = HashSet() // lazy because name is probably not available when constructor is called - val taskBuildDirectory: File by lazy { File(File(project.buildDir, KOTLIN_BUILD_DIR_NAME), name) } + val taskBuildDirectory: File by lazy { File(File(project.buildDir, KOTLIN_BUILD_DIR_NAME), name).apply { mkdirs() } } private val cacheDirectory: File by lazy { File(taskBuildDirectory, CACHES_DIR_NAME) } private val dirtySourcesSinceLastTimeFile: File by lazy { File(taskBuildDirectory, DIRTY_SOURCES_FILE_NAME) } private val lastBuildInfoFile: File by lazy { File(taskBuildDirectory, LAST_BUILD_INFO_FILE_NAME) } @@ -177,6 +178,9 @@ open class KotlinCompile() : AbstractKotlinCompile() { private val compileClasspath: Iterable get() = (classpath + additionalClasspath) .filterTo(LinkedHashSet()) { it.exists() } + private val kapt2GeneratedSourcesDir: File + get() = File(project.buildDir, "generated/source/kapt2") + val kaptOptions = KaptOptions() val pluginOptions = CompilerPluginOptions() @@ -252,10 +256,9 @@ open class KotlinCompile() : AbstractKotlinCompile() { val targetType = "java-production" val moduleName = args.moduleName - val targets = listOf(TargetId(moduleName, targetType)) + val targetId = TargetId(moduleName, targetType) val outputDir = destinationDir - val caches = hashMapOf() - val lookupStorage = LookupStorage(File(cacheDirectory, "lookups")) + val caches = IncrementalCachesManager(targetId, cacheDirectory, outputDir) val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) var currentRemoved = removed.filter { it.isKotlinFile() } val allGeneratedFiles = hashSetOf>() @@ -263,43 +266,6 @@ open class KotlinCompile() : AbstractKotlinCompile() { val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) logger.kotlinDebug { "Last Kotlin Build info for task $path -- $lastBuildInfo" } - fun getOrCreateIncrementalCache(target: TargetId): GradleIncrementalCacheImpl { - val cacheDir = File(cacheDirectory, "increCache.${target.name}") - cacheDir.mkdirs() - return GradleIncrementalCacheImpl(targetDataRoot = cacheDir, targetOutputDir = outputDir, target = target) - } - - fun getIncrementalCache(it: TargetId) = caches.getOrPut(it, { getOrCreateIncrementalCache(it) }) - - fun PsiClass.addLookupSymbols(symbols: MutableSet) { - val fqn = qualifiedName.orEmpty() - - symbols.add(LookupSymbol(name.orEmpty(), if (fqn == name) "" else fqn.removeSuffix("." + name!!))) - methods.forEach { symbols.add(LookupSymbol(it.name, fqn)) } - fields.forEach { symbols.add(LookupSymbol(it.name.orEmpty(), fqn)) } - innerClasses.forEach { it.addLookupSymbols(symbols) } - } - - val dirtyJavaLookupSymbols: Lazy> = lazy { - val symbols = HashSet() - val modifiedJavaFiles = modified.filter { it.isJavaFile() } - if (modifiedJavaFiles.isEmpty()) return@lazy symbols - - val rootDisposable = Disposer.newDisposable() - val configuration = CompilerConfiguration() - val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) - val project = environment.project - val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl - - modifiedJavaFiles.forEach { - val javaFile = psiFileFactory.createFileFromText(it.nameWithoutExtension, Language.findLanguageByID("JAVA")!!, it.readText()) - if (javaFile is PsiJavaFile) { - javaFile.classes.forEach { it.addLookupSymbols(symbols) } - } - } - symbols - } - fun getClasspathChanges(modifiedClasspath: List): ChangesEither { if (modifiedClasspath.isEmpty()) { logger.kotlinDebug { "No classpath changes" } @@ -340,11 +306,10 @@ open class KotlinCompile() : AbstractKotlinCompile() { return ChangesEither.Known(symbols, fqNames) } - fun calculateSourcesToCompile(): Pair, Boolean> { + fun calculateSourcesToCompile(javaFilesProcessor: ChangedJavaFilesProcessor): Pair, Boolean> { fun rebuild(reason: String): Pair, Boolean> { logger.kotlinInfo("Non-incremental compilation will be performed: $reason") - targets.forEach { getIncrementalCache(it).clean() } - lookupStorage.clean() + caches.clean() dirtySourcesSinceLastTimeFile.delete() return Pair(sources.toSet(), false) } @@ -352,12 +317,11 @@ open class KotlinCompile() : AbstractKotlinCompile() { if (!incremental) return rebuild("incremental compilation is not enabled") if (!isIncrementalRequested) return rebuild("inputs' changes are unknown (first or clean build)") - // TODO: store java files in the cache and extract removed symbols from it here - val illegalRemovedFiles = removed.filter { it.isJavaFile() || it.hasClassFileExtension() } - if (illegalRemovedFiles.any()) return rebuild("unsupported removed files: ${filesToString(illegalRemovedFiles)}") + val removedClassFiles = removed.filter(File::hasClassFileExtension) + if (removedClassFiles.any()) return rebuild("Removed class files: ${filesToString(removedClassFiles)}") - val illegalModifiedFiles = modified.filter(File::hasClassFileExtension) - if (illegalModifiedFiles.any()) return rebuild("unsupported modified files: ${filesToString(illegalModifiedFiles)}") + val modifiedClassFiles = modified.filter(File::hasClassFileExtension) + if (modifiedClassFiles.any()) return rebuild("Modified class files: ${filesToString(modifiedClassFiles)}") val modifiedClasspathEntries = modified.filter { it in classpath } val classpathChanges = getClasspathChanges(modifiedClasspathEntries) @@ -367,21 +331,28 @@ open class KotlinCompile() : AbstractKotlinCompile() { if (classpathChanges !is ChangesEither.Known) { throw AssertionError("Unknown implementation of ChangesEither: ${classpathChanges.javaClass}") } + val javaFilesDiff = FileCollectionDiff( + newOrModified = modified.filter { it.isJavaFile() }, + removed = removed.filter { it.isJavaFile() }) + val javaFilesChanges = javaFilesProcessor.process(javaFilesDiff) + val affectedJavaSymbols = when (javaFilesChanges) { + is ChangesEither.Known -> javaFilesChanges.lookupSymbols + is ChangesEither.Unknown -> return rebuild("Could not get changes for java files") + } val dirtyFiles = modified.filter { it.isKotlinFile() }.toMutableSet() val lookupSymbols = HashSet() - lookupSymbols.addAll(dirtyJavaLookupSymbols.value) + lookupSymbols.addAll(affectedJavaSymbols) lookupSymbols.addAll(classpathChanges.lookupSymbols) if (lookupSymbols.any()) { - val dirtyFilesFromLookups = mapLookupSymbolsToFiles(lookupStorage, lookupSymbols, logAction, ::projectRelativePath) + val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, logAction, ::projectRelativePath) dirtyFiles.addAll(dirtyFilesFromLookups) } - val allCaches = targets.map(::getIncrementalCache) - val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, allCaches) } + val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.incrementalCache)) } if (dirtyClassesFqNames.any()) { - val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(allCaches, dirtyClassesFqNames, logAction, ::projectRelativePath) + val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassesFqNames, logAction, ::projectRelativePath) dirtyFiles.addAll(dirtyFilesFromFqNames) } @@ -407,9 +378,7 @@ open class KotlinCompile() : AbstractKotlinCompile() { cleanupOnError() } - lookupStorage.flush(false) - lookupStorage.close() - caches.values.forEach { it.flush(false); it.close() } + caches.close(flush = true) logger.debug("flushed incremental caches") when (exitCode) { @@ -432,8 +401,9 @@ open class KotlinCompile() : AbstractKotlinCompile() { logger.warn(USING_EXPERIMENTAL_INCREMENTAL_MESSAGE) anyClassesCompiled = false + val javaFilesProcessor = ChangedJavaFilesProcessor() // TODO: decide what to do if no files are considered dirty - rebuild or skip the module - var (sourcesToCompile, isIncrementalDecided) = calculateSourcesToCompile() + var (sourcesToCompile, isIncrementalDecided) = calculateSourcesToCompile(javaFilesProcessor) if (isIncrementalDecided) { additionalClasspath.add(destinationDir) @@ -452,12 +422,9 @@ open class KotlinCompile() : AbstractKotlinCompile() { var exitCode = ExitCode.OK while (sourcesToCompile.any() || currentRemoved.any()) { val removedAndModified = (sourcesToCompile + currentRemoved).toList() - val outdatedClasses = targets.flatMap { getIncrementalCache(it).classesBySources(removedAndModified) } - - targets.forEach { getIncrementalCache(it).let { - it.markOutputClassesDirty(removedAndModified) - it.removeClassfilesBySources(removedAndModified) - }} + val outdatedClasses = caches.incrementalCache.classesBySources(removedAndModified) + caches.incrementalCache.markOutputClassesDirty(removedAndModified) + caches.incrementalCache.removeClassfilesBySources(removedAndModified) // can be empty if only removed sources are present if (sourcesToCompile.isNotEmpty()) { @@ -470,7 +437,7 @@ open class KotlinCompile() : AbstractKotlinCompile() { val text = existingSource.map { it.canonicalPath }.joinToString(separator = System.getProperty("line.separator")) dirtySourcesSinceLastTimeFile.writeText(text) - val compilerOutput = compileChanged(targets, existingSource.toSet(), outputDir, args, ::getIncrementalCache, lookupTracker) + val compilerOutput = compileChanged(listOf(targetId), existingSource.toSet(), outputDir, args, { caches.incrementalCache }, lookupTracker) exitCode = compilerOutput.exitCode if (exitCode == ExitCode.OK) { @@ -483,20 +450,36 @@ open class KotlinCompile() : AbstractKotlinCompile() { } allGeneratedFiles.addAll(compilerOutput.generatedFiles) - val compilationResult = updateIncrementalCaches(targets, compilerOutput.generatedFiles, + val compilationResult = updateIncrementalCaches(listOf(targetId), compilerOutput.generatedFiles, compiledWithErrors = exitCode != ExitCode.OK, - getIncrementalCache = { caches[it]!! }) + getIncrementalCache = { caches.incrementalCache }) - lookupStorage.update(lookupTracker, sourcesToCompile, currentRemoved) + caches.lookupCache.update(lookupTracker, sourcesToCompile, currentRemoved) + + val generatedJavaFiles = kapt2GeneratedSourcesDir.walk().filter { it.isJavaFile() }.toList() + val generatedJavaFilesDiff = caches.incrementalCache.compareAndUpdateFileSnapshots(generatedJavaFiles) if (!isIncrementalDecided) { artifactFile?.let { artifactDifferenceRegistry?.remove(it) } break } - val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(caches.values, logAction) - sourcesToCompile = mapLookupSymbolsToFiles(lookupStorage, dirtyLookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile) + - mapClassesFqNamesToFiles(caches.values, dirtyClassFqNames, logAction, ::projectRelativePath, excludes = sourcesToCompile) + val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.incrementalCache), logAction) + val generatedJavaFilesChanges = javaFilesProcessor.process(generatedJavaFilesDiff) + val dirtyKotlinFilesFromJava = when (generatedJavaFilesChanges) { + is ChangesEither.Unknown -> { + logger.kotlinDebug { "Could not get changes for generated java files, recompiling all kotlin" } + isIncrementalDecided = false + sources.toSet() + } + is ChangesEither.Known -> { + mapLookupSymbolsToFiles(caches.lookupCache, generatedJavaFilesChanges.lookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile) + } + else -> throw IllegalStateException("Unknown ChangesEither implementation: $generatedJavaFiles") + } + sourcesToCompile = dirtyKotlinFilesFromJava + + mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, logAction, ::projectRelativePath, excludes = sourcesToCompile) + + mapClassesFqNamesToFiles(listOf(caches.incrementalCache), dirtyClassFqNames, logAction, ::projectRelativePath, excludes = sourcesToCompile) buildDirtyLookupSymbols.addAll(dirtyLookupSymbols) buildDirtyFqNames.addAll(dirtyClassFqNames) @@ -507,8 +490,8 @@ open class KotlinCompile() : AbstractKotlinCompile() { } } - if (exitCode == ExitCode.OK) { - buildDirtyLookupSymbols.addAll(dirtyJavaLookupSymbols.value) + if (exitCode == ExitCode.OK && isIncrementalDecided) { + buildDirtyLookupSymbols.addAll(javaFilesProcessor.allChangedSymbols) } if (artifactFile != null && artifactDifferenceRegistry != null) { val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames) @@ -634,7 +617,6 @@ open class KotlinCompile() : AbstractKotlinCompile() { extension.equals(JavaFileType.INSTANCE.defaultExtension, ignoreCase = true) private fun File.isKapt2GeneratedDirectory(): Boolean { - val kapt2GeneratedSourcesDir = File(project.buildDir, "generated/source/kapt2") if (!kapt2GeneratedSourcesDir.isDirectory) return false return FileUtil.isAncestor(kapt2GeneratedSourcesDir, this, /* strict = */ false) } diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/gradleCacheVersion.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/gradleCacheVersion.kt index 5f9778ddb92..d30ca099f37 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/gradleCacheVersion.kt +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/gradleCacheVersion.kt @@ -4,7 +4,7 @@ import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.incremental.CacheVersion import java.io.File -const val GRADLE_CACHE_VERSION = 0 +const val GRADLE_CACHE_VERSION = 1 const val GRADLE_CACHE_VERSION_FILE_NAME = "gradle-format-version.txt" fun gradleCacheVersion(dataRoot: File): CacheVersion = diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt new file mode 100644 index 00000000000..4a104c9ff74 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.snapshots + +import java.io.File +import java.util.* + +internal class FileSnapshot( + val file: File, + val length: Long, + val hash: ByteArray +) { + init { + assert(!file.isDirectory) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other?.javaClass != javaClass) return false + + other as FileSnapshot + + if (file != other.file) return false + if (length != other.length) return false + if (!Arrays.equals(hash, other.hash)) return false + + return true + } + + override fun hashCode(): Int { + var result = file.hashCode() + result = 31 * result + length.hashCode() + result = 31 * result + Arrays.hashCode(hash) + return result + } + + override fun toString(): String { + return "FileSnapshot(file=$file, length=$length, hash=${Arrays.toString(hash)})" + } +} diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt new file mode 100644 index 00000000000..56c32313fe8 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.snapshots + +import org.jetbrains.kotlin.com.intellij.util.io.DataExternalizer +import java.io.DataInput +import java.io.DataOutput +import java.io.File + +internal object FileSnapshotExternalizer : DataExternalizer { + override fun save(out: DataOutput, value: FileSnapshot) { + out.writeUTF(value.file.canonicalPath) + out.writeLong(value.length) + out.writeInt(value.hash.size) + out.write(value.hash) + } + + override fun read(input: DataInput): FileSnapshot { + val file = File(input.readUTF()) + val length = input.readLong() + val hashSize = input.readInt() + val hash = ByteArray(hashSize) + input.readFully(hash) + return FileSnapshot(file, length, hash) + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt new file mode 100644 index 00000000000..06448b23c15 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.snapshots + +import org.jetbrains.kotlin.incremental.storage.BasicStringMap +import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor +import java.io.File +import java.util.* + +internal class FileSnapshotMap(storageFile: File) : BasicStringMap(storageFile, PathStringDescriptor, FileSnapshotExternalizer) { + override fun dumpValue(value: FileSnapshot): String = + value.toString() + + fun compareAndUpdate(newFiles: Iterable, snapshotProvider: FileSnapshotProvider): FileCollectionDiff { + val newOrModified = ArrayList() + val removed = ArrayList() + + val newPaths = newFiles.mapTo(HashSet()) { it.canonicalPath } + for (oldPath in storage.keys) { + if (oldPath !in newPaths) { + storage.remove(oldPath) + removed.add(File(oldPath)) + } + } + + for (path in newPaths) { + val file = File(path) + val oldSnapshot = storage[path] + val newSnapshot = snapshotProvider[file] + + if (oldSnapshot == null || oldSnapshot != newSnapshot) { + newOrModified.add(file) + storage[path] = newSnapshot + } + } + + return FileCollectionDiff(newOrModified, removed) + } +} + +internal class FileCollectionDiff(val newOrModified: Iterable, val removed: Iterable) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt new file mode 100644 index 00000000000..20aa79f480f --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.snapshots + +import java.io.File + +internal interface FileSnapshotProvider { + operator fun get(file: File): FileSnapshot +} + +internal class SimpleFileSnapshotProviderImpl : FileSnapshotProvider { + override fun get(file: File): FileSnapshot { + val length = file.length() + val hash = file.md5 + return FileSnapshot(file, length, hash) + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt new file mode 100644 index 00000000000..d428e605c94 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.snapshots + +import java.io.File +import java.security.MessageDigest + +val File.md5: ByteArray + get() { + val messageDigest = MessageDigest.getInstance("MD5") + val buffer = ByteArray(4048) + inputStream().use { input -> + while (true) { + val len = input.read(buffer) + if (len < 0) { + break + } + messageDigest.update(buffer, 0, len) + } + } + return messageDigest.digest() + } diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/TestWithWorkingDir.kt b/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/TestWithWorkingDir.kt new file mode 100644 index 00000000000..4574e560757 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/TestWithWorkingDir.kt @@ -0,0 +1,22 @@ +package org.jetbrains.kotlin + +import org.jetbrains.kotlin.com.intellij.openapi.util.io.FileUtil +import org.junit.After +import org.junit.Before +import java.io.File +import kotlin.properties.Delegates + +abstract class TestWithWorkingDir { + protected var workingDir: File by Delegates.notNull() + private set + + @Before + open fun setUp() { + workingDir = FileUtil.createTempDirectory(this.javaClass.simpleName, null) + } + + @After + open fun tearDown() { + workingDir.deleteRecursively() + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt b/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt new file mode 100644 index 00000000000..b3812a1b317 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt @@ -0,0 +1,67 @@ +package org.jetbrains.kotlin.incremental.snapshots + +import org.jetbrains.kotlin.TestWithWorkingDir +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import java.io.File +import kotlin.properties.Delegates + +class FileSnapshotMapTest : TestWithWorkingDir() { + private var snapshotMap: FileSnapshotMap by Delegates.notNull() + + @Before + override fun setUp() { + super.setUp() + val caches = File(workingDir, "caches").apply { mkdirs() } + val snapshotMapFile = File(caches, "snapshots.tab") + snapshotMap = FileSnapshotMap(snapshotMapFile) + } + + @After + override fun tearDown() { + snapshotMap.flush(false) + snapshotMap.close() + super.tearDown() + } + + @Test + fun testSnapshotMap() { + val src = File(workingDir, "src").apply { mkdirs() } + val foo = File(src, "foo").apply { mkdirs() } + + val removedTxt = File(foo, "removed.txt").apply { writeText("removed") } + val unchangedTxt = File(foo, "unchanged.txt").apply { writeText("unchanged") } + val changedTxt = File(foo, "changed.txt").apply { writeText("changed") } + + val snapshotProvider = SimpleFileSnapshotProviderImpl() + val diff1 = snapshotMap.compareAndUpdate(src.filesWithExt("txt"), snapshotProvider) + + assertArrayEquals("diff1.removed", + diff1.removed.toSortedPaths(), + emptyArray()) + assertArrayEquals("diff1.newOrModified", + diff1.newOrModified.toSortedPaths(), + listOf(removedTxt, unchangedTxt, changedTxt).toSortedPaths()) + + removedTxt.delete() + unchangedTxt.writeText("unchanged") + changedTxt.writeText("degnahc") + val newTxt = File(foo, "new.txt").apply { writeText("new") } + + val diff2 = snapshotMap.compareAndUpdate(src.filesWithExt("txt"), snapshotProvider) + assertArrayEquals("diff2.removed", + diff2.removed.toSortedPaths(), + listOf(removedTxt).toSortedPaths()) + assertArrayEquals("diff2.newOrModified", + diff2.newOrModified.toSortedPaths(), + listOf(newTxt, changedTxt).toSortedPaths()) + } + + private fun Iterable.toSortedPaths(): Array = + map { it.canonicalPath }.sorted().toTypedArray() + + private fun File.filesWithExt(ext: String): Iterable = + walk().filter { it.isFile && it.extension.equals(ext, ignoreCase = true) }.toList() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotTest.kt b/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotTest.kt new file mode 100644 index 00000000000..4c294372b82 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-core/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotTest.kt @@ -0,0 +1,76 @@ +package org.jetbrains.kotlin.incremental.snapshots + +import org.jetbrains.kotlin.TestWithWorkingDir +import org.junit.Assert.* +import org.junit.Test +import java.io.* + +class FileSnapshotTest : TestWithWorkingDir() { + private val fileSnapshotProvider: FileSnapshotProvider + get() = SimpleFileSnapshotProviderImpl() + + @Test + fun testExternalizer() { + val file = File(workingDir, "1.txt") + file.writeText("test") + val snapshot = fileSnapshotProvider[file] + val deserializedSnapshot = saveAndReadBack(snapshot) + assertEquals(snapshot, deserializedSnapshot) + } + + @Test + fun testEqualityNoChanges() { + val file = File(workingDir, "1.txt").apply { writeText("file") } + val oldSnapshot = fileSnapshotProvider[file] + val newSnapshot = fileSnapshotProvider[file] + assertEquals(oldSnapshot, newSnapshot) + } + + @Test + fun testEqualityDifferentFile() { + val file1 = File(workingDir, "1.txt").apply { writeText("file1") } + val file2 = File(workingDir, "2.txt").apply { + writeText(file1.readText()) + setLastModified(file1.lastModified()) + } + val oldSnapshot = fileSnapshotProvider[file1] + val newSnapshot = fileSnapshotProvider[file2] + assertNotEquals(oldSnapshot, newSnapshot) + } + + @Test + fun testEqualityDifferentTimestamp() { + val text = "file" + val file = File(workingDir, "1.txt").apply { writeText(text) } + val oldSnapshot = fileSnapshotProvider[file] + Thread.sleep(1000) + file.writeText(text) + val newSnapshot = fileSnapshotProvider[file] + assertEquals(oldSnapshot, newSnapshot) + } + + @Test + fun testEqualityDifferentSize() { + val file = File(workingDir, "1.txt").apply { writeText("file") } + val oldSnapshot = fileSnapshotProvider[file] + file.writeText("file modified") + val newSnapshot = fileSnapshotProvider[file] + assertNotEquals(oldSnapshot, newSnapshot) + } + + @Test + fun testEqualityDifferentHash() { + val file = File(workingDir, "1.txt").apply { writeText("file") } + val oldSnapshot = fileSnapshotProvider[file] + file.writeText("main") + val newSnapshot = fileSnapshotProvider[file] + assertNotEquals(oldSnapshot, newSnapshot) + } + + private fun saveAndReadBack(snapshot: FileSnapshot): FileSnapshot { + val byteOut = ByteArrayOutputStream() + DataOutputStream(byteOut).use { FileSnapshotExternalizer.save(it, snapshot) } + val byteIn = ByteArrayInputStream(byteOut.toByteArray()) + return DataInputStream(byteIn).use { FileSnapshotExternalizer.read(it) } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt2IT.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt2IT.kt index ea130a4cb65..7f51f0c1285 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt2IT.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt2IT.kt @@ -217,4 +217,29 @@ class Kapt2IT: BaseGradleIT() { assertFileExists("build/classes/main/example/RuntimeAnnotatedTestClassGenerated.class") } } + + @Test + fun testRemoveAnnotationIC() { + val project = Project("simple", GRADLE_2_14_VERSION, directoryPrefix = "kapt2") + val options = defaultBuildOptions().copy(incremental = true) + project.setupWorkingDir() + val internalDummyKt = project.projectDir.getFileByName("InternalDummy.kt") + + // add annotation + val exampleAnn = "@example.ExampleAnnotation " + internalDummyKt.modify { it.addBeforeSubstring(exampleAnn, "internal class InternalDummy")} + + project.build("classes", options = options) { + assertSuccessful() + } + + // remove annotation + internalDummyKt.modify { it.replace(exampleAnn, "")} + + project.build("classes", options = options) { + assertSuccessful() + val allMainKotlinSrc = File(project.projectDir, "src/main").allKotlinFiles() + assertCompiledKotlinSources(project.relativize(allMainKotlinSrc)) + } + } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/stringUtil.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/stringUtil.kt new file mode 100644 index 00000000000..6780e198253 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/stringUtil.kt @@ -0,0 +1,4 @@ +package org.jetbrains.kotlin.gradle.util + +fun String.addBeforeSubstring(prefix: String, substring: String): String = + replace(substring, prefix + substring) \ No newline at end of file