diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index cf73a90692f..8066a99462e 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -106,12 +106,6 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR val incrementalCaches = chunk.getTargets().keysToMap { dataManager.getKotlinCache(it) } - for (target in chunk.getTargets()) { - val removedFiles = dirtyFilesHolder.getRemovedFiles(target) - val cache = incrementalCaches[target]!! - removedFiles.forEach { cache.fileIsDeleted(File(it)) } - } - val environment = createCompileEnvironment(incrementalCaches) if (!environment.success()) { environment.reportErrorsTo(messageCollector) @@ -129,6 +123,14 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR compileToJs(chunk, commonArguments, environment, messageCollector, project) } else { + if (IncrementalCompilation.ENABLED) { + for (target in chunk.getTargets()) { + val cache = incrementalCaches[target]!! + val removedAndDirtyFiles = filesToCompile[target] + dirtyFilesHolder.getRemovedFiles(target).map { File(it) } + cache.markOutputClassesDirty(removedAndDirtyFiles) + } + } + compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, filesToCompile, messageCollector) } @@ -146,7 +148,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR recompilationDecision = DO_NOTHING } else { - recompilationDecision = updateKotlinIncrementalCache(compilationErrors, dirtyFilesHolder, incrementalCaches, outputsItemsAndTargets) + recompilationDecision = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, outputsItemsAndTargets) updateJavaMappings(chunk, compilationErrors, context, dirtyFilesHolder, filesToCompile, outputsItemsAndTargets) } @@ -273,7 +275,6 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR private fun updateKotlinIncrementalCache( compilationErrors: Boolean, - dirtyFilesHolder: DirtyFilesHolder, incrementalCaches: Map, outputsItemsAndTargets: List> ): IncrementalCacheImpl.RecompilationDecision { @@ -281,23 +282,16 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR return DO_NOTHING } - if (!compilationErrors) { - incrementalCaches.values().forEach { it.clearRemovedPackageParts () } - } - - for ((target, cache) in incrementalCaches) { - cache.clearCacheForRemovedFiles( - KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target), - target.getOutputDir()!!, - !compilationErrors - ) - } - var recompilationDecision = DO_NOTHING for ((outputItem, target) in outputsItemsAndTargets) { val newDecision = incrementalCaches[target]!!.saveFileToCache(outputItem.getSourceFiles(), outputItem.getOutputFile()) recompilationDecision = recompilationDecision.merge(newDecision) } + + if (!compilationErrors) { + incrementalCaches.values().forEach { it.clearCacheForRemovedClasses() } + } + return recompilationDecision } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index fd8e1219255..483a57b52a6 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -49,6 +49,7 @@ import com.intellij.util.io.BooleanDataDescriptor import java.util.ArrayList import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.utils.Printer +import java.io.DataInputStream val INLINE_ANNOTATION_DESC = "Lkotlin/inline;" @@ -88,7 +89,8 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen val CONSTANTS_MAP = "constants.tab" val INLINE_FUNCTIONS = "inline-functions.tab" val PACKAGE_PARTS = "package-parts.tab" - val REMOVED_PACKAGE_PARTS = "removed-package-parts.tab" + val SOURCE_TO_CLASSES = "source-to-classes.tab" + val DIRTY_OUTPUT_CLASSES = "dirty-output-classes.tab" } private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME) @@ -96,9 +98,10 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen private val constantsMap = ConstantsMap() private val inlineFunctionsMap = InlineFunctionsMap() private val packagePartMap = PackagePartMap() - private val removedPackagePartMap = RemovedPackagePartMap() + private val sourceToClassesMap = SourceToClassesMap() + private val dirtyOutputClassesMap = DirtyOutputClassesMap() - private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap, removedPackagePartMap) + private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap, sourceToClassesMap, dirtyOutputClassesMap) private val cacheFormatVersion = CacheFormatVersion(targetDataRoot) @@ -107,6 +110,15 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return maps.map { it.dump() }.join("\n\n") } + public fun markOutputClassesDirty(removedAndCompiledSources: List) { + for (sourceFile in removedAndCompiledSources) { + val classes = sourceToClassesMap[sourceFile] + classes.forEach { dirtyOutputClassesMap.markDirty(it.getInternalName()) } + + sourceToClassesMap.clearOutputsForSource(sourceFile) + } + } + public fun saveFileToCache(sourceFiles: Collection, classFile: File): RecompilationDecision { if (classFile.extension.toLowerCase() != "class") return DO_NOTHING @@ -119,6 +131,9 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen val className = JvmClassName.byClassId(kotlinClass.getClassId()) val header = kotlinClass.getClassHeader() + dirtyOutputClassesMap.notDirty(className.getInternalName()) + sourceFiles.forEach { sourceToClassesMap.addSourceToClass(it, className) } + val annotationDataEncoded = header.annotationData if (annotationDataEncoded != null) { val data = BitEncoding.decodeBytes(annotationDataEncoded) @@ -147,7 +162,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen if (header.syntheticClassKind == JvmAnnotationNames.KotlinSyntheticClass.Kind.PACKAGE_PART) { assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" } - packagePartMap.putPackagePartSourceData(sourceFiles.first(), className) + packagePartMap.addPackagePart(className) val inlinesChanged = inlineFunctionsMap.process(className, fileBytes) val constantsChanged = constantsMap.process(className, fileBytes) @@ -161,30 +176,19 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return DO_NOTHING } - public fun fileIsDeleted(sourceFile: File) { - val wasPackagePart = packagePartMap[sourceFile] - packagePartMap.remove(sourceFile) - if (wasPackagePart != null) { - removedPackagePartMap.add(wasPackagePart) - } - } - - public fun clearRemovedPackageParts() { - removedPackagePartMap.clear() - } - - // TODO transitional function - deprecated("transitional function") - public fun clearCacheForRemovedFiles(removedSourceFiles: Collection, outDirectory: File, compilationSuccessful: Boolean) { - if (compilationSuccessful) { - inlineFunctionsMap.clearOutdated(outDirectory) - constantsMap.clearOutdated(outDirectory) - protoMap.clearOutdated(outDirectory) + public fun clearCacheForRemovedClasses() { + for (internalClassName in dirtyOutputClassesMap.getDirtyOutputClasses()) { + val className = JvmClassName.byInternalName(internalClassName) + protoMap.remove(className) + packagePartMap.remove(className) + constantsMap.remove(className) + inlineFunctionsMap.remove(className) } + dirtyOutputClassesMap.clear() } public override fun getObsoletePackageParts(sourceFilesToCompile: Collection): Collection { - return (removedPackagePartMap.getRemovedParts() + sourceFilesToCompile.map { packagePartMap[it] }.filterNotNull()).toSet() + return dirtyOutputClassesMap.getDirtyOutputClasses().filter { packagePartMap.isPackagePart(JvmClassName.byInternalName(it)) } } public override fun getPackageData(fqName: String): ByteArray? { @@ -260,29 +264,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen protected abstract fun dumpValue(value: V): String } - private abstract class ClassFileBasedMap : BasicMap() { - - // TODO may be too expensive, because it traverses all files in out directory - public fun clearOutdated(outDirectory: File) { - val keysToRemove = HashSet() - - storage.processKeysWithExistingMapping { key -> - val className = JvmClassName.byInternalName(key!!) - val classFile = File(outDirectory, FileUtil.toSystemDependentName(className.getInternalName()) + ".class") - if (!classFile.exists()) { - keysToRemove.add(key) - } - - true - } - - for (key in keysToRemove) { - storage.remove(key) - } - } - } - - private inner class ProtoMap : ClassFileBasedMap() { + private inner class ProtoMap : BasicMap() { override fun createMap(): PersistentHashMap = PersistentHashMap( File(baseDir, PROTO_MAP), EnumeratorStringDescriptor(), @@ -303,12 +285,16 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return storage[className.getInternalName()] } + public fun remove(className: JvmClassName) { + storage.remove(className.getInternalName()) + } + override fun dumpValue(value: ByteArray): String { return java.lang.Long.toHexString(value.md5()) } } - private inner class ConstantsMap : ClassFileBasedMap>() { + private inner class ConstantsMap : BasicMap>() { override fun createMap(): PersistentHashMap> = PersistentHashMap( File(baseDir, CONSTANTS_MAP), EnumeratorStringDescriptor(), @@ -351,6 +337,10 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return true } + public fun remove(className: JvmClassName) { + put(className, null) + } + override fun dumpValue(value: Map): String { return StringBuilder { append("{") @@ -423,7 +413,7 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen } } - private inner class InlineFunctionsMap : ClassFileBasedMap>() { + private inner class InlineFunctionsMap : BasicMap>() { override fun createMap(): PersistentHashMap> = PersistentHashMap( File(baseDir, INLINE_FUNCTIONS), EnumeratorStringDescriptor(), @@ -482,6 +472,10 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen return true } + public fun remove(className: JvmClassName) { + put(className, null) + } + override fun dumpValue(value: Map): String { return StringBuilder { append("{") @@ -521,42 +515,66 @@ public class IncrementalCacheImpl(targetDataRoot: File) : StorageOwner, Incremen } - - private inner class PackagePartMap : BasicMap() { - // Format of serialization to string: --> - override fun createMap(): PersistentHashMap = PersistentHashMap( - File(baseDir, PACKAGE_PARTS), - EnumeratorStringDescriptor(), - EnumeratorStringDescriptor() - ) - - public fun putPackagePartSourceData(sourceFile: File, className: JvmClassName) { - storage.put(sourceFile.getAbsolutePath(), className.getInternalName()) - } - - public fun remove(sourceFile: File) { - storage.remove(sourceFile.getAbsolutePath()) - } - - public fun get(sourceFile: File): String? { - return storage[sourceFile.getAbsolutePath()] - } - - override fun dumpValue(value: String) = value - } - - private inner class RemovedPackagePartMap : BasicMap() { + private inner class PackagePartMap : BasicMap() { override fun createMap(): PersistentHashMap = PersistentHashMap( - File(baseDir, REMOVED_PACKAGE_PARTS), + File(baseDir, PACKAGE_PARTS), EnumeratorStringDescriptor(), BooleanDataDescriptor.INSTANCE ) - public fun add(name: String) { - storage.put(name, true) + public fun addPackagePart(className: JvmClassName) { + storage.put(className.getInternalName(), true) } - public fun getRemovedParts(): Collection { + public fun remove(className: JvmClassName) { + storage.remove(className.getInternalName()) + } + + public fun isPackagePart(className: JvmClassName): Boolean { + return storage.containsMapping(className.getInternalName()) + } + + override fun dumpValue(value: Boolean) = "" + } + + private inner class SourceToClassesMap : BasicMap>() { + override fun createMap(): PersistentHashMap> = PersistentHashMap( + File(baseDir, SOURCE_TO_CLASSES), + EnumeratorStringDescriptor(), + StringListExternalizer + ) + + public fun clearOutputsForSource(sourceFile: File) { + storage.remove(sourceFile.getAbsolutePath()) + } + + public fun addSourceToClass(sourceFile: File, className: JvmClassName) { + storage.appendData(sourceFile.getAbsolutePath(), { out -> IOUtil.writeUTF(out, className.getInternalName()) }) + } + + public fun get(sourceFile: File): Collection { + return storage[sourceFile.getAbsolutePath()].orEmpty().map { JvmClassName.byInternalName(it) } + } + + override fun dumpValue(value: List) = value.toString() + } + + private inner class DirtyOutputClassesMap : BasicMap() { + override fun createMap(): PersistentHashMap = PersistentHashMap( + File(baseDir, DIRTY_OUTPUT_CLASSES), + EnumeratorStringDescriptor(), + BooleanDataDescriptor.INSTANCE + ) + + public fun markDirty(className: String) { + storage.put(className, true) + } + + public fun notDirty(className: String) { + storage.remove(className) + } + + public fun getDirtyOutputClasses(): Collection { return storage.getAllKeysWithExistingMapping() } @@ -615,3 +633,17 @@ private object ByteArrayExternalizer : DataExternalizer { return buf } } + +private object StringListExternalizer : DataExternalizer> { + override fun save(out: DataOutput, value: List) { + value.forEach { IOUtil.writeUTF(out, it) } + } + + override fun read(`in`: DataInput): List { + val result = ArrayList() + while ((`in` as DataInputStream).available() > 0) { + result.add(IOUtil.readUTF(`in`)) + } + return result + } +}