From 581bc898492d3c086147e02c42311a2152d2ddd8 Mon Sep 17 00:00:00 2001 From: "Alexander.Likhachev" Date: Tue, 10 Jan 2023 00:52:21 +0100 Subject: [PATCH] [IC] Introduce ICContext to simplify configuration propagation to caches #KT-49785 In Progress --- .../incremental/AbstractIncrementalCache.kt | 17 ++--- .../IncrementalCompilationContext.kt | 36 +++++++++ .../kotlin/incremental/IncrementalJsCache.kt | 40 +++++----- .../kotlin/incremental/IncrementalJvmCache.kt | 76 ++++++++++++------- .../kotlin/incremental/LookupStorage.kt | 29 +++---- .../kotlin/incremental/storage/BasicMap.kt | 27 ++++--- .../incremental/storage/ClassAttributesMap.kt | 6 +- .../incremental/storage/ClassOneToManyMap.kt | 17 ++++- .../storage/ComplementarySourceFilesMap.kt | 5 +- .../incremental/storage/DirtyClassesMaps.kt | 17 ++++- .../kotlin/incremental/storage/FileToIdMap.kt | 5 +- .../kotlin/incremental/storage/IdToFileMap.kt | 5 +- .../kotlin/incremental/storage/LookupMap.kt | 13 +++- .../storage/SourceToJsOutputMap.kt | 6 +- .../incremental/storage/SourceToOutputMaps.kt | 13 ++-- .../storage/RelocatableCachesTest.kt | 9 ++- .../incremental/IncrementalCachesManager.kt | 60 ++++----------- .../incremental/IncrementalCompilerRunner.kt | 32 +++++--- .../IncrementalFirJvmCompilerRunner.kt | 2 +- .../IncrementalJsCompilerRunner.kt | 18 ++--- .../IncrementalJvmCompilerRunner.kt | 17 +++-- .../kotlin/incremental/InputsCache.kt | 18 ++--- .../classpathDiff/ClasspathChangesComputer.kt | 3 +- .../incremental/snapshots/FileSnapshotMap.kt | 6 +- .../storage/SourceToOutputFilesMap.kt | 6 +- .../snapshots/FileSnapshotMapTest.kt | 9 ++- .../storage/SourceToOutputFilesMapTest.kt | 6 +- .../kotlin/jps/build/KotlinCompileContext.kt | 7 +- .../jps/incremental/JpsIncrementalCache.kt | 13 ++-- .../jps/incremental/JpsLookupStorage.kt | 14 ++-- .../jps/targets/KotlinJsModuleBuildTarget.kt | 2 +- .../jps/targets/KotlinJvmModuleBuildTarget.kt | 2 +- 32 files changed, 311 insertions(+), 225 deletions(-) create mode 100644 build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt diff --git a/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt index 6c7e699a6b5..90a8cf32cbd 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt @@ -51,7 +51,7 @@ interface IncrementalCacheCommon { */ abstract class AbstractIncrementalCache( workingDir: File, - protected val pathConverter: FileToPathConverter + icContext: IncrementalCompilationContext, ) : BasicMapsOwner(workingDir), IncrementalCacheCommon { companion object { private const val CLASS_ATTRIBUTES = "class-attributes" @@ -78,10 +78,10 @@ abstract class AbstractIncrementalCache( result } - internal val classAttributesMap = registerMap(ClassAttributesMap(CLASS_ATTRIBUTES.storageFile)) - private val subtypesMap = registerMap(SubtypesMap(SUBTYPES.storageFile)) - private val supertypesMap = registerMap(SupertypesMap(SUPERTYPES.storageFile)) - protected val classFqNameToSourceMap = registerMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile, pathConverter)) + internal val classAttributesMap = registerMap(ClassAttributesMap(CLASS_ATTRIBUTES.storageFile, icContext)) + private val subtypesMap = registerMap(SubtypesMap(SUBTYPES.storageFile, icContext)) + private val supertypesMap = registerMap(SupertypesMap(SUPERTYPES.storageFile, icContext)) + protected val classFqNameToSourceMap = registerMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile, icContext)) internal abstract val sourceToClassesMap: AbstractSourceToOutputMap internal abstract val dirtyOutputClassesMap: AbstractDirtyClassesMap @@ -91,7 +91,7 @@ abstract class AbstractIncrementalCache( * about missing parts. * TODO: provide a better solution (maintain an index of expect/actual declarations akin to IncrementalPackagePartProvider) */ - private val complementaryFilesMap = registerMap(ComplementarySourceFilesMap(COMPLEMENTARY_FILES.storageFile, pathConverter)) + private val complementaryFilesMap = registerMap(ComplementarySourceFilesMap(COMPLEMENTARY_FILES.storageFile, icContext)) override fun classesFqNamesBySources(files: Iterable): Collection = files.flatMapTo(HashSet()) { sourceToClassesMap.getFqNames(it) } @@ -187,9 +187,8 @@ abstract class AbstractIncrementalCache( protected class ClassFqNameToSourceMap( storageFile: File, - private val pathConverter: FileToPathConverter - ) : BasicStringMap(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) { - + icContext: IncrementalCompilationContext, + ) : BasicStringMap(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor, icContext) { operator fun set(fqName: FqName, sourceFile: File) { storage[fqName.asString()] = pathConverter.toPath(sourceFile) } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt new file mode 100644 index 00000000000..9450e675638 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.build.report.DoNothingICReporter +import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.incremental.storage.FileToPathConverter +import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter +import java.io.File + +private fun createDefaultPathConverter(rootProjectDir: File?) = IncrementalFileToPathConverter(rootProjectDir) + +class IncrementalCompilationContext( + val pathConverter: FileToPathConverter, + val storeFullFqNamesInLookupCache: Boolean = false, + val transaction: CompilationTransaction = DummyCompilationTransaction(), + val reporter: ICReporter = DoNothingICReporter, + val trackChangesInLookupCache: Boolean = false, +) { + constructor( + rootProjectDir: File?, + storeFullFqNamesInLookupCache: Boolean = false, + transaction: CompilationTransaction = DummyCompilationTransaction(), + reporter: ICReporter = DoNothingICReporter, + trackChangesInLookupCache: Boolean = false, + ) : this( + createDefaultPathConverter(rootProjectDir), + storeFullFqNamesInLookupCache, + transaction, + reporter, + trackChangesInLookupCache, + ) +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt index 51f44929c24..cdffbabafd3 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt @@ -40,10 +40,9 @@ import java.io.File open class IncrementalJsCache( cachesDir: File, - pathConverter: FileToPathConverter, + private val icContext: IncrementalCompilationContext, serializerProtocol: SerializerExtensionProtocol, - private val transaction: CompilationTransaction, -) : AbstractIncrementalCache(cachesDir, pathConverter) { +) : AbstractIncrementalCache(cachesDir, icContext) { companion object { private const val TRANSLATION_RESULT_MAP = "translation-result" private const val IR_TRANSLATION_RESULT_MAP = "ir-translation-result" @@ -57,13 +56,13 @@ open class IncrementalJsCache( private val protoData = ProtoDataProvider(serializerProtocol) - override val sourceToClassesMap = registerMap(SourceToFqNameMap(SOURCE_TO_CLASSES.storageFile, pathConverter)) - override val dirtyOutputClassesMap = registerMap(DirtyClassesFqNameMap(DIRTY_OUTPUT_CLASSES.storageFile)) - private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, pathConverter, protoData)) - private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, pathConverter)) - private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter)) - private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile)) - private val sourceToJsOutputsMap = registerMap(SourceToJsOutputMap(SOURCE_TO_JS_OUTPUT.storageFile, pathConverter)) + override val sourceToClassesMap = registerMap(SourceToFqNameMap(SOURCE_TO_CLASSES.storageFile, icContext)) + override val dirtyOutputClassesMap = registerMap(DirtyClassesFqNameMap(DIRTY_OUTPUT_CLASSES.storageFile, icContext)) + private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, protoData, icContext)) + private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, icContext)) + private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, icContext)) + private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile, icContext)) + private val sourceToJsOutputsMap = registerMap(SourceToJsOutputMap(SOURCE_TO_JS_OUTPUT.storageFile, icContext)) private val dirtySources = hashSetOf() @@ -73,7 +72,7 @@ open class IncrementalJsCache( var header: ByteArray get() = headerFile.readBytes() set(value) { - transaction.registerAddedOrChangedFile(headerFile.toPath()) + icContext.transaction.registerAddedOrChangedFile(headerFile.toPath()) cachesDir.mkdirs() headerFile.writeBytes(value) } @@ -229,10 +228,10 @@ private object TranslationResultValueExternalizer : DataExternalizer(storageFile, TranslationResultValueExternalizer) { + BasicStringMap(storageFile, TranslationResultValueExternalizer, icContext) { override fun dumpValue(value: TranslationResultValue): String = "Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}, InlineData: ${value.inlineData.md5()}" @@ -313,9 +312,9 @@ private object IrTranslationResultValueExternalizer : DataExternalizer(storageFile, IrTranslationResultValueExternalizer) { + BasicStringMap(storageFile, IrTranslationResultValueExternalizer, icContext) { override fun dumpValue(value: IrTranslationResultValue): String = "Filedata: ${value.fileData.md5()}, " + "Types: ${value.types.md5()}, " + @@ -395,8 +394,8 @@ fun getProtoData(sourceFile: File, metadata: ByteArray): Map private class InlineFunctionsMap( storageFile: File, - private val pathConverter: FileToPathConverter -) : BasicStringMap>(storageFile, StringToLongMapExternalizer) { + icContext: IncrementalCompilationContext, +) : BasicStringMap>(storageFile, StringToLongMapExternalizer, icContext) { @Synchronized fun process(srcFile: File, newMap: Map, changesCollector: ChangesCollector) { val key = pathConverter.toPath(srcFile) @@ -439,7 +438,10 @@ private object ByteArrayExternalizer : DataExternalizer { } -private class PackageMetadataMap(storageFile: File) : BasicStringMap(storageFile, ByteArrayExternalizer) { +private class PackageMetadataMap( + storageFile: File, + icContext: IncrementalCompilationContext, +) : BasicStringMap(storageFile, ByteArrayExternalizer, icContext) { fun put(packageName: String, newMetadata: ByteArray) { storage[packageName] = newMetadata } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt index 2a1f27aa7a3..f4c38ce703b 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt @@ -44,11 +44,11 @@ const val KOTLIN_CACHE_DIRECTORY_NAME = "kotlin" open class IncrementalJvmCache( targetDataRoot: File, + icContext: IncrementalCompilationContext, targetOutputDir: File?, - pathConverter: FileToPathConverter ) : AbstractIncrementalCache( workingDir = File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME), - pathConverter = pathConverter + icContext, ), IncrementalCache { companion object { private const val PROTO_MAP = "proto" @@ -64,20 +64,20 @@ open class IncrementalJvmCache( private const val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT } - override val sourceToClassesMap = registerMap(SourceToJvmNameMap(SOURCE_TO_CLASSES.storageFile, pathConverter)) - override val dirtyOutputClassesMap = registerMap(DirtyClassesJvmNameMap(DIRTY_OUTPUT_CLASSES.storageFile)) + override val sourceToClassesMap = registerMap(SourceToJvmNameMap(SOURCE_TO_CLASSES.storageFile, icContext)) + override val dirtyOutputClassesMap = registerMap(DirtyClassesJvmNameMap(DIRTY_OUTPUT_CLASSES.storageFile, icContext)) - private val protoMap = registerMap(ProtoMap(PROTO_MAP.storageFile)) - private val feProtoMap = registerMap(ProtoMap(FE_PROTO_MAP.storageFile)) - private val constantsMap = registerMap(ConstantsMap(CONSTANTS_MAP.storageFile)) - private val packagePartMap = registerMap(PackagePartMap(PACKAGE_PARTS.storageFile)) - private val multifileFacadeToParts = registerMap(MultifileClassFacadeMap(MULTIFILE_CLASS_FACADES.storageFile)) - private val partToMultifileFacade = registerMap(MultifileClassPartMap(MULTIFILE_CLASS_PARTS.storageFile)) - private val inlineFunctionsMap = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile)) + private val protoMap = registerMap(ProtoMap(PROTO_MAP.storageFile, icContext)) + private val feProtoMap = registerMap(ProtoMap(FE_PROTO_MAP.storageFile, icContext)) + private val constantsMap = registerMap(ConstantsMap(CONSTANTS_MAP.storageFile, icContext)) + private val packagePartMap = registerMap(PackagePartMap(PACKAGE_PARTS.storageFile, icContext)) + private val multifileFacadeToParts = registerMap(MultifileClassFacadeMap(MULTIFILE_CLASS_FACADES.storageFile, icContext)) + private val partToMultifileFacade = registerMap(MultifileClassPartMap(MULTIFILE_CLASS_PARTS.storageFile, icContext)) + private val inlineFunctionsMap = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, icContext)) // todo: try to use internal names only? - private val internalNameToSource = registerMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile, pathConverter)) + private val internalNameToSource = registerMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile, icContext)) // gradle only - private val javaSourcesProtoMap = registerMap(JavaSourcesProtoMap(JAVA_SOURCES_PROTO_MAP.storageFile)) + private val javaSourcesProtoMap = registerMap(JavaSourcesProtoMap(JAVA_SOURCES_PROTO_MAP.storageFile, icContext)) private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory" } } @@ -328,7 +328,10 @@ open class IncrementalJvmCache( return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]?.bytes } - private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { + private inner class ProtoMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : BasicStringMap(storageFile, ProtoMapValueExternalizer, icContext) { @Synchronized fun process(kotlinClassInfo: KotlinClassInfo, changesCollector: ChangesCollector) { @@ -396,8 +399,11 @@ open class IncrementalJvmCache( } } - private inner class JavaSourcesProtoMap(storageFile: File) : - BasicStringMap(storageFile, JavaClassProtoMapValueExternalizer) { + private inner class JavaSourcesProtoMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : + BasicStringMap(storageFile, JavaClassProtoMapValueExternalizer, icContext) { @Synchronized fun process(jvmClassName: JvmClassName, newData: SerializedJavaClass, changesCollector: ChangesCollector) { @@ -431,8 +437,11 @@ open class IncrementalJvmCache( } // todo: reuse code with InlineFunctionsMap? - private inner class ConstantsMap(storageFile: File) : - BasicStringMap>(storageFile, MapExternalizer(StringExternalizer, ConstantValueExternalizer)) { + private inner class ConstantsMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : + BasicStringMap>(storageFile, MapExternalizer(StringExternalizer, ConstantValueExternalizer), icContext) { operator fun contains(className: JvmClassName): Boolean = className.internalName in storage @@ -484,7 +493,10 @@ open class IncrementalJvmCache( value.dumpMap(Any::toString) } - private inner class PackagePartMap(storageFile: File) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE) { + private inner class PackagePartMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE, icContext) { fun addPackagePart(className: JvmClassName) { storage[className.internalName] = true } @@ -499,8 +511,11 @@ open class IncrementalJvmCache( override fun dumpValue(value: Boolean) = "" } - private inner class MultifileClassFacadeMap(storageFile: File) : - BasicStringMap>(storageFile, StringCollectionExternalizer) { + private inner class MultifileClassFacadeMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : + BasicStringMap>(storageFile, StringCollectionExternalizer, icContext) { @Synchronized operator fun set(className: JvmClassName, partNames: Collection) { @@ -521,8 +536,11 @@ open class IncrementalJvmCache( override fun dumpValue(value: Collection): String = value.dumpCollection() } - private inner class MultifileClassPartMap(storageFile: File) : - BasicStringMap(storageFile, EnumeratorStringDescriptor.INSTANCE) { + private inner class MultifileClassPartMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : + BasicStringMap(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { @Synchronized fun set(partName: String, facadeName: String) { @@ -542,8 +560,8 @@ open class IncrementalJvmCache( inner class InternalNameToSourcesMap( storageFile: File, - private val pathConverter: FileToPathConverter - ) : BasicStringMap>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer) { + icContext: IncrementalCompilationContext, + ) : BasicStringMap>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer, icContext) { operator fun set(internalName: String, sourceFiles: Collection) { storage[internalName] = pathConverter.toPaths(sourceFiles) } @@ -559,10 +577,14 @@ open class IncrementalJvmCache( value.dumpCollection() } - private inner class InlineFunctionsMap(storageFile: File) : + private inner class InlineFunctionsMap( + storageFile: File, + icContext: IncrementalCompilationContext, + ) : BasicStringMap>( storageFile, - MapExternalizer(InlineFunctionOrAccessorExternalizer, LongExternalizer) + MapExternalizer(InlineFunctionOrAccessorExternalizer, LongExternalizer), + icContext ) { @Synchronized diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index 62b3463ace7..b73cc928fbf 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -32,22 +32,22 @@ import java.util.* open class LookupStorage( targetDataDir: File, - pathConverter: FileToPathConverter, - storeFullFqNames: Boolean = false, - private val trackChanges: Boolean = false, - private val transaction: CompilationTransaction = DummyCompilationTransaction(), + private val icContext: IncrementalCompilationContext, ) : BasicMapsOwner(targetDataDir) { val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder") companion object { - private val DELETED_TO_SIZE_TRESHOLD = 0.5 - private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000 + private const val DELETED_TO_SIZE_THRESHOLD = 0.5 + private const val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000 } + private val trackChanges + get() = icContext.trackChangesInLookupCache + private val countersFile = "counters".storageFile - private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter)) - private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter)) - private val lookupMap = TrackedLookupMap(registerMap(LookupMap("lookups".storageFile, storeFullFqNames)), trackChanges) + private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, icContext)) + private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, icContext)) + private val lookupMap = TrackedLookupMap(registerMap(LookupMap("lookups".storageFile, icContext)), trackChanges) @Volatile private var size: Int = 0 @@ -57,8 +57,9 @@ open class LookupStorage( try { if (countersFile.exists()) { val lines = countersFile.readLines() - size = lines.firstOrNull()?.toIntOrNull() ?: throw IOException("$countersFile exists, but it is empty. " + - "Counters file is corrupted" + size = lines.firstOrNull()?.toIntOrNull() ?: throw IOException( + "$countersFile exists, but it is empty. " + + "Counters file is corrupted" ) oldSize = size } @@ -104,7 +105,7 @@ open class LookupStorage( } - if (size > MINIMUM_GARBAGE_COLLECTIBLE_SIZE && filtered.size.toDouble() / fileIds.size.toDouble() < DELETED_TO_SIZE_TRESHOLD) { + if (size > MINIMUM_GARBAGE_COLLECTIBLE_SIZE && filtered.size.toDouble() / fileIds.size.toDouble() < DELETED_TO_SIZE_THRESHOLD) { lookupMap[key] = filtered } @@ -135,7 +136,7 @@ open class LookupStorage( @Synchronized override fun clean() { - transaction.deleteFile(countersFile.toPath()) + icContext.transaction.deleteFile(countersFile.toPath()) size = 0 @@ -147,7 +148,7 @@ open class LookupStorage( try { if (size != oldSize) { if (size > 0) { - transaction.registerAddedOrChangedFile(countersFile.toPath()) + icContext.transaction.registerAddedOrChangedFile(countersFile.toPath()) if (!countersFile.exists()) { countersFile.parentFile.mkdirs() countersFile.createNewFile() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt index fbec6c5262b..cc0ab9b8a0f 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt @@ -20,16 +20,21 @@ import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.KeyDescriptor import org.jetbrains.annotations.TestOnly +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.utils.Printer import java.io.File abstract class BasicMap, V>( - internal val storageFile: File, - keyDescriptor: KeyDescriptor, - valueExternalizer: DataExternalizer + internal val storageFile: File, + keyDescriptor: KeyDescriptor, + valueExternalizer: DataExternalizer, + protected val icContext: IncrementalCompilationContext, ) { protected val storage: LazyStorage = CachingLazyStorage(storageFile, keyDescriptor, valueExternalizer) + protected val pathConverter + get() = icContext.pathConverter + fun clean() { storage.clean() } @@ -74,14 +79,16 @@ abstract class BasicMap, V>( } abstract class BasicStringMap( - storageFile: File, - keyDescriptor: KeyDescriptor, - valueExternalizer: DataExternalizer -) : BasicMap(storageFile, keyDescriptor, valueExternalizer) { + storageFile: File, + keyDescriptor: KeyDescriptor, + valueExternalizer: DataExternalizer, + icContext: IncrementalCompilationContext, +) : BasicMap(storageFile, keyDescriptor, valueExternalizer, icContext) { constructor( - storageFile: File, - valueExternalizer: DataExternalizer - ) : this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer) + storageFile: File, + valueExternalizer: DataExternalizer, + icContext: IncrementalCompilationContext, + ) : this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer, icContext) override fun dumpKey(key: String): String = key } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassAttributesMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassAttributesMap.kt index 7b853709576..5094be4d25c 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassAttributesMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassAttributesMap.kt @@ -17,6 +17,7 @@ package org.jetbrains.kotlin.incremental.storage import com.intellij.util.io.DataExternalizer +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.name.FqName import java.io.DataInput import java.io.DataOutput @@ -35,8 +36,9 @@ internal object ICClassesAttributesExternalizer : DataExternalizer(storageFile, ICClassesAttributesExternalizer) { + storageFile: File, + icContext: IncrementalCompilationContext, +) : BasicStringMap(storageFile, ICClassesAttributesExternalizer, icContext) { override fun dumpValue(value: ICClassesAttributes): String = value.toString() operator fun set(key: FqName, value: ICClassesAttributes) { diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt index 8b2e64a2b56..5f6789276a3 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt @@ -16,11 +16,15 @@ package org.jetbrains.kotlin.incremental.storage +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.dumpCollection import org.jetbrains.kotlin.name.FqName import java.io.File -internal open class ClassOneToManyMap(storageFile: File) : BasicStringMap>(storageFile, StringCollectionExternalizer) { +internal open class ClassOneToManyMap( + storageFile: File, + icContext: IncrementalCompilationContext, +) : BasicStringMap>(storageFile, StringCollectionExternalizer, icContext) { override fun dumpValue(value: Collection): String = value.dumpCollection() @Synchronized @@ -56,5 +60,12 @@ internal open class ClassOneToManyMap(storageFile: File) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer) { + icContext: IncrementalCompilationContext, +) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) { operator fun set(sourceFile: File, complementaryFiles: Collection) { storage[pathConverter.toPath(sourceFile)] = pathConverter.toPaths(complementaryFiles) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/DirtyClassesMaps.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/DirtyClassesMaps.kt index a6673385706..85cdf4c11d4 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/DirtyClassesMaps.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/DirtyClassesMaps.kt @@ -17,16 +17,25 @@ package org.jetbrains.kotlin.incremental.storage import com.intellij.util.io.BooleanDataDescriptor +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName import java.io.File -internal class DirtyClassesJvmNameMap(storageFile: File) : AbstractDirtyClassesMap(JvmClassNameTransformer, storageFile) -internal class DirtyClassesFqNameMap(storageFile: File) : AbstractDirtyClassesMap(FqNameTransformer, storageFile) +internal class DirtyClassesJvmNameMap( + storageFile: File, + icContext: IncrementalCompilationContext, +) : AbstractDirtyClassesMap(JvmClassNameTransformer, storageFile, icContext) +internal class DirtyClassesFqNameMap( + storageFile: File, + icContext: IncrementalCompilationContext, +) : AbstractDirtyClassesMap(FqNameTransformer, storageFile, icContext) internal abstract class AbstractDirtyClassesMap( - private val nameTransformer: NameTransformer, storageFile: File -) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE) { + private val nameTransformer: NameTransformer, + storageFile: File, + icContext: IncrementalCompilationContext, +) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE, icContext) { fun markDirty(className: Name) { storage[nameTransformer.asString(className)] = true } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt index 1c3d750a02a..4352a1b9aed 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt @@ -16,12 +16,13 @@ package org.jetbrains.kotlin.incremental.storage +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import java.io.File internal class FileToIdMap( file: File, - private val pathConverter: FileToPathConverter -) : BasicStringMap(file, IntExternalizer) { + icContext: IncrementalCompilationContext, +) : BasicStringMap(file, IntExternalizer, icContext) { override fun dumpValue(value: Int): String = value.toString() operator fun get(file: File): Int? = storage[pathConverter.toPath(file)] diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt index e43469adea7..4e7a8799a66 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt @@ -18,12 +18,13 @@ package org.jetbrains.kotlin.incremental.storage import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.ExternalIntegerKeyDescriptor +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import java.io.File internal class IdToFileMap( file: File, - private val pathConverter: FileToPathConverter -) : BasicMap(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE) { + icContext: IncrementalCompilationContext, +) : BasicMap(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE, icContext) { override fun dumpKey(key: Int): String = key.toString() override fun dumpValue(value: String): String = value diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt index 9ac0720ba73..8abf9069666 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt @@ -16,10 +16,19 @@ package org.jetbrains.kotlin.incremental.storage +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import java.io.File -class LookupMap(storage: File, storeFullFqNames: Boolean) : - BasicMap>(storage, LookupSymbolKeyDescriptor(storeFullFqNames), IntCollectionExternalizer) { +class LookupMap( + storage: File, + icContext: IncrementalCompilationContext, +) : + BasicMap>( + storage, + LookupSymbolKeyDescriptor(icContext.storeFullFqNamesInLookupCache), + IntCollectionExternalizer, + icContext, + ) { override fun dumpKey(key: LookupSymbolKey): String = key.toString() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt index e9664ca8599..31206097a21 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt @@ -5,10 +5,14 @@ package org.jetbrains.kotlin.incremental.storage +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.dumpCollection import java.io.File -class SourceToJsOutputMap(storageFile: File, private val pathConverter: FileToPathConverter) : BasicStringMap>(storageFile, StringCollectionExternalizer) { +class SourceToJsOutputMap( + storageFile: File, + icContext: IncrementalCompilationContext, +) : BasicStringMap>(storageFile, StringCollectionExternalizer, icContext) { override fun dumpValue(value: Collection): String = value.dumpCollection() @Synchronized diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt index 0bfe06e18fa..90a9d20a833 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.incremental.storage +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.dumpCollection import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName @@ -23,19 +24,19 @@ import java.io.File internal class SourceToJvmNameMap( storageFile: File, - pathConverter: FileToPathConverter -) : AbstractSourceToOutputMap(JvmClassNameTransformer, storageFile, pathConverter) + icContext: IncrementalCompilationContext, +) : AbstractSourceToOutputMap(JvmClassNameTransformer, storageFile, icContext) internal class SourceToFqNameMap( storageFile: File, - pathConverter: FileToPathConverter -) : AbstractSourceToOutputMap(FqNameTransformer, storageFile, pathConverter) + icContext: IncrementalCompilationContext, +) : AbstractSourceToOutputMap(FqNameTransformer, storageFile, icContext) internal abstract class AbstractSourceToOutputMap( private val nameTransformer: NameTransformer, storageFile: File, - private val pathConverter: FileToPathConverter -) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer) { + icContext: IncrementalCompilationContext, +) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) { fun clearOutputsForSource(sourceFile: File) { remove(pathConverter.toPath(sourceFile)) } diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt index 03e2029e72d..15e382687ae 100644 --- a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt +++ b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt @@ -9,6 +9,7 @@ import com.intellij.util.containers.MultiMap import org.jetbrains.kotlin.TestWithWorkingDir import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.cli.common.toBooleanLenient +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.LookupStorage import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories @@ -50,11 +51,11 @@ class RelocatableCachesTest : TestWithWorkingDir() { private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean, storeFullFqNames: Boolean = false) { val storageRoot = projectRoot.storageRoot val fileToPathConverter = RelativeFileToPathConverter(projectRoot) - val lookupStorage = LookupStorage( - storageRoot, - fileToPathConverter, - storeFullFqNames = storeFullFqNames + val icContext = IncrementalCompilationContext( + pathConverter = fileToPathConverter, + storeFullFqNamesInLookupCache = storeFullFqNames, ) + val lookupStorage = LookupStorage(storageRoot, icContext) val files = LinkedHashSet() val symbols = LinkedHashSet() val lookups = MultiMap.createOrderedSet() diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 558e9a13ce6..81e1f94b7c5 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -17,22 +17,15 @@ package org.jetbrains.kotlin.incremental import com.google.common.io.Closer -import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner -import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol import java.io.Closeable import java.io.File abstract class IncrementalCachesManager>( + icContext: IncrementalCompilationContext, cachesRootDir: File, - rootProjectDir: File?, - protected val reporter: ICReporter, - transaction: CompilationTransaction, - storeFullFqNamesInLookupCache: Boolean = false, - trackChangesInLookupCache: Boolean = false ) : Closeable { - val pathConverter = IncrementalFileToPathConverter(rootProjectDir) private val caches = arrayListOf() private var isClosed = false @@ -46,15 +39,8 @@ abstract class IncrementalCachesManager( - cacheDirectory, - rootProjectDir, - reporter, - transaction, - storeFullFqNamesInLookupCache, - trackChangesInLookupCache -) { - private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() } - override val platformCache = IncrementalJvmCache(jvmCacheDir, outputDir, pathConverter).apply { registerCache() } + icContext: IncrementalCompilationContext, + outputDir: File?, + cachesRootDir: File, +) : IncrementalCachesManager(icContext, cachesRootDir) { + private val jvmCacheDir = File(cachesRootDir, "jvm").apply { mkdirs() } + override val platformCache = IncrementalJvmCache(jvmCacheDir, icContext, outputDir).apply { registerCache() } } class IncrementalJsCachesManager( - cachesRootDir: File, - rootProjectDir: File?, - reporter: ICReporter, + icContext: IncrementalCompilationContext, serializerProtocol: SerializerExtensionProtocol, - storeFullFqNamesInLookupCache: Boolean, - transaction: CompilationTransaction = DummyCompilationTransaction(), -) : IncrementalCachesManager( - cachesRootDir, - rootProjectDir, - reporter, - transaction, - storeFullFqNamesInLookupCache, -) { + cachesRootDir: File, +) : IncrementalCachesManager(icContext, cachesRootDir) { private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() } - override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol, transaction).apply { registerCache() } + override val platformCache = IncrementalJsCache(jsCacheFile, icContext, serializerProtocol).apply { registerCache() } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 26537996160..b8f0193eb21 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -80,7 +80,15 @@ abstract class IncrementalCompilerRunner< private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) protected open val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS - protected abstract fun createCacheManager(args: Args, projectDir: File?, transaction: CompilationTransaction): CacheManager + /** + * Creates an instance of [IncrementalCompilationContext] that holds common incremental compilation context mostly required for [CacheManager] + */ + protected abstract fun createIncrementalCompilationContext( + projectDir: File?, + transaction: CompilationTransaction + ): IncrementalCompilationContext + + protected abstract fun createCacheManager(icContext: IncrementalCompilationContext, args: Args): CacheManager protected abstract fun destinationDir(args: Args): File fun compile( @@ -159,7 +167,8 @@ abstract class IncrementalCompilerRunner< changedFiles as ChangedFiles.Known? createTransaction().use { transaction -> - val caches = createCacheManager(args, projectDir, transaction) + val icContext = createIncrementalCompilationContext(projectDir, transaction) + val caches = createCacheManager(icContext, args) fun compile(): ICResult { // Step 1: Get changed files @@ -199,13 +208,13 @@ abstract class IncrementalCompilerRunner< // Step 3: Compile incrementally val exitCode = try { compileImpl( + icContext, compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector, - transaction ) } catch (e: Throwable) { return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e) @@ -269,8 +278,8 @@ abstract class IncrementalCompilerRunner< reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" } cleanOrCreateDirectories(outputDirsToClean) } - val transaction = DummyCompilationTransaction() - return createCacheManager(args, projectDir, transaction).use { caches -> + val icContext = createIncrementalCompilationContext(projectDir, DummyCompilationTransaction()) + return createCacheManager(icContext, args).use { caches -> if (trackChangedFiles) { caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) } @@ -278,7 +287,7 @@ abstract class IncrementalCompilerRunner< AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args)) } else null - compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector, transaction) + compileImpl(icContext, CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector) } } @@ -386,18 +395,18 @@ abstract class IncrementalCompilerRunner< ): Pair> private fun compileImpl( + icContext: IncrementalCompilationContext, compilationMode: CompilationMode, allSourceFiles: List, args: Args, caches: CacheManager, abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true messageCollector: MessageCollector, - transaction: CompilationTransaction, ): ExitCode { performWorkBeforeCompilation(compilationMode, args) val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } - val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction) + val exitCode = doCompile(icContext, caches, compilationMode, allKotlinFiles, args, abiSnapshotData, messageCollector) performWorkAfterCompilation(compilationMode, exitCode, caches) return exitCode @@ -426,13 +435,13 @@ abstract class IncrementalCompilerRunner< } private fun doCompile( + icContext: IncrementalCompilationContext, + caches: CacheManager, compilationMode: CompilationMode, allKotlinSources: List, args: Args, - caches: CacheManager, abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true originalMessageCollector: MessageCollector, - transaction: CompilationTransaction, ): ExitCode { val dirtySources = when (compilationMode) { is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableLinkedSet() @@ -443,6 +452,7 @@ abstract class IncrementalCompilerRunner< val buildDirtyLookupSymbols = HashSet() val buildDirtyFqNames = HashSet() val allDirtySources = HashSet() + val transaction = icContext.transaction var exitCode = ExitCode.OK @@ -454,7 +464,7 @@ abstract class IncrementalCompilerRunner< val complementaryFiles = caches.platformCache.getComplementaryFilesRecursive(dirtySources) dirtySources.addAll(complementaryFiles) caches.platformCache.markDirty(dirtySources) - caches.inputsCache.removeOutputForSourceFiles(dirtySources, transaction) + caches.inputsCache.removeOutputForSourceFiles(dirtySources) val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING) val expectActualTracker = ExpectActualTrackerImpl() diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt index de0691bd1d5..e0d8b4898a2 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt @@ -259,7 +259,7 @@ class IncrementalFirJvmCompilerRunner( else -> it + newDirtyFilesOutputsScope } } - caches.inputsCache.removeOutputForSourceFiles(newDirtySources, DummyCompilationTransaction()) + caches.inputsCache.removeOutputForSourceFiles(newDirtySources) newDirtySources.forEach { dirtySources.add(KtIoFileSourceFile(it)) } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 59bba0d2318..9e58f705f7d 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -98,18 +98,16 @@ class IncrementalJsCompilerRunner( withAbiSnapshot = withAbiSnapshot, preciseCompilationResultsBackup = preciseCompilationResultsBackup, ) { - - override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJsCachesManager { - val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol - return IncrementalJsCachesManager( - cacheDirectory, - projectDir, - reporter, - serializerProtocol, - storeFullFqNamesInLookupCache = withAbiSnapshot, + override fun createIncrementalCompilationContext(projectDir: File?, transaction: CompilationTransaction) = + IncrementalCompilationContext( transaction = transaction, + rootProjectDir = projectDir, + reporter = reporter, + storeFullFqNamesInLookupCache = withAbiSnapshot, ) - } + + override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JSCompilerArguments) = + IncrementalJsCachesManager(icContext, if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol, cacheDirectory) override fun destinationDir(args: K2JSCompilerArguments): File { return if (args.isIrBackendEnabled()) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index c6847c563ea..59173736e46 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -150,17 +150,18 @@ open class IncrementalJvmCompilerRunner( withAbiSnapshot = withAbiSnapshot, preciseCompilationResultsBackup = preciseCompilationResultsBackup, ) { - override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJvmCachesManager = - IncrementalJvmCachesManager( - cacheDirectory, - projectDir, - File(args.destination), - reporter, - storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, - trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun, + override fun createIncrementalCompilationContext(projectDir: File?, transaction: CompilationTransaction) = + IncrementalCompilationContext( transaction = transaction, + rootProjectDir = projectDir, + reporter = reporter, + trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun, + storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, ) + override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments) = + IncrementalJvmCachesManager(icContext, args.destination?.let { File(it) }, cacheDirectory) + override fun destinationDir(args: K2JVMCompilerArguments): File = args.destinationAsFile diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt index 39086fd7bf2..5a5a9267c32 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt @@ -18,35 +18,29 @@ package org.jetbrains.kotlin.incremental import com.intellij.util.containers.MultiMap import org.jetbrains.kotlin.build.GeneratedFile -import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.build.report.debug import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner -import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.incremental.storage.SourceToOutputFilesMap import java.io.File class InputsCache( workingDir: File, - private val reporter: ICReporter, - pathConverter: FileToPathConverter + private val icContext: IncrementalCompilationContext, ) : BasicMapsOwner(workingDir) { companion object { private const val SOURCE_SNAPSHOTS = "source-snapshot" private const val SOURCE_TO_OUTPUT_FILES = "source-to-output" } - internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile, pathConverter)) - private val sourceToOutputMap = registerMap(SourceToOutputFilesMap(SOURCE_TO_OUTPUT_FILES.storageFile, pathConverter)) + internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile, icContext)) + private val sourceToOutputMap = registerMap(SourceToOutputFilesMap(SOURCE_TO_OUTPUT_FILES.storageFile, icContext)) - fun removeOutputForSourceFiles( - sources: Iterable, - transaction: CompilationTransaction, - ) { + fun removeOutputForSourceFiles(sources: Iterable) { for (sourceFile in sources) { sourceToOutputMap.remove(sourceFile).forEach { - reporter.debug { "Deleting $it on clearing cache for $sourceFile" } - transaction.deleteFile(it.toPath()) + icContext.reporter.debug { "Deleting $it on clearing cache for $sourceFile" } + icContext.transaction.deleteFile(it.toPath()) } } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt index 32b1003c6ff..9b70a5bbc61 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt @@ -203,7 +203,8 @@ object ClasspathChangesComputer { ): ProgramSymbolSet { val workingDir = FileUtil.createTempDirectory(this::class.java.simpleName, "_WorkingDir_${UUID.randomUUID()}", /* deleteOnExit */ true) - val incrementalJvmCache = IncrementalJvmCache(workingDir, /* targetOutputDir */ null, FileToAbsolutePathConverter) + val icContext = IncrementalCompilationContext(pathConverter = FileToAbsolutePathConverter) + val incrementalJvmCache = IncrementalJvmCache(workingDir, icContext, null) // Step 1: // - Add previous class snapshots to incrementalJvmCache. diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt index d1055887f14..6e7fc85b4a9 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt @@ -17,16 +17,16 @@ package org.jetbrains.kotlin.incremental.snapshots import org.jetbrains.kotlin.incremental.ChangedFiles +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.storage.BasicStringMap -import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor import java.io.File import java.util.* class FileSnapshotMap( storageFile: File, - private val pathConverter: FileToPathConverter -) : BasicStringMap(storageFile, PathStringDescriptor, FileSnapshotExternalizer) { + icContext: IncrementalCompilationContext, +) : BasicStringMap(storageFile, PathStringDescriptor, FileSnapshotExternalizer, icContext) { override fun dumpValue(value: FileSnapshot): String = value.toString() diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt index c917a2393aa..321aebcd29a 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt @@ -5,14 +5,14 @@ package org.jetbrains.kotlin.incremental.storage +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.dumpCollection import java.io.File class SourceToOutputFilesMap( storageFile: File, - private val pathConverter: FileToPathConverter -) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer) { - + icContext: IncrementalCompilationContext, +) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) { operator fun set(sourceFile: File, outputFiles: Collection) { storage[pathConverter.toPath(sourceFile)] = outputFiles.map(pathConverter::toPath) } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt index 5ddcffaefd2..0fdcb6ae005 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.incremental.snapshots import org.jetbrains.kotlin.TestWithWorkingDir +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter import org.junit.After @@ -17,15 +18,17 @@ import kotlin.properties.Delegates class FileSnapshotMapTest : TestWithWorkingDir() { private var snapshotMap: FileSnapshotMap by Delegates.notNull() - private var pathConverter: FileToPathConverter by Delegates.notNull() @Before override fun setUp() { super.setUp() val caches = File(workingDir, "caches").apply { mkdirs() } val snapshotMapFile = File(caches, "snapshots.tab") - pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile)) - snapshotMap = FileSnapshotMap(snapshotMapFile, pathConverter) + val pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile)) + val icContext = IncrementalCompilationContext( + pathConverter = pathConverter + ) + snapshotMap = FileSnapshotMap(snapshotMapFile, icContext) } @After diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt index b645fe017b6..3b8ddf48a4d 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.incremental.storage import org.jetbrains.kotlin.TestWithWorkingDir +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.junit.After import org.junit.Assert.assertArrayEquals import org.junit.Before @@ -23,7 +24,10 @@ class SourceToOutputFilesMapTest : TestWithWorkingDir() { val caches = File(workingDir, "caches").apply { mkdirs() } val stofMapFile = File(caches, "stof.tab") pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile)) - stofMap = SourceToOutputFilesMap(stofMapFile, pathConverter) + val icContext = IncrementalCompilationContext( + pathConverter = pathConverter + ) + stofMap = SourceToOutputFilesMap(stofMapFile, icContext) } @After diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt index cf75df7ba2c..873ded033d2 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt @@ -13,7 +13,10 @@ import org.jetbrains.jps.incremental.fs.CompilationRound import org.jetbrains.jps.incremental.messages.BuildMessage import org.jetbrains.jps.incremental.messages.CompilerMessage import org.jetbrains.kotlin.build.joinToReadableString +import org.jetbrains.kotlin.build.report.DoNothingICReporter import org.jetbrains.kotlin.config.CompilerRunnerConstants.KOTLIN_COMPILER_NAME +import org.jetbrains.kotlin.incremental.DummyCompilationTransaction +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.jps.KotlinJpsBundle @@ -81,7 +84,9 @@ class KotlinCompileContext(val jpsContext: CompileContext) { val fileToPathConverter: FileToPathConverter = JpsFileToPathConverter(jpsContext.projectDescriptor.project) - val lookupStorageManager = JpsLookupStorageManager(dataManager, fileToPathConverter) + val icContext = IncrementalCompilationContext(pathConverter = fileToPathConverter) + + val lookupStorageManager = JpsLookupStorageManager(dataManager, icContext) /** * Flag to prevent rebuilding twice. diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt index 4c29e54f50c..0e768f6df88 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt @@ -21,10 +21,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.StorageOwner -import org.jetbrains.kotlin.incremental.DummyCompilationTransaction -import org.jetbrains.kotlin.incremental.IncrementalCacheCommon -import org.jetbrains.kotlin.incremental.IncrementalJsCache -import org.jetbrains.kotlin.incremental.IncrementalJvmCache +import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.jps.build.KotlinBuilder import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget @@ -38,8 +35,8 @@ interface JpsIncrementalCache : IncrementalCacheCommon, StorageOwner { class JpsIncrementalJvmCache( target: ModuleBuildTarget, paths: BuildDataPaths, - pathConverter: FileToPathConverter -) : IncrementalJvmCache(paths.getTargetDataRoot(target), target.outputDir, pathConverter), JpsIncrementalCache { + icContext: IncrementalCompilationContext +) : IncrementalJvmCache(paths.getTargetDataRoot(target), icContext, target.outputDir), JpsIncrementalCache { override fun addJpsDependentCache(cache: JpsIncrementalCache) { if (cache is JpsIncrementalJvmCache) { addDependentCache(cache) @@ -54,8 +51,8 @@ class JpsIncrementalJvmCache( class JpsIncrementalJsCache( target: ModuleBuildTarget, paths: BuildDataPaths, - pathConverter: FileToPathConverter -) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol, DummyCompilationTransaction()), JpsIncrementalCache { + icContext: IncrementalCompilationContext +) : IncrementalJsCache(paths.getTargetDataRoot(target), icContext, JsSerializerProtocol), JpsIncrementalCache { override fun addJpsDependentCache(cache: JpsIncrementalCache) { if (cache is JpsIncrementalJsCache) { addDependentCache(cache) diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt index 0fd2eaaa7be..8675c6a5199 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt @@ -21,8 +21,8 @@ import org.jetbrains.jps.builders.storage.BuildDataCorruptedException import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.StorageOwner +import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.LookupStorage -import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import java.io.File import java.io.IOException @@ -30,9 +30,9 @@ private object LookupStorageLock class JpsLookupStorageManager( private val buildDataManager: BuildDataManager, - pathConverter: FileToPathConverter + icContext: IncrementalCompilationContext, ) { - private val storageProvider = JpsLookupStorageProvider(pathConverter) + private val storageProvider = JpsLookupStorageProvider(icContext) fun cleanLookupStorage(log: Logger) { synchronized(LookupStorageLock) { @@ -58,14 +58,14 @@ class JpsLookupStorageManager( } private class JpsLookupStorageProvider( - private val pathConverter: FileToPathConverter + private val icContext: IncrementalCompilationContext ) : StorageProvider() { override fun createStorage(targetDataDir: File): JpsLookupStorage = - JpsLookupStorage(targetDataDir, pathConverter) + JpsLookupStorage(targetDataDir, icContext) } private class JpsLookupStorage( targetDataDir: File, - pathConverter: FileToPathConverter - ) : StorageOwner, LookupStorage(targetDataDir, pathConverter) + icContext: IncrementalCompilationContext + ) : StorageOwner, LookupStorage(targetDataDir, icContext) } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt index 70997609492..927917973cd 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt @@ -214,7 +214,7 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu } override fun createCacheStorage(paths: BuildDataPaths) = - JpsIncrementalJsCache(jpsModuleBuildTarget, paths, kotlinContext.fileToPathConverter) + JpsIncrementalJsCache(jpsModuleBuildTarget, paths, kotlinContext.icContext) override fun updateCaches( dirtyFilesHolder: KotlinDirtySourceFilesHolder, diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt index c35433dce92..4ebfb408d79 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt @@ -60,7 +60,7 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB get() = IncrementalCompilation.isEnabledForJvm() override fun createCacheStorage(paths: BuildDataPaths) = - JpsIncrementalJvmCache(jpsModuleBuildTarget, paths, kotlinContext.fileToPathConverter) + JpsIncrementalJvmCache(jpsModuleBuildTarget, paths, kotlinContext.icContext) override val compilerArgumentsFileName get() = JVM_BUILD_META_INFO_FILE_NAME