[IC] Register JS IC header file and lookup counter in transaction

#KT-49785 In Progress

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt

@@ -41,7 +41,8 @@ import java.io.File
 open class IncrementalJsCache(
     cachesDir: File,
     pathConverter: FileToPathConverter,
-    serializerProtocol: SerializerExtensionProtocol
+    serializerProtocol: SerializerExtensionProtocol,
+    private val transaction: CompilationTransaction,
 ) : AbstractIncrementalCache<FqName>(cachesDir, pathConverter) {
     companion object {
         private const val TRANSLATION_RESULT_MAP = "translation-result"
@@ -72,6 +73,7 @@ open class IncrementalJsCache(
     var header: ByteArray
         get() = headerFile.readBytes()
         set(value) {
+            transaction.registerAddedOrChangedFile(headerFile.toPath())
             cachesDir.mkdirs()
             headerFile.writeBytes(value)
         }

build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt

@@ -34,7 +34,8 @@ open class LookupStorage(
     targetDataDir: File,
     pathConverter: FileToPathConverter,
     storeFullFqNames: Boolean = false,
-    private val trackChanges: Boolean = false
+    private val trackChanges: Boolean = false,
+    private val transaction: CompilationTransaction = DummyCompilationTransaction(),
 ) : BasicMapsOwner(targetDataDir) {
     val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")
 
@@ -134,9 +135,7 @@ open class LookupStorage(
 
     @Synchronized
     override fun clean() {
-        if (countersFile.exists()) {
+        transaction.deleteFile(countersFile.toPath())
-            countersFile.delete()
-        }
 
         size = 0
 
@@ -148,6 +147,7 @@ open class LookupStorage(
         try {
             if (size != oldSize) {
                 if (size > 0) {
+                    transaction.registerAddedOrChangedFile(countersFile.toPath())
                     if (!countersFile.exists()) {
                         countersFile.parentFile.mkdirs()
                         countersFile.createNewFile()

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -28,6 +28,7 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
     cachesRootDir: File,
     rootProjectDir: File?,
     protected val reporter: ICReporter,
+    transaction: CompilationTransaction,
     storeFullFqNamesInLookupCache: Boolean = false,
     trackChangesInLookupCache: Boolean = false
 ) : Closeable {
@@ -47,7 +48,13 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
 
     val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter, pathConverter).apply { registerCache() }
     val lookupCache: LookupStorage =
-        LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache, trackChangesInLookupCache).apply { registerCache() }
+        LookupStorage(
+            lookupCacheDir,
+            pathConverter,
+            storeFullFqNamesInLookupCache,
+            trackChangesInLookupCache,
+            transaction,
+        ).apply { registerCache() }
     abstract val platformCache: PlatformCache
 
     @Suppress("UnstableApiUsage")
@@ -81,11 +88,13 @@ class IncrementalJvmCachesManager(
     outputDir: File,
     reporter: ICReporter,
     storeFullFqNamesInLookupCache: Boolean = false,
-    trackChangesInLookupCache: Boolean = false
+    trackChangesInLookupCache: Boolean = false,
+    transaction: CompilationTransaction = DummyCompilationTransaction(),
 ) : IncrementalCachesManager<IncrementalJvmCache>(
     cacheDirectory,
     rootProjectDir,
     reporter,
+    transaction,
     storeFullFqNamesInLookupCache,
     trackChangesInLookupCache
 ) {
@@ -98,8 +107,15 @@ class IncrementalJsCachesManager(
     rootProjectDir: File?,
     reporter: ICReporter,
     serializerProtocol: SerializerExtensionProtocol,
-    storeFullFqNamesInLookupCache: Boolean
+    storeFullFqNamesInLookupCache: Boolean,
-) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, rootProjectDir, reporter, storeFullFqNamesInLookupCache) {
+    transaction: CompilationTransaction = DummyCompilationTransaction(),
+) : IncrementalCachesManager<IncrementalJsCache>(
+    cachesRootDir,
+    rootProjectDir,
+    reporter,
+    transaction,
+    storeFullFqNamesInLookupCache,
+) {
     private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() }
-    override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol).apply { registerCache() }
+    override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol, transaction).apply { registerCache() }
 }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -80,7 +80,7 @@ abstract class IncrementalCompilerRunner<
     private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
     protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
 
-    protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager
+    protected abstract fun createCacheManager(args: Args, projectDir: File?, transaction: CompilationTransaction): CacheManager
     protected abstract fun destinationDir(args: Args): File
 
     fun compile(
@@ -158,86 +158,95 @@ abstract class IncrementalCompilerRunner<
         }
         changedFiles as ChangedFiles.Known?
 
-        val caches = createCacheManager(args, projectDir)
+        createTransaction().use { transaction ->
+            val caches = createCacheManager(args, projectDir, transaction)
 
-        fun compile(): ICResult {
+            fun compile(): ICResult {
-            // Step 1: Get changed files
+                // Step 1: Get changed files
-            val knownChangedFiles: ChangedFiles.Known = try {
+                val knownChangedFiles: ChangedFiles.Known = try {
-                getChangedFiles(changedFiles, allSourceFiles, caches)
+                    getChangedFiles(changedFiles, allSourceFiles, caches)
-            } catch (e: Throwable) {
+                } catch (e: Throwable) {
-                return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
+                    return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
-            }
-
-            val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
-
-            // Step 2: Compute files to recompile
-            val compilationMode = try {
-                reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
-                    calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
                 }
-            } catch (e: Throwable) {
-                return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
-            }
 
-            if (compilationMode is CompilationMode.Rebuild) {
+                val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
-                return ICResult.RequiresRebuild(compilationMode.reason)
-            }
 
-            val abiSnapshotData = if (withAbiSnapshot) {
+                // Step 2: Compute files to recompile
-                if (!abiSnapshotFile.exists()) {
+                val compilationMode = try {
-                    reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
+                    reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
-                    return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
+                        calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
+                    }
+                } catch (e: Throwable) {
+                    return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
                 }
-                reporter.info { "Incremental compilation with ABI snapshot enabled" }
-                AbiSnapshotData(
-                    snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
-                    classpathAbiSnapshot = classpathAbiSnapshot!!
-                )
-            } else null
 
-            // Step 3: Compile incrementally
+                if (compilationMode is CompilationMode.Rebuild) {
-            val exitCode = try {
+                    return ICResult.RequiresRebuild(compilationMode.reason)
-                compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector)
+                }
-            } catch (e: Throwable) {
+
-                return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
+                val abiSnapshotData = if (withAbiSnapshot) {
+                    if (!abiSnapshotFile.exists()) {
+                        reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
+                        return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
+                    }
+                    reporter.info { "Incremental compilation with ABI snapshot enabled" }
+                    AbiSnapshotData(
+                        snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
+                        classpathAbiSnapshot = classpathAbiSnapshot!!
+                    )
+                } else null
+
+                // Step 3: Compile incrementally
+                val exitCode = try {
+                    compileImpl(
+                        compilationMode as CompilationMode.Incremental,
+                        allSourceFiles,
+                        args,
+                        caches,
+                        abiSnapshotData,
+                        messageCollector,
+                        transaction
+                    )
+                } catch (e: Throwable) {
+                    return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
+                }
+
+                return ICResult.Completed(exitCode)
             }
 
-            return ICResult.Completed(exitCode)
+            fun closeCaches(caches: CacheManager, activeException: Throwable) {
-        }
+                try {
+                    caches.close()
+                } catch (e: Throwable) {
+                    activeException.addSuppressed(e)
+                }
+            }
 
-        fun closeCaches(caches: CacheManager, activeException: Throwable) {
+            // Because `caches` is a Closeable resource, it is important to close them in both cases:
+            //    1. in the event of an exception
+            //    2. after a normal execution
+            // Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the
+            // future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in
+            // `compileNonIncrementally`).
+            val icResult = try {
+                compile()
+            } catch (e: Throwable) {
+                // Case 1 - Close the caches upon an exception
+                closeCaches(caches, e)
+                throw e
+            }
+
+            // Case 2 - Close the caches after a normal execution
             try {
                 caches.close()
             } catch (e: Throwable) {
-                activeException.addSuppressed(e)
+                return ICResult.Failed(
+                    IC_FAILED_TO_CLOSE_CACHES,
+                    RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e)
+                )
             }
+            return icResult
         }
-
-        // Because `caches` is a Closeable resource, it is important to close them in both cases:
-        //    1. in the event of an exception
-        //    2. after a normal execution
-        // Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the
-        // future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in
-        // `compileNonIncrementally`).
-        val icResult = try {
-            compile()
-        } catch (e: Throwable) {
-            // Case 1 - Close the caches upon an exception
-            closeCaches(caches, e)
-            throw e
-        }
-
-        // Case 2 - Close the caches after a normal execution
-        try {
-            caches.close()
-        } catch (e: Throwable) {
-            return ICResult.Failed(
-                IC_FAILED_TO_CLOSE_CACHES,
-                RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e)
-            )
-        }
-
-        return icResult
     }
 
     private fun compileNonIncrementally(
@@ -257,7 +266,8 @@ abstract class IncrementalCompilerRunner<
             reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" }
             cleanOrCreateDirectories(outputDirsToClean)
         }
-        return createCacheManager(args, projectDir).use { caches ->
+        val transaction = DummyCompilationTransaction()
+        return createCacheManager(args, projectDir, transaction).use { caches ->
             if (trackChangedFiles) {
                 caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
             }
@@ -265,7 +275,7 @@ abstract class IncrementalCompilerRunner<
                 AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
             } else null
 
-            compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector)
+            compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector, transaction)
         }
     }
 
@@ -378,14 +388,13 @@ abstract class IncrementalCompilerRunner<
         args: Args,
         caches: CacheManager,
         abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
-        messageCollector: MessageCollector
+        messageCollector: MessageCollector,
+        transaction: CompilationTransaction,
     ): ExitCode {
         performWorkBeforeCompilation(compilationMode, args)
 
         val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
-        val exitCode = createTransaction().use { transaction ->
+        val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)
-            doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)
-        }
 
         performWorkAfterCompilation(compilationMode, exitCode, caches)
         return exitCode

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -99,14 +99,15 @@ class IncrementalJsCompilerRunner(
     preciseCompilationResultsBackup = preciseCompilationResultsBackup,
 ) {
 
-    override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
+    override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJsCachesManager {
         val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
         return IncrementalJsCachesManager(
             cacheDirectory,
             projectDir,
             reporter,
             serializerProtocol,
-            storeFullFqNamesInLookupCache = withAbiSnapshot
+            storeFullFqNamesInLookupCache = withAbiSnapshot,
+            transaction = transaction,
         )
     }
 

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -150,14 +150,15 @@ open class IncrementalJvmCompilerRunner(
     withAbiSnapshot = withAbiSnapshot,
     preciseCompilationResultsBackup = preciseCompilationResultsBackup,
 ) {
-    override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
+    override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJvmCachesManager =
         IncrementalJvmCachesManager(
             cacheDirectory,
             projectDir,
             File(args.destination),
             reporter,
             storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
-            trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
+            trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun,
+            transaction = transaction,
         )
 
     override fun destinationDir(args: K2JVMCompilerArguments): File =

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt

@@ -21,6 +21,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider
 import org.jetbrains.jps.incremental.ModuleBuildTarget
 import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.incremental.storage.StorageOwner
+import org.jetbrains.kotlin.incremental.DummyCompilationTransaction
 import org.jetbrains.kotlin.incremental.IncrementalCacheCommon
 import org.jetbrains.kotlin.incremental.IncrementalJsCache
 import org.jetbrains.kotlin.incremental.IncrementalJvmCache
@@ -54,7 +55,7 @@ class JpsIncrementalJsCache(
     target: ModuleBuildTarget,
     paths: BuildDataPaths,
     pathConverter: FileToPathConverter
-) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol), JpsIncrementalCache {
+) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol, DummyCompilationTransaction()), JpsIncrementalCache {
     override fun addJpsDependentCache(cache: JpsIncrementalCache) {
         if (cache is JpsIncrementalJsCache) {
             addDependentCache(cache)