[JS IR] Use a guard file for preventing incremental caches corruption

If a developer interrupts (kill gradle process) a compilation process or
 an internal error interrupts the compilation process,
 the incremental cache files may be corrupted.
 The patch creates a special guard file, which allows detecting
 if the previous compilation was not successful,
 and in case of any issues drops the incremental cache files.

^KT-56581 Fixed

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -61,10 +61,7 @@ import org.jetbrains.kotlin.incremental.js.IncrementalNextRoundChecker
 import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
 import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
-import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdater
+import org.jetbrains.kotlin.ir.backend.js.ic.*
-import org.jetbrains.kotlin.ir.backend.js.ic.DirtyFileState
-import org.jetbrains.kotlin.ir.backend.js.ic.JsExecutableProducer
-import org.jetbrains.kotlin.ir.backend.js.ic.ModuleArtifact
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerIr
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputsBuilt
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
@@ -298,20 +295,30 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             messageCollector.report(INFO, "Produce executable: $outputDirPath")
             messageCollector.report(INFO, "Cache directory: ${arguments.cacheDirectory}")
 
-            if (icCaches.isNotEmpty()) {
+            if (icCaches != null) {
                 val beforeIc2Js = System.currentTimeMillis()
 
+                // We use one cache directory for both caches: JS AST and JS code.
+                // This guard MUST be unlocked after a successful preparing icCaches (see prepareIcCaches()).
+                // Do not use IncrementalCacheGuard::acquire() - it may drop an entire cache here, and
+                // it breaks the logic from JsExecutableProducer(), therefore use IncrementalCacheGuard::tryAcquire() instead
+                // TODO: One day, when we will lower IR and produce JS AST per module,
+                //      think about using different directories for JS AST and JS code.
+                icCaches.cacheGuard.tryAcquire()
+
                 val jsExecutableProducer = JsExecutableProducer(
                     mainModuleName = moduleName,
                     moduleKind = moduleKind,
                     sourceMapsInfo = SourceMapsInfo.from(configurationJs),
-                    caches = icCaches,
+                    caches = icCaches.artifacts,
                     relativeRequirePath = true
                 )
 
                 val (outputs, rebuiltModules) = jsExecutableProducer.buildExecutable(arguments.irPerModule, outJsProgram = false)
                 outputs.writeAll(outputDir, outputName, arguments.generateDts, moduleName, moduleKind)
 
+                icCaches.cacheGuard.release()
+
                 messageCollector.report(INFO, "Executable production duration (IC): ${System.currentTimeMillis() - beforeIc2Js}ms")
                 for ((event, duration) in jsExecutableProducer.getStopwatchLaps()) {
                     messageCollector.report(INFO, "  $event: ${(duration / 1e6).toInt()}ms")
@@ -609,6 +616,8 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
         )
     }
 
+    class IcCachesArtifacts(val artifacts: List<ModuleArtifact>, val cacheGuard: IncrementalCacheGuard)
+
     private fun prepareIcCaches(
         arguments: K2JSCompilerArguments,
         messageCollector: MessageCollector,
@@ -617,11 +626,17 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
         friendLibraries: List<String>,
         configurationJs: CompilerConfiguration,
         mainCallArguments: List<String>?
-    ): List<ModuleArtifact> {
+    ): IcCachesArtifacts? {
         val cacheDirectory = arguments.cacheDirectory
 
         // TODO: Use JS IR IC infrastructure for WASM?
-        val icCaches = if (!arguments.wasm && cacheDirectory != null) {
+        if (!arguments.wasm && cacheDirectory != null) {
+            val cacheGuard = IncrementalCacheGuard(cacheDirectory).also {
+                if (it.acquire() == IncrementalCacheGuard.AcquireStatus.CACHE_CLEARED) {
+                    messageCollector.report(INFO, "Cache guard file detected, cache directory '$cacheDirectory' cleared")
+                }
+            }
+
             messageCollector.report(INFO, "")
             messageCollector.report(INFO, "Building cache:")
             messageCollector.report(INFO, "to: $outputDir")
@@ -650,6 +665,8 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             )
 
             val artifacts = cacheUpdater.actualizeCaches()
+            cacheGuard.release()
+
             messageCollector.report(INFO, "IC rebuilt overall time: ${System.currentTimeMillis() - start}ms")
             for ((event, duration) in cacheUpdater.getStopwatchLastLaps()) {
                 messageCollector.report(INFO, "  $event: ${(duration / 1e6).toInt()}ms")
@@ -678,9 +695,9 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
                 }
             }
 
-            artifacts
+            return IcCachesArtifacts(artifacts, cacheGuard)
-        } else emptyList()
+        }
-        return icCaches
+        return null
     }
 
     override fun setupPlatformSpecificArgumentsAndServices(

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheGuard.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import java.io.File
+
+class IncrementalCacheGuard(cacheDir: String) {
+    enum class AcquireStatus { OK, CACHE_CLEARED }
+
+    private val cacheRoot = File(cacheDir)
+    private val guardFile = cacheRoot.resolve("cache.guard")
+
+    fun acquire(): AcquireStatus {
+        val cacheCleared = guardFile.exists()
+        if (cacheCleared) {
+            cacheRoot.deleteRecursively()
+        }
+        tryAcquire()
+        return if (cacheCleared) AcquireStatus.CACHE_CLEARED else AcquireStatus.OK
+    }
+
+    fun tryAcquire() {
+        cacheRoot.mkdirs()
+        guardFile.createNewFile()
+    }
+
+    fun release() {
+        guardFile.delete()
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsIrBeIncrementalCompilationIT.kt

@@ -13,6 +13,7 @@ import kotlin.io.path.appendText
 import kotlin.io.path.readText
 import kotlin.io.path.writeText
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
 import kotlin.test.assertNotEquals
 import kotlin.test.assertTrue
 
@@ -54,11 +55,16 @@ class Kotlin2JsIrBeIncrementalCompilationIT : KGPBaseTest() {
             }
 
             srcFile.writeText(badCode)
+
+            val guardFile = projectPath.resolve("app/build/klib/cache/cache.guard").toFile()
             for (i in 0..1) {
                 buildAndFail("nodeDevelopmentRun") {
                     assertTasksFailed(":app:compileDevelopmentExecutableKotlinJs")
                     val failedBuildCacheFiles = readCacheFiles()
-                    assertEquals(successfulBuildCacheFiles, failedBuildCacheFiles, "The cache files should not be modified")
+                    val expectedFiles = successfulBuildCacheFiles.toMutableMap()
+                    expectedFiles[guardFile.absolutePath] = ByteArray(0).contentHashCode()
+                    assertEquals(expectedFiles, failedBuildCacheFiles, "The cache files should not be modified")
+                    guardFile.delete()
                 }
             }
 
@@ -198,4 +204,33 @@ class Kotlin2JsIrBeIncrementalCompilationIT : KGPBaseTest() {
             }
         }
     }
+
+
+    @DisplayName("Test cache invalidation after detecting guard file")
+    @GradleTest
+    fun testCacheGuardInvalidation(gradleVersion: GradleVersion) {
+        project("kotlin2JsIrICProject", gradleVersion) {
+            build("nodeDevelopmentRun") {
+                assertTasksExecuted(":compileDevelopmentExecutableKotlinJs")
+                assertOutputContains("module [main] was built clean")
+                assertOutputContains(">>> TEST OUT: Hello, Gradle.")
+            }
+
+            val cacheGuard = projectPath.resolve("build/klib/cache/cache.guard").toFile()
+            assertFalse(cacheGuard.exists(), "Cache guard file should be removed after successful build")
+
+            val srcFile = projectPath.resolve("src/main/kotlin/Main.kt").toFile()
+            srcFile.writeText(srcFile.readText().replace("greeting(\"Gradle\")", "greeting(\"Kotlin\")"))
+
+            cacheGuard.createNewFile()
+            build("nodeDevelopmentRun") {
+                assertTasksExecuted(":compileDevelopmentExecutableKotlinJs")
+                assertOutputContains(Regex("Cache guard file detected, cache directory '.+' cleared"))
+                assertOutputContains("module [main] was built clean")
+                assertOutputContains(">>> TEST OUT: Hello, Kotlin.")
+            }
+
+            assertFalse(cacheGuard.exists(), "Cache guard file should be removed after successful build")
+        }
+    }
 }