[IC] Register JS IC header file and lookup counter in transaction

#KT-49785 In Progress
2022-11-22 23:07:44 +01:00
parent 20ed029ba5
commit 274c9b6294
8 changed files with 118 additions and 88 deletions
@@ -1,178 +0,0 @@
-/*
- * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.incremental
-
-import org.jetbrains.kotlin.build.report.BuildReporter
-import org.jetbrains.kotlin.build.report.debug
-import org.jetbrains.kotlin.build.report.metrics.BuildTime
-import org.jetbrains.kotlin.build.report.metrics.measure
-import org.jetbrains.kotlin.compilerRunner.OutputItemsCollector
-import org.jetbrains.kotlin.konan.file.use
-import java.io.Closeable
-import java.io.File
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.StandardCopyOption
-
-/**
- * A compilation transaction that is able to track compilation result files changes and maybe to revert them back.
- */
-interface CompilationTransaction : Closeable {
-    /**
-     * This method should be called before creating a new file or changing an existing file.
-     */
-    fun registerAddedOrChangedFile(outputFile: Path)
-
-    /**
-     * This method should be used to perform a file removal.
-     */
-    fun deleteFile(outputFile: Path)
-
-    /**
-     * Marks the transaction as successful, so it should not revert changes if it is able to perform revert.
-     */
-    fun markAsSuccessful()
-}
-
-/**
- * A dummy implementation of compilation transaction
- */
-class DummyCompilationTransaction : CompilationTransaction {
-    override fun registerAddedOrChangedFile(outputFile: Path) {
-        // do nothing
-    }
-
-    override fun deleteFile(outputFile: Path) {
-        if (Files.exists(outputFile)) {
-            Files.delete(outputFile)
-        }
-    }
-
-    override fun markAsSuccessful() {
-        // do nothing
-    }
-
-    override fun close() {
-        // do nothing
-    }
-
-}
-
-/**
- * A recoverable implementation of compilation transaction.
- * Tracks all files changes during a compilation and reverts them back if [markAsSuccessful] isn't called
- * In the case of a successful compilation [stashDir] is removed after is compilation.
- * In the case of an unsuccessful compilation [stashDir] is also removed, but the backed-up files restored to their origin location.
- */
-class RecoverableCompilationTransaction(
-    private val reporter: BuildReporter,
-    private val stashDir: Path,
-) : CompilationTransaction {
-    private val fileRelocationRegistry = hashMapOf<Path, Path?>()
-    private var filesCounter = 0
-    private var successful = false
-
-    /**
-     * Moves the original [outputFile] before change to the [stashDir].
-     * If the [outputFile] doesn't exist then it's marked to be removed if the transaction is unsuccessful.
-     */
-    override fun registerAddedOrChangedFile(outputFile: Path) {
-        if (isFileRelocationIsAlreadyRegisteredFor(outputFile)) return
-        reporter.measure(BuildTime.PRECISE_BACKUP_OUTPUT) {
-            if (Files.exists(outputFile)) {
-                stashFile(outputFile)
-            } else {
-                reporter.debug { "Marking the $outputFile file as newly added" }
-                fileRelocationRegistry[outputFile] = null
-            }
-        }
-    }
-
-    /**
-     * Moves the original [outputFile] to the [stashDir] instead of deleting.
-     */
-    override fun deleteFile(outputFile: Path) {
-        if (isFileRelocationIsAlreadyRegisteredFor(outputFile)) {
-            if (Files.exists(outputFile)) {
-                reporter.debug { "Deleting $outputFile" }
-                Files.delete(outputFile)
-            }
-            return
-        }
-        reporter.measure(BuildTime.PRECISE_BACKUP_OUTPUT) {
-            stashFile(outputFile)
-        }
-    }
-
-    private fun stashFile(outputFile: Path) {
-        val relocatedFilePath = getNextRelocatedFilePath()
-        reporter.debug { "Moving $outputFile to the stash as $relocatedFilePath" }
-        fileRelocationRegistry[outputFile] = relocatedFilePath
-        Files.move(outputFile, relocatedFilePath)
-    }
-
-    private fun getNextRelocatedFilePath(): Path = stashDir.resolve("$filesCounter.backup").also { filesCounter++ }
-
-    private fun isFileRelocationIsAlreadyRegisteredFor(outputFile: Path) = outputFile in fileRelocationRegistry
-
-    /**
-     * Reverts all the file changes registered in this transaction.
-     * If the value for a key is null, then it's the file that was created during the transaction, so the file will be just removed.
-     */
-    private fun revertChanges() {
-        reporter.debug { "Reverting changes" }
-        reporter.measure(BuildTime.RESTORE_OUTPUT_FROM_BACKUP) {
-            for ((originPath, relocatedPath) in fileRelocationRegistry) {
-                if (relocatedPath == null) {
-                    if (Files.exists(originPath)) {
-                        Files.delete(originPath)
-                    }
-                    continue
-                }
-                Files.move(relocatedPath, originPath, StandardCopyOption.REPLACE_EXISTING)
-            }
-        }
-    }
-
-    /**
-     * Deletes the [stashDir].
-     */
-    private fun cleanupStash() {
-        reporter.debug { "Cleaning up stash" }
-        reporter.measure(BuildTime.CLEAN_BACKUP_STASH) {
-            Files.walk(stashDir).use {
-                it.sorted(Comparator.reverseOrder())
-                    .forEach(Files::delete)
-            }
-        }
-    }
-
-    override fun markAsSuccessful() {
-        successful = true
-    }
-
-    override fun close() {
-        if (successful) {
-            cleanupStash()
-        } else {
-            revertChanges()
-            cleanupStash()
-        }
-    }
-}
-
-/**
- * A delegating [OutputItemsCollector] implementation that registers compiler output changes in the [transaction]
- */
-class TransactionOutputsRegistrar(
-    private val transaction: CompilationTransaction,
-    private val origin: OutputItemsCollector
-) : OutputItemsCollector {
-    override fun add(sourceFiles: Collection<File>, outputFile: File) {
-        transaction.registerAddedOrChangedFile(outputFile.toPath())
-        origin.add(sourceFiles, outputFile)
-    }
-}
@@ -28,6 +28,7 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
    cachesRootDir: File,
    rootProjectDir: File?,
    protected val reporter: ICReporter,
+    transaction: CompilationTransaction,
    storeFullFqNamesInLookupCache: Boolean = false,
    trackChangesInLookupCache: Boolean = false
 ) : Closeable {
@@ -47,7 +48,13 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache

    val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter, pathConverter).apply { registerCache() }
    val lookupCache: LookupStorage =
-        LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache, trackChangesInLookupCache).apply { registerCache() }
+        LookupStorage(
+            lookupCacheDir,
+            pathConverter,
+            storeFullFqNamesInLookupCache,
+            trackChangesInLookupCache,
+            transaction,
+        ).apply { registerCache() }
    abstract val platformCache: PlatformCache

    @Suppress("UnstableApiUsage")
@@ -81,11 +88,13 @@ class IncrementalJvmCachesManager(
    outputDir: File,
    reporter: ICReporter,
    storeFullFqNamesInLookupCache: Boolean = false,
-    trackChangesInLookupCache: Boolean = false
+    trackChangesInLookupCache: Boolean = false,
+    transaction: CompilationTransaction = DummyCompilationTransaction(),
 ) : IncrementalCachesManager<IncrementalJvmCache>(
    cacheDirectory,
    rootProjectDir,
    reporter,
+    transaction,
    storeFullFqNamesInLookupCache,
    trackChangesInLookupCache
 ) {
@@ -98,8 +107,15 @@ class IncrementalJsCachesManager(
    rootProjectDir: File?,
    reporter: ICReporter,
    serializerProtocol: SerializerExtensionProtocol,
-    storeFullFqNamesInLookupCache: Boolean
-) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, rootProjectDir, reporter, storeFullFqNamesInLookupCache) {
+    storeFullFqNamesInLookupCache: Boolean,
+    transaction: CompilationTransaction = DummyCompilationTransaction(),
+) : IncrementalCachesManager<IncrementalJsCache>(
+    cachesRootDir,
+    rootProjectDir,
+    reporter,
+    transaction,
+    storeFullFqNamesInLookupCache,
+) {
    private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() }
-    override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol).apply { registerCache() }
+    override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol, transaction).apply { registerCache() }
 }
@@ -80,7 +80,7 @@ abstract class IncrementalCompilerRunner<
    private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
    protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS

-    protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager
+    protected abstract fun createCacheManager(args: Args, projectDir: File?, transaction: CompilationTransaction): CacheManager
    protected abstract fun destinationDir(args: Args): File

    fun compile(
@@ -158,86 +158,95 @@ abstract class IncrementalCompilerRunner<
        }
        changedFiles as ChangedFiles.Known?

-        val caches = createCacheManager(args, projectDir)
+        createTransaction().use { transaction ->
+            val caches = createCacheManager(args, projectDir, transaction)

-        fun compile(): ICResult {
-            // Step 1: Get changed files
-            val knownChangedFiles: ChangedFiles.Known = try {
-                getChangedFiles(changedFiles, allSourceFiles, caches)
-            } catch (e: Throwable) {
-                return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
-            }
-
-            val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
-
-            // Step 2: Compute files to recompile
-            val compilationMode = try {
-                reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
-                    calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
+            fun compile(): ICResult {
+                // Step 1: Get changed files
+                val knownChangedFiles: ChangedFiles.Known = try {
+                    getChangedFiles(changedFiles, allSourceFiles, caches)
+                } catch (e: Throwable) {
+                    return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
                }
-            } catch (e: Throwable) {
-                return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
-            }

-            if (compilationMode is CompilationMode.Rebuild) {
-                return ICResult.RequiresRebuild(compilationMode.reason)
-            }
+                val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null

-            val abiSnapshotData = if (withAbiSnapshot) {
-                if (!abiSnapshotFile.exists()) {
-                    reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
-                    return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
+                // Step 2: Compute files to recompile
+                val compilationMode = try {
+                    reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
+                        calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
+                    }
+                } catch (e: Throwable) {
+                    return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
                }
-                reporter.info { "Incremental compilation with ABI snapshot enabled" }
-                AbiSnapshotData(
-                    snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
-                    classpathAbiSnapshot = classpathAbiSnapshot!!
-                )
-            } else null

-            // Step 3: Compile incrementally
-            val exitCode = try {
-                compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector)
-            } catch (e: Throwable) {
-                return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
+                if (compilationMode is CompilationMode.Rebuild) {
+                    return ICResult.RequiresRebuild(compilationMode.reason)
+                }
+
+                val abiSnapshotData = if (withAbiSnapshot) {
+                    if (!abiSnapshotFile.exists()) {
+                        reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
+                        return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
+                    }
+                    reporter.info { "Incremental compilation with ABI snapshot enabled" }
+                    AbiSnapshotData(
+                        snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
+                        classpathAbiSnapshot = classpathAbiSnapshot!!
+                    )
+                } else null
+
+                // Step 3: Compile incrementally
+                val exitCode = try {
+                    compileImpl(
+                        compilationMode as CompilationMode.Incremental,
+                        allSourceFiles,
+                        args,
+                        caches,
+                        abiSnapshotData,
+                        messageCollector,
+                        transaction
+                    )
+                } catch (e: Throwable) {
+                    return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
+                }
+
+                return ICResult.Completed(exitCode)
            }

-            return ICResult.Completed(exitCode)
-        }
+            fun closeCaches(caches: CacheManager, activeException: Throwable) {
+                try {
+                    caches.close()
+                } catch (e: Throwable) {
+                    activeException.addSuppressed(e)
+                }
+            }

-        fun closeCaches(caches: CacheManager, activeException: Throwable) {
+            // Because `caches` is a Closeable resource, it is important to close them in both cases:
+            //    1. in the event of an exception
+            //    2. after a normal execution
+            // Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the
+            // future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in
+            // `compileNonIncrementally`).
+            val icResult = try {
+                compile()
+            } catch (e: Throwable) {
+                // Case 1 - Close the caches upon an exception
+                closeCaches(caches, e)
+                throw e
+            }
+
+            // Case 2 - Close the caches after a normal execution
            try {
                caches.close()
            } catch (e: Throwable) {
-                activeException.addSuppressed(e)
+                return ICResult.Failed(
+                    IC_FAILED_TO_CLOSE_CACHES,
+                    RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e)
+                )
            }
+            return icResult
        }
-
-        // Because `caches` is a Closeable resource, it is important to close them in both cases:
-        //    1. in the event of an exception
-        //    2. after a normal execution
-        // Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the
-        // future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in
-        // `compileNonIncrementally`).
-        val icResult = try {
-            compile()
-        } catch (e: Throwable) {
-            // Case 1 - Close the caches upon an exception
-            closeCaches(caches, e)
-            throw e
-        }
-
-        // Case 2 - Close the caches after a normal execution
-        try {
-            caches.close()
-        } catch (e: Throwable) {
-            return ICResult.Failed(
-                IC_FAILED_TO_CLOSE_CACHES,
-                RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e)
-            )
-        }
-
-        return icResult
    }

    private fun compileNonIncrementally(
@@ -257,7 +266,8 @@ abstract class IncrementalCompilerRunner<
            reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" }
            cleanOrCreateDirectories(outputDirsToClean)
        }
-        return createCacheManager(args, projectDir).use { caches ->
+        val transaction = DummyCompilationTransaction()
+        return createCacheManager(args, projectDir, transaction).use { caches ->
            if (trackChangedFiles) {
                caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
            }
@@ -265,7 +275,7 @@ abstract class IncrementalCompilerRunner<
                AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
            } else null

-            compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector)
+            compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector, transaction)
        }
    }

@@ -378,14 +388,13 @@ abstract class IncrementalCompilerRunner<
        args: Args,
        caches: CacheManager,
        abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
-        messageCollector: MessageCollector
+        messageCollector: MessageCollector,
+        transaction: CompilationTransaction,
    ): ExitCode {
        performWorkBeforeCompilation(compilationMode, args)

        val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
-        val exitCode = createTransaction().use { transaction ->
-            doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)
-        }
+        val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)

        performWorkAfterCompilation(compilationMode, exitCode, caches)
        return exitCode
@@ -99,14 +99,15 @@ class IncrementalJsCompilerRunner(
    preciseCompilationResultsBackup = preciseCompilationResultsBackup,
 ) {

-    override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
+    override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJsCachesManager {
        val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
        return IncrementalJsCachesManager(
            cacheDirectory,
            projectDir,
            reporter,
            serializerProtocol,
-            storeFullFqNamesInLookupCache = withAbiSnapshot
+            storeFullFqNamesInLookupCache = withAbiSnapshot,
+            transaction = transaction,
        )
    }

@@ -150,14 +150,15 @@ open class IncrementalJvmCompilerRunner(
    withAbiSnapshot = withAbiSnapshot,
    preciseCompilationResultsBackup = preciseCompilationResultsBackup,
 ) {
-    override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
+    override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJvmCachesManager =
        IncrementalJvmCachesManager(
            cacheDirectory,
            projectDir,
            File(args.destination),
            reporter,
            storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
-            trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
+            trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun,
+            transaction = transaction,
        )

    override fun destinationDir(args: K2JVMCompilerArguments): File =