[JS IR] Add comments about JS IR BE IC

2023-12-05 11:51:05 +01:00
parent 389095e8f7
commit 56e1c5cbc6
10 changed files with 132 additions and 1 deletions
@@ -30,10 +30,17 @@ import java.nio.file.Files
 import java.util.EnumSet

 fun interface JsIrCompilerICInterface {
+    /**
+     * It is expected that the method implementation runs a lowering pipeline
+     * and produces a list of generators capable of generating JS AST fragments.
+     */
    fun compile(allModules: Collection<IrModuleFragment>, dirtyFiles: Collection<IrFile>): List<() -> JsIrProgramFragments>
 }

 fun interface JsIrCompilerICInterfaceFactory {
+    /**
+     * It is expected that the method implementation creates a backend context and initializes all builtins and intrinsics.
+     */
    fun createCompilerForIC(
        mainModule: IrModuleFragment,
        configuration: CompilerConfiguration
@@ -51,6 +58,20 @@ enum class DirtyFileState(val str: String) {
    REMOVED_FILE("removed file")
 }

+/**
+ * This class is the entry point for the incremental compilation routine.
+ * The most interesting params:
+ * @param cacheDir - the directory where the incremental cache updater will store its caches. [CacheUpdater] maintains the directory fully.
+ * @param compilerInterfaceFactory - is a factory that creates an instance of the compiler used for building dirty files.
+ *
+ * The main public methods are:
+ *  [actualizeCaches] - performs the entire incremental compilation routine;
+ *  [getDirtyFileLastStats] - retrieves stats about dirty files, explaining why the file has been marked as dirty (see [DirtyFileState]);
+ *  [getStopwatchLastLaps] - retrieves the durations of the incremental compilation subroutines;
+ *
+ *  For a better understanding of what happens here, pay attention to [stopwatch] usages.
+ *  In every place, it has a short description about the code it measures.
+ */
 class CacheUpdater(
    mainModule: String,
    private val allModules: Collection<String>,
@@ -764,6 +785,18 @@ class CacheUpdater(
        rebuiltFragments
    }

+    /**
+     * This method performs the following routine:
+     *  - Estimates dirty files that must be relowered;
+     *  - Creates a compiler instance by calling [compilerInterfaceFactory];
+     *  - Runs the compiler (lowering pipeline) for the dirty files (see [JsIrCompilerICInterface]);
+     *  - Transforms lowered IR to JS AST fragments [JsIrProgramFragments];
+     *  - Saves the cache data on the disk.
+     *
+     *  @return A module artifact list, where [ModuleArtifact] represents a compiled klib.
+     *   It contains either paths to files with serialized JS AST or the deserialized [JsIrProgramFragments] objects themselves
+     *   for every file in the generating JS module. The list should be used for building the final JS module in [JsExecutableProducer]
+     */
    fun actualizeCaches(): List<ModuleArtifact> {
        stopwatch.clear()
        dirtyFileStats.clear()
@@ -103,11 +103,17 @@ private fun collectImplementedSymbol(deserializedSymbols: Map<IdSignature, IrSym
    }
 }

+/**
+ * This sealed class is used to retrieve declaration signatures for a specific source file.
+ */
 internal sealed class FileSignatureProvider(val irFile: IrFile, val srcFile: KotlinSourceFile) {
    abstract fun getSignatureToIndexMapping(): Map<IdSignature, Int>
    abstract fun getReachableSignatures(): Set<IdSignature>
    abstract fun getImplementedSymbols(): Map<IdSignature, IrSymbol>

+    /**
+     * This is a basic implementation that allows retrieving signatures from a deserialized klib via [IrFileDeserializer].
+     */
    class DeserializedFromKlib(private val fileDeserializer: IrFileDeserializer, srcFile: KotlinSourceFile) : FileSignatureProvider(fileDeserializer.file, srcFile) {
        override fun getSignatureToIndexMapping(): Map<IdSignature, Int> {
            return fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
@@ -130,6 +136,13 @@ internal sealed class FileSignatureProvider(val irFile: IrFile, val srcFile: Kot
        }
    }

+    /**
+     * This is a special implementation that allows retrieving signatures for function type interfaces.
+     * We need this special implementation because function type interfaces are not serialized into klib,
+     * and the parent [IrFile] is generated on the fly during klib deserialization.
+     * Therefore, there is no corresponding [IrFileDeserializer] for the parent [IrFile].
+     * For more details, see [org.jetbrains.kotlin.ir.backend.js.FunctionTypeInterfacePackages.makePackageAccessor].
+     */
    class GeneratedFunctionTypeInterface(file: IrFile, srcFile: KotlinSourceFile) : FileSignatureProvider(file, srcFile) {
        private val allSignatures = run {
            val topLevelSymbols = buildMap {
@@ -13,6 +13,9 @@ import org.jetbrains.kotlin.protobuf.CodedOutputStream
 import java.io.ByteArrayOutputStream
 import java.io.File

+/**
+ * This class manages the incremental cache for a specific klib.
+ */
 internal class IncrementalCache(private val library: KotlinLibraryHeader, val cacheDir: File) {
    companion object {
        private const val CACHE_HEADER = "ic.header.bin"
@@ -220,6 +223,17 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca
        }
    }

+    /**
+     * Fetches cached data for a specific [srcFile].
+     *
+     * @param loadSignatures
+     *  If false, it loads only file names for the (direct and inverse) dependencies.
+     *  If true, it also loads the declaration signatures [IdSignature] and declaration hashes [ICHash] for every dependency.
+     *
+     * Note that if the file is modified or removed, the signatures cannot be loaded, and [loadSignatures] must be false.
+     * This is because [IdSignatureSerialization] uses declaration indexes from the klib for serialization and deserialization.
+     * If the file is modified, the indexes can be modified (for example, shifted) too, leading to incorrect deserialization.
+     */
    private fun fetchSourceFileMetadata(srcFile: KotlinSourceFile, loadSignatures: Boolean) =
        kotlinLibrarySourceFileMetadata.getOrPut(srcFile) {
            val deserializer = idSignatureSerialization.getIdSignatureDeserializer(srcFile)
@@ -9,6 +9,10 @@ import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.*
 import org.jetbrains.kotlin.serialization.js.ModuleKind

+/**
+ * This class is responsible for incrementally producing the final JavaScript code based on the provided cache artifacts.
+ * @param caches - Cache artifacts, which are instances of [ModuleArtifact], can be obtained using [CacheUpdater.actualizeCaches].
+ */
 class JsExecutableProducer(
    private val mainModuleName: String,
    private val moduleKind: ModuleKind,
@@ -12,6 +12,9 @@ import org.jetbrains.kotlin.protobuf.CodedOutputStream
 import org.jetbrains.kotlin.utils.addToStdlib.runIf
 import java.io.File

+/**
+ * This class maintains incremental cache files used by [JsExecutableProducer] for per-file compilation mode.
+ */
 class JsPerFileCache(private val moduleArtifacts: List<ModuleArtifact>) : JsMultiArtifactCache<JsPerFileCache.CachedFileInfo>() {
    companion object {
        private const val JS_MODULE_HEADER = "js.module.header.bin"
@@ -470,4 +473,4 @@ class JsPerFileCache(private val moduleArtifacts: List<ModuleArtifact>) : JsMult
        val mainHeader: JsIrModuleHeader,
        val exportHeader: JsIrModuleHeader?
    )
-}
+}
@@ -9,6 +9,9 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.*
 import org.jetbrains.kotlin.serialization.js.ModuleKind
 import java.io.File

+/**
+ * This class maintains incremental cache files used by [JsExecutableProducer] for per-module compilation mode.
+ */
 class JsPerModuleCache(
    private val moduleKind: ModuleKind,
    private val moduleArtifacts: List<ModuleArtifact>
@@ -12,6 +12,9 @@ import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import java.io.File

+/**
+ * This interface represents the abstract klib and is mainly used for detecting modified files.
+ */
 internal interface KotlinLibraryHeader {
    val libraryFile: KotlinLibraryFile

@@ -23,6 +26,9 @@ internal interface KotlinLibraryHeader {
    val jsOutputName: String?
 }

+/**
+ * This implementation represents the existing klib that is present on the disk.
+ */
 internal class KotlinLoadedLibraryHeader(
    private val library: KotlinLibrary,
    private val internationService: IrInterningService
@@ -74,6 +80,10 @@ internal class KotlinLoadedLibraryHeader(
    }
 }

+/**
+ * This implementation represents the removed klib, which no longer exists.
+ * Its main aim is to correctly handle the removed files; for example, we must invalidate all reverse dependencies.
+ */
 internal class KotlinRemovedLibraryHeader(private val libCacheDir: File) : KotlinLibraryHeader {
    override val libraryFile: KotlinLibraryFile
        get() = icError("removed library name is unavailable; cache dir: ${libCacheDir.absolutePath}")
@@ -26,6 +26,11 @@ value class KotlinLibraryFile(val path: String) {
    override fun toString(): String = File(path).name
 }

+/**
+ * Represents a unique source file name from the klib.
+ * @param path - File name as it is in the klib.
+ * @param id - A unique index of the file, see [fromSources] comment.
+ */
 class KotlinSourceFile private constructor(val path: String, val id: Int) {
    fun toProtoStream(out: CodedOutputStream) {
        out.writeStringNoTag(path)
@@ -11,6 +11,12 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.safeModuleName
 import org.jetbrains.kotlin.ir.backend.js.utils.serialization.deserializeJsIrProgramFragment
 import java.io.File

+/**
+ * This class encapsulates the JS AST for a specific kt file, which can be either dirty or not.
+ * @param srcFilePath - Path to the kt file from the klib.
+ * @param fragments - The JS AST itself. It is non-null if the kt file is dirty, and its IR has been lowered and transformed into JS AST.
+ * @param astArtifact - Path to a serialized JS AST. It is typically used to obtain the JS AST if the kt file is unmodified.
+ */
 class SrcFileArtifact(val srcFilePath: String, private val fragments: JsIrProgramFragments?, private val astArtifact: File? = null) {
    fun loadJsIrFragments(): JsIrProgramFragments? {
        if (fragments != null) {
@@ -22,6 +28,14 @@ class SrcFileArtifact(val srcFilePath: String, private val fragments: JsIrProgra
    fun isModified() = fragments != null
 }

+/**
+ * This class encapsulates the JS AST for the entire klib file.
+ * @param fileArtifacts - A JS AST list for each kt file in the klib.
+ * @param artifactsDir - A directory where the JS AST cache is stored.
+ *
+ * The directory [artifactsDir] is used by both [JsPerFileCache] and [JsPerModuleCache] for storing their own caches.
+ * This dirty hack allows keeping caches for [CacheUpdater] and [JsExecutableProducer] in one directory.
+ */
 class ModuleArtifact(
    moduleName: String,
    val fileArtifacts: List<SrcFileArtifact>,
@@ -0,0 +1,32 @@
+# JS IR backend incremental compilation
+
+### Overview 
+
+The package `org.jetbrains.kotlin.ir.backend.js.ic` is responsible for incremental code generation.
+It incrementally transforms the klib IR into the final JavaScript code. 
+In other words, the code in this package receives an already-built klib,
+detects dirty files from the klib with the modified IR,
+additionally identifies files for which the IR must be lowered again
+(for example, all callers of the modified inline function must be lowered again),
+runs the lowering pipeline for both the dirty files and additional dirty files, and in the end,
+incrementally generates JavaScript code. All the logic here is based on the IR from the klib.
+
+Note that incremental klib compilation (compiling `kt` files to `klib`) is a separate process handled in another place.
+
+### Details
+
+Here are the two main classes:
+
+- Class [CacheUpdater](CacheUpdater.kt):
+    - Responsible for detecting all dirty files. This is a crucial part of the process. It includes:
+      - Identifying files with modified IR (modified code).
+      - Maintaining direct and inverse dependency graphs (see the class `KotlinSourceFileExports` and its inheritors) to detect additional dirty files that require re-lowering.
+    - Loads IR for the dirty files.
+    - Instantiates the compiler.
+    - Runs the lowering pipeline for the dirty files.
+    - Transforms the lowered IR to JS AST (see `JsIrProgramFragments`).
+    - Maintains the cache files on disk (see class [IncrementalCache](IncrementalCache.kt)).
+
+- Class [JsExecutableProducer](JsExecutableProducer.kt):
+    - Responsible for incrementally producing JavaScript code from the JS AST (see `JsIrProgramFragments`, usually obtained from `CacheUpdater`, contained in `SrcFileArtifact` as a main part of `ModuleArtifact`).
+    - Maintains the cache files (different from those in `CacheUpdater`) on disk (see classes [JsPerFileCache](JsPerFileCache.kt) and [JsPerModuleCache](JsPerModuleCache.kt)).