From 56e1c5cbc61dd7222440754cc2b1516b42c616fa Mon Sep 17 00:00:00 2001 From: Alexander Korepanov Date: Tue, 5 Dec 2023 11:51:05 +0100 Subject: [PATCH] [JS IR] Add comments about JS IR BE IC --- .../kotlin/ir/backend/js/ic/CacheUpdater.kt | 33 +++++++++++++++++++ .../ir/backend/js/ic/IdSignatureSource.kt | 13 ++++++++ .../ir/backend/js/ic/IncrementalCache.kt | 14 ++++++++ .../ir/backend/js/ic/JsExecutableProducer.kt | 4 +++ .../kotlin/ir/backend/js/ic/JsPerFileCache.kt | 5 ++- .../ir/backend/js/ic/JsPerModuleCache.kt | 3 ++ .../ir/backend/js/ic/KotlinLibraryHeader.kt | 10 ++++++ .../backend/js/ic/KotlinSourceFileMetadata.kt | 5 +++ .../kotlin/ir/backend/js/ic/ModuleArtifact.kt | 14 ++++++++ .../kotlin/ir/backend/js/ic/README.md | 32 ++++++++++++++++++ 10 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/README.md diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index c5302b43e2c..3ae2dc01518 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -30,10 +30,17 @@ import java.nio.file.Files import java.util.EnumSet fun interface JsIrCompilerICInterface { + /** + * It is expected that the method implementation runs a lowering pipeline + * and produces a list of generators capable of generating JS AST fragments. + */ fun compile(allModules: Collection, dirtyFiles: Collection): List<() -> JsIrProgramFragments> } fun interface JsIrCompilerICInterfaceFactory { + /** + * It is expected that the method implementation creates a backend context and initializes all builtins and intrinsics. + */ fun createCompilerForIC( mainModule: IrModuleFragment, configuration: CompilerConfiguration @@ -51,6 +58,20 @@ enum class DirtyFileState(val str: String) { REMOVED_FILE("removed file") } +/** + * This class is the entry point for the incremental compilation routine. + * The most interesting params: + * @param cacheDir - the directory where the incremental cache updater will store its caches. [CacheUpdater] maintains the directory fully. + * @param compilerInterfaceFactory - is a factory that creates an instance of the compiler used for building dirty files. + * + * The main public methods are: + * [actualizeCaches] - performs the entire incremental compilation routine; + * [getDirtyFileLastStats] - retrieves stats about dirty files, explaining why the file has been marked as dirty (see [DirtyFileState]); + * [getStopwatchLastLaps] - retrieves the durations of the incremental compilation subroutines; + * + * For a better understanding of what happens here, pay attention to [stopwatch] usages. + * In every place, it has a short description about the code it measures. + */ class CacheUpdater( mainModule: String, private val allModules: Collection, @@ -764,6 +785,18 @@ class CacheUpdater( rebuiltFragments } + /** + * This method performs the following routine: + * - Estimates dirty files that must be relowered; + * - Creates a compiler instance by calling [compilerInterfaceFactory]; + * - Runs the compiler (lowering pipeline) for the dirty files (see [JsIrCompilerICInterface]); + * - Transforms lowered IR to JS AST fragments [JsIrProgramFragments]; + * - Saves the cache data on the disk. + * + * @return A module artifact list, where [ModuleArtifact] represents a compiled klib. + * It contains either paths to files with serialized JS AST or the deserialized [JsIrProgramFragments] objects themselves + * for every file in the generating JS module. The list should be used for building the final JS module in [JsExecutableProducer] + */ fun actualizeCaches(): List { stopwatch.clear() dirtyFileStats.clear() diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureSource.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureSource.kt index b104450db48..77718b64935 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureSource.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureSource.kt @@ -103,11 +103,17 @@ private fun collectImplementedSymbol(deserializedSymbols: Map abstract fun getReachableSignatures(): Set abstract fun getImplementedSymbols(): Map + /** + * This is a basic implementation that allows retrieving signatures from a deserialized klib via [IrFileDeserializer]. + */ class DeserializedFromKlib(private val fileDeserializer: IrFileDeserializer, srcFile: KotlinSourceFile) : FileSignatureProvider(fileDeserializer.file, srcFile) { override fun getSignatureToIndexMapping(): Map { return fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() @@ -130,6 +136,13 @@ internal sealed class FileSignatureProvider(val irFile: IrFile, val srcFile: Kot } } + /** + * This is a special implementation that allows retrieving signatures for function type interfaces. + * We need this special implementation because function type interfaces are not serialized into klib, + * and the parent [IrFile] is generated on the fly during klib deserialization. + * Therefore, there is no corresponding [IrFileDeserializer] for the parent [IrFile]. + * For more details, see [org.jetbrains.kotlin.ir.backend.js.FunctionTypeInterfacePackages.makePackageAccessor]. + */ class GeneratedFunctionTypeInterface(file: IrFile, srcFile: KotlinSourceFile) : FileSignatureProvider(file, srcFile) { private val allSignatures = run { val topLevelSymbols = buildMap { diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt index 4fffee9dd30..540bd569ac9 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt @@ -13,6 +13,9 @@ import org.jetbrains.kotlin.protobuf.CodedOutputStream import java.io.ByteArrayOutputStream import java.io.File +/** + * This class manages the incremental cache for a specific klib. + */ internal class IncrementalCache(private val library: KotlinLibraryHeader, val cacheDir: File) { companion object { private const val CACHE_HEADER = "ic.header.bin" @@ -220,6 +223,17 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca } } + /** + * Fetches cached data for a specific [srcFile]. + * + * @param loadSignatures + * If false, it loads only file names for the (direct and inverse) dependencies. + * If true, it also loads the declaration signatures [IdSignature] and declaration hashes [ICHash] for every dependency. + * + * Note that if the file is modified or removed, the signatures cannot be loaded, and [loadSignatures] must be false. + * This is because [IdSignatureSerialization] uses declaration indexes from the klib for serialization and deserialization. + * If the file is modified, the indexes can be modified (for example, shifted) too, leading to incorrect deserialization. + */ private fun fetchSourceFileMetadata(srcFile: KotlinSourceFile, loadSignatures: Boolean) = kotlinLibrarySourceFileMetadata.getOrPut(srcFile) { val deserializer = idSignatureSerialization.getIdSignatureDeserializer(srcFile) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsExecutableProducer.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsExecutableProducer.kt index 346450599e5..fbb72684bbf 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsExecutableProducer.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsExecutableProducer.kt @@ -9,6 +9,10 @@ import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.* import org.jetbrains.kotlin.serialization.js.ModuleKind +/** + * This class is responsible for incrementally producing the final JavaScript code based on the provided cache artifacts. + * @param caches - Cache artifacts, which are instances of [ModuleArtifact], can be obtained using [CacheUpdater.actualizeCaches]. + */ class JsExecutableProducer( private val mainModuleName: String, private val moduleKind: ModuleKind, diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerFileCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerFileCache.kt index 4f6149bcc4f..0ef53828765 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerFileCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerFileCache.kt @@ -12,6 +12,9 @@ import org.jetbrains.kotlin.protobuf.CodedOutputStream import org.jetbrains.kotlin.utils.addToStdlib.runIf import java.io.File +/** + * This class maintains incremental cache files used by [JsExecutableProducer] for per-file compilation mode. + */ class JsPerFileCache(private val moduleArtifacts: List) : JsMultiArtifactCache() { companion object { private const val JS_MODULE_HEADER = "js.module.header.bin" @@ -470,4 +473,4 @@ class JsPerFileCache(private val moduleArtifacts: List) : JsMult val mainHeader: JsIrModuleHeader, val exportHeader: JsIrModuleHeader? ) -} \ No newline at end of file +} diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerModuleCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerModuleCache.kt index 1f8cbc9bcc0..a904ef0a947 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerModuleCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsPerModuleCache.kt @@ -9,6 +9,9 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.* import org.jetbrains.kotlin.serialization.js.ModuleKind import java.io.File +/** + * This class maintains incremental cache files used by [JsExecutableProducer] for per-module compilation mode. + */ class JsPerModuleCache( private val moduleKind: ModuleKind, private val moduleArtifacts: List diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt index 107531a37db..a2605c25fac 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt @@ -12,6 +12,9 @@ import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import java.io.File +/** + * This interface represents the abstract klib and is mainly used for detecting modified files. + */ internal interface KotlinLibraryHeader { val libraryFile: KotlinLibraryFile @@ -23,6 +26,9 @@ internal interface KotlinLibraryHeader { val jsOutputName: String? } +/** + * This implementation represents the existing klib that is present on the disk. + */ internal class KotlinLoadedLibraryHeader( private val library: KotlinLibrary, private val internationService: IrInterningService @@ -74,6 +80,10 @@ internal class KotlinLoadedLibraryHeader( } } +/** + * This implementation represents the removed klib, which no longer exists. + * Its main aim is to correctly handle the removed files; for example, we must invalidate all reverse dependencies. + */ internal class KotlinRemovedLibraryHeader(private val libCacheDir: File) : KotlinLibraryHeader { override val libraryFile: KotlinLibraryFile get() = icError("removed library name is unavailable; cache dir: ${libCacheDir.absolutePath}") diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinSourceFileMetadata.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinSourceFileMetadata.kt index 261378a09c8..c767afbeb26 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinSourceFileMetadata.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinSourceFileMetadata.kt @@ -26,6 +26,11 @@ value class KotlinLibraryFile(val path: String) { override fun toString(): String = File(path).name } +/** + * Represents a unique source file name from the klib. + * @param path - File name as it is in the klib. + * @param id - A unique index of the file, see [fromSources] comment. + */ class KotlinSourceFile private constructor(val path: String, val id: Int) { fun toProtoStream(out: CodedOutputStream) { out.writeStringNoTag(path) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt index a85fd497332..1eb46a1e5b9 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt @@ -11,6 +11,12 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.safeModuleName import org.jetbrains.kotlin.ir.backend.js.utils.serialization.deserializeJsIrProgramFragment import java.io.File +/** + * This class encapsulates the JS AST for a specific kt file, which can be either dirty or not. + * @param srcFilePath - Path to the kt file from the klib. + * @param fragments - The JS AST itself. It is non-null if the kt file is dirty, and its IR has been lowered and transformed into JS AST. + * @param astArtifact - Path to a serialized JS AST. It is typically used to obtain the JS AST if the kt file is unmodified. + */ class SrcFileArtifact(val srcFilePath: String, private val fragments: JsIrProgramFragments?, private val astArtifact: File? = null) { fun loadJsIrFragments(): JsIrProgramFragments? { if (fragments != null) { @@ -22,6 +28,14 @@ class SrcFileArtifact(val srcFilePath: String, private val fragments: JsIrProgra fun isModified() = fragments != null } +/** + * This class encapsulates the JS AST for the entire klib file. + * @param fileArtifacts - A JS AST list for each kt file in the klib. + * @param artifactsDir - A directory where the JS AST cache is stored. + * + * The directory [artifactsDir] is used by both [JsPerFileCache] and [JsPerModuleCache] for storing their own caches. + * This dirty hack allows keeping caches for [CacheUpdater] and [JsExecutableProducer] in one directory. + */ class ModuleArtifact( moduleName: String, val fileArtifacts: List, diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/README.md b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/README.md new file mode 100644 index 00000000000..05e3a3e336a --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/README.md @@ -0,0 +1,32 @@ +# JS IR backend incremental compilation + +### Overview + +The package `org.jetbrains.kotlin.ir.backend.js.ic` is responsible for incremental code generation. +It incrementally transforms the klib IR into the final JavaScript code. +In other words, the code in this package receives an already-built klib, +detects dirty files from the klib with the modified IR, +additionally identifies files for which the IR must be lowered again +(for example, all callers of the modified inline function must be lowered again), +runs the lowering pipeline for both the dirty files and additional dirty files, and in the end, +incrementally generates JavaScript code. All the logic here is based on the IR from the klib. + +Note that incremental klib compilation (compiling `kt` files to `klib`) is a separate process handled in another place. + +### Details + +Here are the two main classes: + +- Class [CacheUpdater](CacheUpdater.kt): + - Responsible for detecting all dirty files. This is a crucial part of the process. It includes: + - Identifying files with modified IR (modified code). + - Maintaining direct and inverse dependency graphs (see the class `KotlinSourceFileExports` and its inheritors) to detect additional dirty files that require re-lowering. + - Loads IR for the dirty files. + - Instantiates the compiler. + - Runs the lowering pipeline for the dirty files. + - Transforms the lowered IR to JS AST (see `JsIrProgramFragments`). + - Maintains the cache files on disk (see class [IncrementalCache](IncrementalCache.kt)). + +- Class [JsExecutableProducer](JsExecutableProducer.kt): + - Responsible for incrementally producing JavaScript code from the JS AST (see `JsIrProgramFragments`, usually obtained from `CacheUpdater`, contained in `SrcFileArtifact` as a main part of `ModuleArtifact`). + - Maintains the cache files (different from those in `CacheUpdater`) on disk (see classes [JsPerFileCache](JsPerFileCache.kt) and [JsPerModuleCache](JsPerModuleCache.kt)).