From 274c9b629469b491e632f674e32361ec2b25911e Mon Sep 17 00:00:00 2001 From: Alexander Likhachev Date: Tue, 22 Nov 2022 23:07:44 +0100 Subject: [PATCH] [IC] Register JS IC header file and lookup counter in transaction #KT-49785 In Progress --- .../incremental/CompilationTransaction.kt | 0 .../kotlin/incremental/IncrementalJsCache.kt | 4 +- .../kotlin/incremental/LookupStorage.kt | 8 +- .../incremental/IncrementalCachesManager.kt | 26 ++- .../incremental/IncrementalCompilerRunner.kt | 155 +++++++++--------- .../IncrementalJsCompilerRunner.kt | 5 +- .../IncrementalJvmCompilerRunner.kt | 5 +- .../jps/incremental/JpsIncrementalCache.kt | 3 +- 8 files changed, 118 insertions(+), 88 deletions(-) rename {compiler/incremental-compilation-impl => build-common}/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt (100%) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt b/build-common/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt similarity index 100% rename from compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt rename to build-common/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt index 6c42a688834..94fedc5be70 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt @@ -41,7 +41,8 @@ import java.io.File open class IncrementalJsCache( cachesDir: File, pathConverter: FileToPathConverter, - serializerProtocol: SerializerExtensionProtocol + serializerProtocol: SerializerExtensionProtocol, + private val transaction: CompilationTransaction, ) : AbstractIncrementalCache(cachesDir, pathConverter) { companion object { private const val TRANSLATION_RESULT_MAP = "translation-result" @@ -72,6 +73,7 @@ open class IncrementalJsCache( var header: ByteArray get() = headerFile.readBytes() set(value) { + transaction.registerAddedOrChangedFile(headerFile.toPath()) cachesDir.mkdirs() headerFile.writeBytes(value) } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index d613ab4e662..62b3463ace7 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -34,7 +34,8 @@ open class LookupStorage( targetDataDir: File, pathConverter: FileToPathConverter, storeFullFqNames: Boolean = false, - private val trackChanges: Boolean = false + private val trackChanges: Boolean = false, + private val transaction: CompilationTransaction = DummyCompilationTransaction(), ) : BasicMapsOwner(targetDataDir) { val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder") @@ -134,9 +135,7 @@ open class LookupStorage( @Synchronized override fun clean() { - if (countersFile.exists()) { - countersFile.delete() - } + transaction.deleteFile(countersFile.toPath()) size = 0 @@ -148,6 +147,7 @@ open class LookupStorage( try { if (size != oldSize) { if (size > 0) { + transaction.registerAddedOrChangedFile(countersFile.toPath()) if (!countersFile.exists()) { countersFile.parentFile.mkdirs() countersFile.createNewFile() diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index f1caeeeac4b..558e9a13ce6 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -28,6 +28,7 @@ abstract class IncrementalCachesManager( cacheDirectory, rootProjectDir, reporter, + transaction, storeFullFqNamesInLookupCache, trackChangesInLookupCache ) { @@ -98,8 +107,15 @@ class IncrementalJsCachesManager( rootProjectDir: File?, reporter: ICReporter, serializerProtocol: SerializerExtensionProtocol, - storeFullFqNamesInLookupCache: Boolean -) : IncrementalCachesManager(cachesRootDir, rootProjectDir, reporter, storeFullFqNamesInLookupCache) { + storeFullFqNamesInLookupCache: Boolean, + transaction: CompilationTransaction = DummyCompilationTransaction(), +) : IncrementalCachesManager( + cachesRootDir, + rootProjectDir, + reporter, + transaction, + storeFullFqNamesInLookupCache, +) { private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() } - override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol).apply { registerCache() } + override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol, transaction).apply { registerCache() } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 7bee2896d4f..cf5e347746d 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -80,7 +80,7 @@ abstract class IncrementalCompilerRunner< private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME) protected open val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS - protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager + protected abstract fun createCacheManager(args: Args, projectDir: File?, transaction: CompilationTransaction): CacheManager protected abstract fun destinationDir(args: Args): File fun compile( @@ -158,86 +158,95 @@ abstract class IncrementalCompilerRunner< } changedFiles as ChangedFiles.Known? - val caches = createCacheManager(args, projectDir) + createTransaction().use { transaction -> + val caches = createCacheManager(args, projectDir, transaction) - fun compile(): ICResult { - // Step 1: Get changed files - val knownChangedFiles: ChangedFiles.Known = try { - getChangedFiles(changedFiles, allSourceFiles, caches) - } catch (e: Throwable) { - return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e) - } - - val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null - - // Step 2: Compute files to recompile - val compilationMode = try { - reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) { - calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap()) + fun compile(): ICResult { + // Step 1: Get changed files + val knownChangedFiles: ChangedFiles.Known = try { + getChangedFiles(changedFiles, allSourceFiles, caches) + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e) } - } catch (e: Throwable) { - return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e) - } - if (compilationMode is CompilationMode.Rebuild) { - return ICResult.RequiresRebuild(compilationMode.reason) - } + val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null - val abiSnapshotData = if (withAbiSnapshot) { - if (!abiSnapshotFile.exists()) { - reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" } - return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT) + // Step 2: Compute files to recompile + val compilationMode = try { + reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) { + calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap()) + } + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e) } - reporter.info { "Incremental compilation with ABI snapshot enabled" } - AbiSnapshotData( - snapshot = AbiSnapshotImpl.read(abiSnapshotFile), - classpathAbiSnapshot = classpathAbiSnapshot!! - ) - } else null - // Step 3: Compile incrementally - val exitCode = try { - compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector) - } catch (e: Throwable) { - return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e) + if (compilationMode is CompilationMode.Rebuild) { + return ICResult.RequiresRebuild(compilationMode.reason) + } + + val abiSnapshotData = if (withAbiSnapshot) { + if (!abiSnapshotFile.exists()) { + reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" } + return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT) + } + reporter.info { "Incremental compilation with ABI snapshot enabled" } + AbiSnapshotData( + snapshot = AbiSnapshotImpl.read(abiSnapshotFile), + classpathAbiSnapshot = classpathAbiSnapshot!! + ) + } else null + + // Step 3: Compile incrementally + val exitCode = try { + compileImpl( + compilationMode as CompilationMode.Incremental, + allSourceFiles, + args, + caches, + abiSnapshotData, + messageCollector, + transaction + ) + } catch (e: Throwable) { + return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e) + } + + return ICResult.Completed(exitCode) } - return ICResult.Completed(exitCode) - } + fun closeCaches(caches: CacheManager, activeException: Throwable) { + try { + caches.close() + } catch (e: Throwable) { + activeException.addSuppressed(e) + } + } - fun closeCaches(caches: CacheManager, activeException: Throwable) { + // Because `caches` is a Closeable resource, it is important to close them in both cases: + // 1. in the event of an exception + // 2. after a normal execution + // Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the + // future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in + // `compileNonIncrementally`). + val icResult = try { + compile() + } catch (e: Throwable) { + // Case 1 - Close the caches upon an exception + closeCaches(caches, e) + throw e + } + + // Case 2 - Close the caches after a normal execution try { caches.close() } catch (e: Throwable) { - activeException.addSuppressed(e) + return ICResult.Failed( + IC_FAILED_TO_CLOSE_CACHES, + RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e) + ) } + return icResult } - - // Because `caches` is a Closeable resource, it is important to close them in both cases: - // 1. in the event of an exception - // 2. after a normal execution - // Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the - // future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in - // `compileNonIncrementally`). - val icResult = try { - compile() - } catch (e: Throwable) { - // Case 1 - Close the caches upon an exception - closeCaches(caches, e) - throw e - } - - // Case 2 - Close the caches after a normal execution - try { - caches.close() - } catch (e: Throwable) { - return ICResult.Failed( - IC_FAILED_TO_CLOSE_CACHES, - RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e) - ) - } - - return icResult } private fun compileNonIncrementally( @@ -257,7 +266,8 @@ abstract class IncrementalCompilerRunner< reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" } cleanOrCreateDirectories(outputDirsToClean) } - return createCacheManager(args, projectDir).use { caches -> + val transaction = DummyCompilationTransaction() + return createCacheManager(args, projectDir, transaction).use { caches -> if (trackChangedFiles) { caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) } @@ -265,7 +275,7 @@ abstract class IncrementalCompilerRunner< AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args)) } else null - compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector) + compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector, transaction) } } @@ -378,14 +388,13 @@ abstract class IncrementalCompilerRunner< args: Args, caches: CacheManager, abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true - messageCollector: MessageCollector + messageCollector: MessageCollector, + transaction: CompilationTransaction, ): ExitCode { performWorkBeforeCompilation(compilationMode, args) val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) } - val exitCode = createTransaction().use { transaction -> - doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction) - } + val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction) performWorkAfterCompilation(compilationMode, exitCode, caches) return exitCode diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index f2b5f073c14..59bba0d2318 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -99,14 +99,15 @@ class IncrementalJsCompilerRunner( preciseCompilationResultsBackup = preciseCompilationResultsBackup, ) { - override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager { + override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJsCachesManager { val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol return IncrementalJsCachesManager( cacheDirectory, projectDir, reporter, serializerProtocol, - storeFullFqNamesInLookupCache = withAbiSnapshot + storeFullFqNamesInLookupCache = withAbiSnapshot, + transaction = transaction, ) } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 12abb07e7e0..c6847c563ea 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -150,14 +150,15 @@ open class IncrementalJvmCompilerRunner( withAbiSnapshot = withAbiSnapshot, preciseCompilationResultsBackup = preciseCompilationResultsBackup, ) { - override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager = + override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJvmCachesManager = IncrementalJvmCachesManager( cacheDirectory, projectDir, File(args.destination), reporter, storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, - trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun + trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun, + transaction = transaction, ) override fun destinationDir(args: K2JVMCompilerArguments): File = diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt index 3e4203fc14d..4c29e54f50c 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt @@ -21,6 +21,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.StorageOwner +import org.jetbrains.kotlin.incremental.DummyCompilationTransaction import org.jetbrains.kotlin.incremental.IncrementalCacheCommon import org.jetbrains.kotlin.incremental.IncrementalJsCache import org.jetbrains.kotlin.incremental.IncrementalJvmCache @@ -54,7 +55,7 @@ class JpsIncrementalJsCache( target: ModuleBuildTarget, paths: BuildDataPaths, pathConverter: FileToPathConverter -) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol), JpsIncrementalCache { +) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol, DummyCompilationTransaction()), JpsIncrementalCache { override fun addJpsDependentCache(cache: JpsIncrementalCache) { if (cache is JpsIncrementalJsCache) { addDependentCache(cache)