azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IC] Register JS IC header file and lookup counter in transaction

Browse Source 
#KT-49785 In Progress
This commit is contained in:
Alexander Likhachev
2022-11-22 23:07:44 +01:00
committed by Space Team
parent 20ed029ba5
commit 274c9b6294
8 changed files with 118 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt → build-common/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt
View File
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt
+3 -1
View File
@@ -41,7 +41,8 @@ import java.io.File
open class IncrementalJsCache(
cachesDir: File,
pathConverter: FileToPathConverter,
serializerProtocol: SerializerExtensionProtocol
serializerProtocol: SerializerExtensionProtocol,
private val transaction: CompilationTransaction,
) : AbstractIncrementalCache<FqName>(cachesDir, pathConverter) {
companion object {
private const val TRANSLATION_RESULT_MAP = "translation-result"
@@ -72,6 +73,7 @@ open class IncrementalJsCache(
var header: ByteArray
get() = headerFile.readBytes()
set(value) {
transaction.registerAddedOrChangedFile(headerFile.toPath())
cachesDir.mkdirs()
headerFile.writeBytes(value)
}
build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+4 -4
View File
@@ -34,7 +34,8 @@ open class LookupStorage(
targetDataDir: File,
pathConverter: FileToPathConverter,
storeFullFqNames: Boolean = false,
private val trackChanges: Boolean = false
private val trackChanges: Boolean = false,
private val transaction: CompilationTransaction = DummyCompilationTransaction(),
) : BasicMapsOwner(targetDataDir) {
val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")
@@ -134,9 +135,7 @@ open class LookupStorage(
@Synchronized
override fun clean() {
if (countersFile.exists()) {
countersFile.delete()
}
transaction.deleteFile(countersFile.toPath())
size = 0
@@ -148,6 +147,7 @@ open class LookupStorage(
try {
if (size != oldSize) {
if (size > 0) {
transaction.registerAddedOrChangedFile(countersFile.toPath())
if (!countersFile.exists()) {
countersFile.parentFile.mkdirs()
countersFile.createNewFile()
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt
+21 -5
View File
@@ -28,6 +28,7 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
cachesRootDir: File,
rootProjectDir: File?,
protected val reporter: ICReporter,
transaction: CompilationTransaction,
storeFullFqNamesInLookupCache: Boolean = false,
trackChangesInLookupCache: Boolean = false
) : Closeable {
@@ -47,7 +48,13 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter, pathConverter).apply { registerCache() }
val lookupCache: LookupStorage =
LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache, trackChangesInLookupCache).apply { registerCache() }
LookupStorage(
lookupCacheDir,
pathConverter,
storeFullFqNamesInLookupCache,
trackChangesInLookupCache,
transaction,
).apply { registerCache() }
abstract val platformCache: PlatformCache
@Suppress("UnstableApiUsage")
@@ -81,11 +88,13 @@ class IncrementalJvmCachesManager(
outputDir: File,
reporter: ICReporter,
storeFullFqNamesInLookupCache: Boolean = false,
trackChangesInLookupCache: Boolean = false
trackChangesInLookupCache: Boolean = false,
transaction: CompilationTransaction = DummyCompilationTransaction(),
) : IncrementalCachesManager<IncrementalJvmCache>(
cacheDirectory,
rootProjectDir,
reporter,
transaction,
storeFullFqNamesInLookupCache,
trackChangesInLookupCache
) {
@@ -98,8 +107,15 @@ class IncrementalJsCachesManager(
rootProjectDir: File?,
reporter: ICReporter,
serializerProtocol: SerializerExtensionProtocol,
storeFullFqNamesInLookupCache: Boolean
) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, rootProjectDir, reporter, storeFullFqNamesInLookupCache) {
storeFullFqNamesInLookupCache: Boolean,
transaction: CompilationTransaction = DummyCompilationTransaction(),
) : IncrementalCachesManager<IncrementalJsCache>(
cachesRootDir,
rootProjectDir,
reporter,
transaction,
storeFullFqNamesInLookupCache,
) {
private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() }
override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol).apply { registerCache() }
override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol, transaction).apply { registerCache() }
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt
+82 -73
View File
@@ -80,7 +80,7 @@ abstract class IncrementalCompilerRunner<
private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager
protected abstract fun createCacheManager(args: Args, projectDir: File?, transaction: CompilationTransaction): CacheManager
protected abstract fun destinationDir(args: Args): File
fun compile(
@@ -158,86 +158,95 @@ abstract class IncrementalCompilerRunner<
}
changedFiles as ChangedFiles.Known?
val caches = createCacheManager(args, projectDir)
createTransaction().use { transaction ->
val caches = createCacheManager(args, projectDir, transaction)
fun compile(): ICResult {
// Step 1: Get changed files
val knownChangedFiles: ChangedFiles.Known = try {
getChangedFiles(changedFiles, allSourceFiles, caches)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
}
val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
// Step 2: Compute files to recompile
val compilationMode = try {
reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
fun compile(): ICResult {
// Step 1: Get changed files
val knownChangedFiles: ChangedFiles.Known = try {
getChangedFiles(changedFiles, allSourceFiles, caches)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
}
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
}
if (compilationMode is CompilationMode.Rebuild) {
return ICResult.RequiresRebuild(compilationMode.reason)
}
val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
val abiSnapshotData = if (withAbiSnapshot) {
if (!abiSnapshotFile.exists()) {
reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
// Step 2: Compute files to recompile
val compilationMode = try {
reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
calculateSourcesToCompile(caches, knownChangedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
}
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
}
reporter.info { "Incremental compilation with ABI snapshot enabled" }
AbiSnapshotData(
snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
classpathAbiSnapshot = classpathAbiSnapshot!!
)
} else null
// Step 3: Compile incrementally
val exitCode = try {
compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
if (compilationMode is CompilationMode.Rebuild) {
return ICResult.RequiresRebuild(compilationMode.reason)
}
val abiSnapshotData = if (withAbiSnapshot) {
if (!abiSnapshotFile.exists()) {
reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
}
reporter.info { "Incremental compilation with ABI snapshot enabled" }
AbiSnapshotData(
snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
classpathAbiSnapshot = classpathAbiSnapshot!!
)
} else null
// Step 3: Compile incrementally
val exitCode = try {
compileImpl(
compilationMode as CompilationMode.Incremental,
allSourceFiles,
args,
caches,
abiSnapshotData,
messageCollector,
transaction
)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
}
return ICResult.Completed(exitCode)
}
return ICResult.Completed(exitCode)
}
fun closeCaches(caches: CacheManager, activeException: Throwable) {
try {
caches.close()
} catch (e: Throwable) {
activeException.addSuppressed(e)
}
}
fun closeCaches(caches: CacheManager, activeException: Throwable) {
// Because `caches` is a Closeable resource, it is important to close them in both cases:
// 1. in the event of an exception
// 2. after a normal execution
// Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the
// future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in
// `compileNonIncrementally`).
val icResult = try {
compile()
} catch (e: Throwable) {
// Case 1 - Close the caches upon an exception
closeCaches(caches, e)
throw e
}
// Case 2 - Close the caches after a normal execution
try {
caches.close()
} catch (e: Throwable) {
activeException.addSuppressed(e)
return ICResult.Failed(
IC_FAILED_TO_CLOSE_CACHES,
RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e)
)
}
return icResult
}
// Because `caches` is a Closeable resource, it is important to close them in both cases:
// 1. in the event of an exception
// 2. after a normal execution
// Note: Historically, closing caches used to throw exceptions sometimes, so currently we want to collect those exceptions. In the
// future, if closing caches is safe, we can simplify the code by using Kotlin's `Closable.use` function (similar to the code in
// `compileNonIncrementally`).
val icResult = try {
compile()
} catch (e: Throwable) {
// Case 1 - Close the caches upon an exception
closeCaches(caches, e)
throw e
}
// Case 2 - Close the caches after a normal execution
try {
caches.close()
} catch (e: Throwable) {
return ICResult.Failed(
IC_FAILED_TO_CLOSE_CACHES,
RuntimeException("Failed to close caches, previous ICResult `$icResult` was discarded", e)
)
}
return icResult
}
private fun compileNonIncrementally(
@@ -257,7 +266,8 @@ abstract class IncrementalCompilerRunner<
reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" }
cleanOrCreateDirectories(outputDirsToClean)
}
return createCacheManager(args, projectDir).use { caches ->
val transaction = DummyCompilationTransaction()
return createCacheManager(args, projectDir, transaction).use { caches ->
if (trackChangedFiles) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
}
@@ -265,7 +275,7 @@ abstract class IncrementalCompilerRunner<
AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
} else null
compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector)
compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector, transaction)
}
}
@@ -378,14 +388,13 @@ abstract class IncrementalCompilerRunner<
args: Args,
caches: CacheManager,
abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
messageCollector: MessageCollector
messageCollector: MessageCollector,
transaction: CompilationTransaction,
): ExitCode {
performWorkBeforeCompilation(compilationMode, args)
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
val exitCode = createTransaction().use { transaction ->
doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)
}
val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)
performWorkAfterCompilation(compilationMode, exitCode, caches)
return exitCode
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt
+3 -2
View File
@@ -99,14 +99,15 @@ class IncrementalJsCompilerRunner(
preciseCompilationResultsBackup = preciseCompilationResultsBackup,
) {
override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJsCachesManager {
val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
return IncrementalJsCachesManager(
cacheDirectory,
projectDir,
reporter,
serializerProtocol,
storeFullFqNamesInLookupCache = withAbiSnapshot
storeFullFqNamesInLookupCache = withAbiSnapshot,
transaction = transaction,
)
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt
+3 -2
View File
@@ -150,14 +150,15 @@ open class IncrementalJvmCompilerRunner(
withAbiSnapshot = withAbiSnapshot,
preciseCompilationResultsBackup = preciseCompilationResultsBackup,
) {
override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJvmCachesManager =
IncrementalJvmCachesManager(
cacheDirectory,
projectDir,
File(args.destination),
reporter,
storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun,
transaction = transaction,
)
override fun destinationDir(args: K2JVMCompilerArguments): File =
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt
+2 -1
View File
@@ -21,6 +21,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.incremental.storage.StorageOwner
import org.jetbrains.kotlin.incremental.DummyCompilationTransaction
import org.jetbrains.kotlin.incremental.IncrementalCacheCommon
import org.jetbrains.kotlin.incremental.IncrementalJsCache
import org.jetbrains.kotlin.incremental.IncrementalJvmCache
@@ -54,7 +55,7 @@ class JpsIncrementalJsCache(
target: ModuleBuildTarget,
paths: BuildDataPaths,
pathConverter: FileToPathConverter
) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol), JpsIncrementalCache {
) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol, DummyCompilationTransaction()), JpsIncrementalCache {
override fun addJpsDependentCache(cache: JpsIncrementalCache) {
if (cache is JpsIncrementalJsCache) {
addDependentCache(cache)