azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IC] Introduce ICContext to simplify configuration propagation to caches

Browse Source 
#KT-49785 In Progress
This commit is contained in:
Alexander.Likhachev
2023-01-10 00:52:21 +01:00
committed by Space Team
parent 7171c2531c
commit 581bc89849
32 changed files with 311 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt
+13 -47
View File
@@ -17,22 +17,15 @@
package org.jetbrains.kotlin.incremental
import com.google.common.io.Closer
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
import java.io.Closeable
import java.io.File
abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache<*>>(
icContext: IncrementalCompilationContext,
cachesRootDir: File,
rootProjectDir: File?,
protected val reporter: ICReporter,
transaction: CompilationTransaction,
storeFullFqNamesInLookupCache: Boolean = false,
trackChangesInLookupCache: Boolean = false
) : Closeable {
val pathConverter = IncrementalFileToPathConverter(rootProjectDir)
private val caches = arrayListOf<BasicMapsOwner>()
private var isClosed = false
@@ -46,15 +39,8 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
private val inputSnapshotsCacheDir = File(cachesRootDir, "inputs").apply { mkdirs() }
private val lookupCacheDir = File(cachesRootDir, "lookups").apply { mkdirs() }
val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter, pathConverter).apply { registerCache() }
val lookupCache: LookupStorage =
LookupStorage(
lookupCacheDir,
pathConverter,
storeFullFqNamesInLookupCache,
trackChangesInLookupCache,
transaction,
).apply { registerCache() }
val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, icContext).apply { registerCache() }
val lookupCache: LookupStorage = LookupStorage(lookupCacheDir, icContext).apply { registerCache() }
abstract val platformCache: PlatformCache
@Suppress("UnstableApiUsage")
@@ -83,39 +69,19 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
}
class IncrementalJvmCachesManager(
cacheDirectory: File,
rootProjectDir: File?,
outputDir: File,
reporter: ICReporter,
storeFullFqNamesInLookupCache: Boolean = false,
trackChangesInLookupCache: Boolean = false,
transaction: CompilationTransaction = DummyCompilationTransaction(),
) : IncrementalCachesManager<IncrementalJvmCache>(
cacheDirectory,
rootProjectDir,
reporter,
transaction,
storeFullFqNamesInLookupCache,
trackChangesInLookupCache
) {
private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() }
override val platformCache = IncrementalJvmCache(jvmCacheDir, outputDir, pathConverter).apply { registerCache() }
icContext: IncrementalCompilationContext,
outputDir: File?,
cachesRootDir: File,
) : IncrementalCachesManager<IncrementalJvmCache>(icContext, cachesRootDir) {
private val jvmCacheDir = File(cachesRootDir, "jvm").apply { mkdirs() }
override val platformCache = IncrementalJvmCache(jvmCacheDir, icContext, outputDir).apply { registerCache() }
}
class IncrementalJsCachesManager(
cachesRootDir: File,
rootProjectDir: File?,
reporter: ICReporter,
icContext: IncrementalCompilationContext,
serializerProtocol: SerializerExtensionProtocol,
storeFullFqNamesInLookupCache: Boolean,
transaction: CompilationTransaction = DummyCompilationTransaction(),
) : IncrementalCachesManager<IncrementalJsCache>(
cachesRootDir,
rootProjectDir,
reporter,
transaction,
storeFullFqNamesInLookupCache,
) {
cachesRootDir: File,
) : IncrementalCachesManager<IncrementalJsCache>(icContext, cachesRootDir) {
private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() }
override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol, transaction).apply { registerCache() }
override val platformCache = IncrementalJsCache(jsCacheFile, icContext, serializerProtocol).apply { registerCache() }
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt
+21 -11
View File
@@ -80,7 +80,15 @@ abstract class IncrementalCompilerRunner<
private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
protected abstract fun createCacheManager(args: Args, projectDir: File?, transaction: CompilationTransaction): CacheManager
/**
* Creates an instance of [IncrementalCompilationContext] that holds common incremental compilation context mostly required for [CacheManager]
*/
protected abstract fun createIncrementalCompilationContext(
projectDir: File?,
transaction: CompilationTransaction
): IncrementalCompilationContext
protected abstract fun createCacheManager(icContext: IncrementalCompilationContext, args: Args): CacheManager
protected abstract fun destinationDir(args: Args): File
fun compile(
@@ -159,7 +167,8 @@ abstract class IncrementalCompilerRunner<
changedFiles as ChangedFiles.Known?
createTransaction().use { transaction ->
val caches = createCacheManager(args, projectDir, transaction)
val icContext = createIncrementalCompilationContext(projectDir, transaction)
val caches = createCacheManager(icContext, args)
fun compile(): ICResult {
// Step 1: Get changed files
@@ -199,13 +208,13 @@ abstract class IncrementalCompilerRunner<
// Step 3: Compile incrementally
val exitCode = try {
compileImpl(
icContext,
compilationMode as CompilationMode.Incremental,
allSourceFiles,
args,
caches,
abiSnapshotData,
messageCollector,
transaction
)
} catch (e: Throwable) {
return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
@@ -269,8 +278,8 @@ abstract class IncrementalCompilerRunner<
reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" }
cleanOrCreateDirectories(outputDirsToClean)
}
val transaction = DummyCompilationTransaction()
return createCacheManager(args, projectDir, transaction).use { caches ->
val icContext = createIncrementalCompilationContext(projectDir, DummyCompilationTransaction())
return createCacheManager(icContext, args).use { caches ->
if (trackChangedFiles) {
caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
}
@@ -278,7 +287,7 @@ abstract class IncrementalCompilerRunner<
AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
} else null
compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector, transaction)
compileImpl(icContext, CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector)
}
}
@@ -386,18 +395,18 @@ abstract class IncrementalCompilerRunner<
): Pair<ExitCode, Collection<File>>
private fun compileImpl(
icContext: IncrementalCompilationContext,
compilationMode: CompilationMode,
allSourceFiles: List<File>,
args: Args,
caches: CacheManager,
abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
messageCollector: MessageCollector,
transaction: CompilationTransaction,
): ExitCode {
performWorkBeforeCompilation(compilationMode, args)
val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector, transaction)
val exitCode = doCompile(icContext, caches, compilationMode, allKotlinFiles, args, abiSnapshotData, messageCollector)
performWorkAfterCompilation(compilationMode, exitCode, caches)
return exitCode
@@ -426,13 +435,13 @@ abstract class IncrementalCompilerRunner<
}
private fun doCompile(
icContext: IncrementalCompilationContext,
caches: CacheManager,
compilationMode: CompilationMode,
allKotlinSources: List<File>,
args: Args,
caches: CacheManager,
abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
originalMessageCollector: MessageCollector,
transaction: CompilationTransaction,
): ExitCode {
val dirtySources = when (compilationMode) {
is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableLinkedSet()
@@ -443,6 +452,7 @@ abstract class IncrementalCompilerRunner<
val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
val buildDirtyFqNames = HashSet<FqName>()
val allDirtySources = HashSet<File>()
val transaction = icContext.transaction
var exitCode = ExitCode.OK
@@ -454,7 +464,7 @@ abstract class IncrementalCompilerRunner<
val complementaryFiles = caches.platformCache.getComplementaryFilesRecursive(dirtySources)
dirtySources.addAll(complementaryFiles)
caches.platformCache.markDirty(dirtySources)
caches.inputsCache.removeOutputForSourceFiles(dirtySources, transaction)
caches.inputsCache.removeOutputForSourceFiles(dirtySources)
val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
val expectActualTracker = ExpectActualTrackerImpl()
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt
+1 -1
View File
@@ -259,7 +259,7 @@ class IncrementalFirJvmCompilerRunner(
else -> it + newDirtyFilesOutputsScope
}
}
caches.inputsCache.removeOutputForSourceFiles(newDirtySources, DummyCompilationTransaction())
caches.inputsCache.removeOutputForSourceFiles(newDirtySources)
newDirtySources.forEach {
dirtySources.add(KtIoFileSourceFile(it))
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt
+8 -10
View File
@@ -98,18 +98,16 @@ class IncrementalJsCompilerRunner(
withAbiSnapshot = withAbiSnapshot,
preciseCompilationResultsBackup = preciseCompilationResultsBackup,
) {
override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJsCachesManager {
val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
return IncrementalJsCachesManager(
cacheDirectory,
projectDir,
reporter,
serializerProtocol,
storeFullFqNamesInLookupCache = withAbiSnapshot,
override fun createIncrementalCompilationContext(projectDir: File?, transaction: CompilationTransaction) =
IncrementalCompilationContext(
transaction = transaction,
rootProjectDir = projectDir,
reporter = reporter,
storeFullFqNamesInLookupCache = withAbiSnapshot,
)
}
override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JSCompilerArguments) =
IncrementalJsCachesManager(icContext, if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol, cacheDirectory)
override fun destinationDir(args: K2JSCompilerArguments): File {
return if (args.isIrBackendEnabled())
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt
+9 -8
View File
@@ -150,17 +150,18 @@ open class IncrementalJvmCompilerRunner(
withAbiSnapshot = withAbiSnapshot,
preciseCompilationResultsBackup = preciseCompilationResultsBackup,
) {
override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?, transaction: CompilationTransaction): IncrementalJvmCachesManager =
IncrementalJvmCachesManager(
cacheDirectory,
projectDir,
File(args.destination),
reporter,
storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun,
override fun createIncrementalCompilationContext(projectDir: File?, transaction: CompilationTransaction) =
IncrementalCompilationContext(
transaction = transaction,
rootProjectDir = projectDir,
reporter = reporter,
trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun,
storeFullFqNamesInLookupCache = withAbiSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
)
override fun createCacheManager(icContext: IncrementalCompilationContext, args: K2JVMCompilerArguments) =
IncrementalJvmCachesManager(icContext, args.destination?.let { File(it) }, cacheDirectory)
override fun destinationDir(args: K2JVMCompilerArguments): File =
args.destinationAsFile
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt
+6 -12
View File
@@ -18,35 +18,29 @@ package org.jetbrains.kotlin.incremental
import com.intellij.util.containers.MultiMap
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.debug
import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap
import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.incremental.storage.FileToPathConverter
import org.jetbrains.kotlin.incremental.storage.SourceToOutputFilesMap
import java.io.File
class InputsCache(
workingDir: File,
private val reporter: ICReporter,
pathConverter: FileToPathConverter
private val icContext: IncrementalCompilationContext,
) : BasicMapsOwner(workingDir) {
companion object {
private const val SOURCE_SNAPSHOTS = "source-snapshot"
private const val SOURCE_TO_OUTPUT_FILES = "source-to-output"
}
internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile, pathConverter))
private val sourceToOutputMap = registerMap(SourceToOutputFilesMap(SOURCE_TO_OUTPUT_FILES.storageFile, pathConverter))
internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile, icContext))
private val sourceToOutputMap = registerMap(SourceToOutputFilesMap(SOURCE_TO_OUTPUT_FILES.storageFile, icContext))
fun removeOutputForSourceFiles(
sources: Iterable<File>,
transaction: CompilationTransaction,
) {
fun removeOutputForSourceFiles(sources: Iterable<File>) {
for (sourceFile in sources) {
sourceToOutputMap.remove(sourceFile).forEach {
reporter.debug { "Deleting $it on clearing cache for $sourceFile" }
transaction.deleteFile(it.toPath())
icContext.reporter.debug { "Deleting $it on clearing cache for $sourceFile" }
icContext.transaction.deleteFile(it.toPath())
}
}
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt
+2 -1
View File
@@ -203,7 +203,8 @@ object ClasspathChangesComputer {
): ProgramSymbolSet {
val workingDir =
FileUtil.createTempDirectory(this::class.java.simpleName, "_WorkingDir_${UUID.randomUUID()}", /* deleteOnExit */ true)
val incrementalJvmCache = IncrementalJvmCache(workingDir, /* targetOutputDir */ null, FileToAbsolutePathConverter)
val icContext = IncrementalCompilationContext(pathConverter = FileToAbsolutePathConverter)
val incrementalJvmCache = IncrementalJvmCache(workingDir, icContext, null)
// Step 1:
// - Add previous class snapshots to incrementalJvmCache.
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt
+3 -3
View File
@@ -17,16 +17,16 @@
package org.jetbrains.kotlin.incremental.snapshots
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.storage.BasicStringMap
import org.jetbrains.kotlin.incremental.storage.FileToPathConverter
import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
import java.io.File
import java.util.*
class FileSnapshotMap(
storageFile: File,
private val pathConverter: FileToPathConverter
) : BasicStringMap<FileSnapshot>(storageFile, PathStringDescriptor, FileSnapshotExternalizer) {
icContext: IncrementalCompilationContext,
) : BasicStringMap<FileSnapshot>(storageFile, PathStringDescriptor, FileSnapshotExternalizer, icContext) {
override fun dumpValue(value: FileSnapshot): String =
value.toString()
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt
+3 -3
View File
@@ -5,14 +5,14 @@
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File
class SourceToOutputFilesMap(
storageFile: File,
private val pathConverter: FileToPathConverter
) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
icContext: IncrementalCompilationContext,
) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) {
operator fun set(sourceFile: File, outputFiles: Collection<File>) {
storage[pathConverter.toPath(sourceFile)] = outputFiles.map(pathConverter::toPath)
}
compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt
+6 -3
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.incremental.snapshots
import org.jetbrains.kotlin.TestWithWorkingDir
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.storage.FileToPathConverter
import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
import org.junit.After
@@ -17,15 +18,17 @@ import kotlin.properties.Delegates
class FileSnapshotMapTest : TestWithWorkingDir() {
private var snapshotMap: FileSnapshotMap by Delegates.notNull()
private var pathConverter: FileToPathConverter by Delegates.notNull()
@Before
override fun setUp() {
super.setUp()
val caches = File(workingDir, "caches").apply { mkdirs() }
val snapshotMapFile = File(caches, "snapshots.tab")
pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile))
snapshotMap = FileSnapshotMap(snapshotMapFile, pathConverter)
val pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile))
val icContext = IncrementalCompilationContext(
pathConverter = pathConverter
)
snapshotMap = FileSnapshotMap(snapshotMapFile, icContext)
}
@After
compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt
+5 -1
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.TestWithWorkingDir
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.junit.After
import org.junit.Assert.assertArrayEquals
import org.junit.Before
@@ -23,7 +24,10 @@ class SourceToOutputFilesMapTest : TestWithWorkingDir() {
val caches = File(workingDir, "caches").apply { mkdirs() }
val stofMapFile = File(caches, "stof.tab")
pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile))
stofMap = SourceToOutputFilesMap(stofMapFile, pathConverter)
val icContext = IncrementalCompilationContext(
pathConverter = pathConverter
)
stofMap = SourceToOutputFilesMap(stofMapFile, icContext)
}
@After