From 8a49c29a360c9e9cca528ba609877d6909a1b8a0 Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Fri, 12 Apr 2019 16:35:13 +0300 Subject: [PATCH] Allow customizing source file path conversion in lookup storage Original commit: 00de7b6c44afe89af2dd02b5aa4d1776a7726542 --- .../jps/build/AbstractIncrementalJpsTest.kt | 11 ++-- .../kotlin/jps/build/KotlinBuilder.kt | 20 +++---- .../kotlin/jps/build/KotlinCompileContext.kt | 10 ++-- .../jps/build/KotlinJavaBuilderExtension.kt | 5 +- .../jps/incremental/JpsLookupStorage.kt | 54 ++++++++++++------- 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt index d577dfb2620..cfcd0b52056 100644 --- a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt +++ b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt @@ -204,7 +204,7 @@ abstract class AbstractIncrementalJpsTest( return MakeResult( log = logger.log, makeFailed = false, - mappingsDump = createMappingsDump(projectDescriptor), + mappingsDump = createMappingsDump(projectDescriptor, kotlinCompileContext), name = name ) } @@ -327,9 +327,10 @@ abstract class AbstractIncrementalJpsTest( } private fun createMappingsDump( - project: ProjectDescriptor + project: ProjectDescriptor, + kotlinContext: KotlinCompileContext ) = createKotlinIncrementalCacheDump(project) + "\n\n\n" + - createLookupCacheDump(project) + "\n\n\n" + + createLookupCacheDump(kotlinContext) + "\n\n\n" + createCommonMappingsDump(project) + "\n\n\n" + createJavaMappingsDump(project) @@ -348,13 +349,13 @@ abstract class AbstractIncrementalJpsTest( } } - private fun createLookupCacheDump(project: ProjectDescriptor): String { + private fun createLookupCacheDump(kotlinContext: KotlinCompileContext): String { val sb = StringBuilder() val p = Printer(sb) p.println("Begin of Lookup Maps") p.println() - project.dataManager.withLookupStorage { lookupStorage -> + kotlinContext.lookupStorageManager.withLookupStorage { lookupStorage -> lookupStorage.forceGC() p.print(lookupStorage.dump(lookupsDuringTest)) } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index ae3bb106cd9..db54b5978ad 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -45,7 +45,7 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.ICReporterBase import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache -import org.jetbrains.kotlin.jps.incremental.withLookupStorage +import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager import org.jetbrains.kotlin.jps.model.kotlinKind import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget @@ -249,7 +249,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val changesCollector = ChangesCollector() removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) } - val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, context.projectDescriptor.dataManager) + val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, kotlinContext.lookupStorageManager) fsOperations.markFilesForCurrentRound(affectedByRemovedClasses) } @@ -471,12 +471,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { ) } - updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder) + updateLookupStorage(lookupTracker, kotlinContext.lookupStorageManager, kotlinDirtyFilesHolder) if (!isChunkRebuilding) { changesCollector.processChangesUsingLookups( kotlinDirtyFilesHolder.allDirtyFiles, - dataManager, + kotlinContext.lookupStorageManager, fsOperations, incrementalCaches.values ) @@ -643,13 +643,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { private fun updateLookupStorage( lookupTracker: LookupTracker, - dataManager: BuildDataManager, + lookupStorageManager: JpsLookupStorageManager, dirtyFilesHolder: KotlinDirtySourceFilesHolder ) { if (lookupTracker !is LookupTrackerImpl) throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}") - dataManager.withLookupStorage { lookupStorage -> + lookupStorageManager.withLookupStorage { lookupStorage -> lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence()) lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values) } @@ -673,7 +673,7 @@ private class JpsICReporter : ICReporterBase() { private fun ChangesCollector.processChangesUsingLookups( compiledFiles: Set, - dataManager: BuildDataManager, + lookupStorageManager: JpsLookupStorageManager, fsOperations: FSOperationsHelper, caches: Iterable ) { @@ -682,7 +682,7 @@ private fun ChangesCollector.processChangesUsingLookups( reporter.reportVerbose { "Start processing changes" } - val dirtyFiles = getDirtyFiles(allCaches, dataManager) + val dirtyFiles = getDirtyFiles(allCaches, lookupStorageManager) fsOperations.markInChunkOrDependents(dirtyFiles.asIterable(), excludeFiles = compiledFiles) reporter.reportVerbose { "End of processing changes" } @@ -690,11 +690,11 @@ private fun ChangesCollector.processChangesUsingLookups( private fun ChangesCollector.getDirtyFiles( caches: Iterable, - dataManager: BuildDataManager + lookupStorageManager: JpsLookupStorageManager ): Set { val reporter = JpsICReporter() val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(caches, reporter) - val dirtyFilesFromLookups = dataManager.withLookupStorage { + val dirtyFilesFromLookups = lookupStorageManager.withLookupStorage { mapLookupSymbolsToFiles(it, dirtyLookupSymbols, reporter) } return dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter) diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt index ec3ff7385b1..604ac019265 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt @@ -70,6 +70,8 @@ class KotlinCompileContext(val jpsContext: CompileContext) { val sourceFileToPathConverter: SourceFileToPathConverter = SourceFileToCanonicalPathConverter + val lookupStorageManager = JpsLookupStorageManager(dataManager, sourceFileToPathConverter) + /** * Flag to prevent rebuilding twice. * @@ -107,7 +109,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) { // try to perform a lookup // request rebuild if storage is corrupted try { - dataManager.withLookupStorage { + lookupStorageManager.withLookupStorage { it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>")) } } catch (e: Exception) { @@ -189,13 +191,11 @@ class KotlinCompileContext(val jpsContext: CompileContext) { KotlinBuilder.LOG.info("Rebuilding all Kotlin: $reason") - val dataManager = jpsContext.projectDescriptor.dataManager - targetsIndex.chunks.forEach { markChunkForRebuildBeforeBuild(it) } - dataManager.cleanLookupStorage(KotlinBuilder.LOG) + lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG) } private fun markChunkForRebuildBeforeBuild(chunk: KotlinChunk) { @@ -223,7 +223,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) { private fun clearLookupCache() { KotlinBuilder.LOG.info("Clearing lookup cache") - dataManager.cleanLookupStorage(KotlinBuilder.LOG) + lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG) initialLookupsCacheStateDiff.manager.writeVersion() } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt index 63c9e0ba471..b3636498cec 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt @@ -10,7 +10,6 @@ import org.jetbrains.jps.builders.java.JavaBuilderExtension import org.jetbrains.jps.builders.java.dependencyView.Callbacks import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.kotlin.incremental.LookupSymbol -import org.jetbrains.kotlin.jps.incremental.withLookupStorage import java.io.File import java.util.concurrent.Executors import java.util.concurrent.Future @@ -24,7 +23,7 @@ class KotlinJavaBuilderExtension : JavaBuilderExtension() { private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.ConstantAffectionResolver { private val pool = Executors.newSingleThreadExecutor() - private val dataManager = context.projectDescriptor.dataManager + private val kotlinContext by lazy { context.kotlin } override fun request( ownerClassName: String, @@ -54,7 +53,7 @@ private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.Co } pool.submit { if (!future.isCancelled) { - dataManager.withLookupStorage { storage -> + kotlinContext.lookupStorageManager.withLookupStorage { storage -> val paths = storage.get(LookupSymbol(name = fieldName, scope = ownerClassName)) future.result(paths.map { File(it) }) } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt index c68c128b36f..45330f65cf0 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt @@ -22,36 +22,50 @@ import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.StorageOwner import org.jetbrains.kotlin.incremental.LookupStorage +import org.jetbrains.kotlin.incremental.storage.SourceFileToPathConverter import java.io.File import java.io.IOException private object LookupStorageLock -fun BuildDataManager.cleanLookupStorage(log: Logger) { - synchronized(LookupStorageLock) { - try { - cleanTargetStorages(KotlinDataContainerTarget) - } catch (e: IOException) { - if (!dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) { - log.debug("Could not clear lookup storage caches", e) +class JpsLookupStorageManager( + private val buildDataManager: BuildDataManager, + sourcePathConverter: SourceFileToPathConverter +) { + private val storageProvider = JpsLookupStorageProvider(sourcePathConverter) + + fun cleanLookupStorage(log: Logger) { + synchronized(LookupStorageLock) { + try { + buildDataManager.cleanTargetStorages(KotlinDataContainerTarget) + } catch (e: IOException) { + if (!buildDataManager.dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) { + log.debug("Could not clear lookup storage caches", e) + } } } } -} -fun BuildDataManager.withLookupStorage(fn: (LookupStorage) -> T): T { - synchronized(LookupStorageLock) { - try { - val lookupStorage = getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider) - return fn(lookupStorage) - } catch (e: IOException) { - throw BuildDataCorruptedException(e) + fun withLookupStorage(fn: (LookupStorage) -> T): T { + synchronized(LookupStorageLock) { + try { + val lookupStorage = buildDataManager.getStorage(KotlinDataContainerTarget, storageProvider) + return fn(lookupStorage) + } catch (e: IOException) { + throw BuildDataCorruptedException(e) + } } } -} -private object JpsLookupStorageProvider : StorageProvider() { - override fun createStorage(targetDataDir: File): JpsLookupStorage = JpsLookupStorage(targetDataDir) -} + private class JpsLookupStorageProvider( + private val sourcePathConverter: SourceFileToPathConverter + ) : StorageProvider() { + override fun createStorage(targetDataDir: File): JpsLookupStorage = + JpsLookupStorage(targetDataDir, sourcePathConverter) + } -private class JpsLookupStorage(targetDataDir: File) : StorageOwner, LookupStorage(targetDataDir) + private class JpsLookupStorage( + targetDataDir: File, + sourcePathConverter: SourceFileToPathConverter + ) : StorageOwner, LookupStorage(targetDataDir, sourcePathConverter) +}