Allow customizing source file path conversion in lookup storage

Original commit: 00de7b6c44

jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt

@@ -204,7 +204,7 @@ abstract class AbstractIncrementalJpsTest(
                 return MakeResult(
                     log = logger.log,
                     makeFailed = false,
-                    mappingsDump = createMappingsDump(projectDescriptor),
+                    mappingsDump = createMappingsDump(projectDescriptor, kotlinCompileContext),
                     name = name
                 )
             }
@@ -327,9 +327,10 @@ abstract class AbstractIncrementalJpsTest(
     }
 
     private fun createMappingsDump(
-        project: ProjectDescriptor
+        project: ProjectDescriptor,
+        kotlinContext: KotlinCompileContext
     ) = createKotlinIncrementalCacheDump(project) + "\n\n\n" +
-            createLookupCacheDump(project) + "\n\n\n" +
+            createLookupCacheDump(kotlinContext) + "\n\n\n" +
             createCommonMappingsDump(project) + "\n\n\n" +
             createJavaMappingsDump(project)
 
@@ -348,13 +349,13 @@ abstract class AbstractIncrementalJpsTest(
         }
     }
 
-    private fun createLookupCacheDump(project: ProjectDescriptor): String {
+    private fun createLookupCacheDump(kotlinContext: KotlinCompileContext): String {
         val sb = StringBuilder()
         val p = Printer(sb)
         p.println("Begin of Lookup Maps")
         p.println()
 
-        project.dataManager.withLookupStorage { lookupStorage ->
+        kotlinContext.lookupStorageManager.withLookupStorage { lookupStorage ->
             lookupStorage.forceGC()
             p.print(lookupStorage.dump(lookupsDuringTest))
         }

jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -45,7 +45,7 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.ICReporterBase
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
-import org.jetbrains.kotlin.jps.incremental.withLookupStorage
+import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
 import org.jetbrains.kotlin.jps.model.kotlinKind
 import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget
 import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget
@@ -249,7 +249,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
 
         val changesCollector = ChangesCollector()
         removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) }
-        val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, context.projectDescriptor.dataManager)
+        val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, kotlinContext.lookupStorageManager)
 
         fsOperations.markFilesForCurrentRound(affectedByRemovedClasses)
     }
@@ -471,12 +471,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
                 )
             }
 
-            updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder)
+            updateLookupStorage(lookupTracker, kotlinContext.lookupStorageManager, kotlinDirtyFilesHolder)
 
             if (!isChunkRebuilding) {
                 changesCollector.processChangesUsingLookups(
                     kotlinDirtyFilesHolder.allDirtyFiles,
-                    dataManager,
+                    kotlinContext.lookupStorageManager,
                     fsOperations,
                     incrementalCaches.values
                 )
@@ -643,13 +643,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
 
     private fun updateLookupStorage(
         lookupTracker: LookupTracker,
-        dataManager: BuildDataManager,
+        lookupStorageManager: JpsLookupStorageManager,
         dirtyFilesHolder: KotlinDirtySourceFilesHolder
     ) {
         if (lookupTracker !is LookupTrackerImpl)
             throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
 
-        dataManager.withLookupStorage { lookupStorage ->
+        lookupStorageManager.withLookupStorage { lookupStorage ->
             lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence())
             lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values)
         }
@@ -673,7 +673,7 @@ private class JpsICReporter : ICReporterBase() {
 
 private fun ChangesCollector.processChangesUsingLookups(
     compiledFiles: Set<File>,
-    dataManager: BuildDataManager,
+    lookupStorageManager: JpsLookupStorageManager,
     fsOperations: FSOperationsHelper,
     caches: Iterable<JpsIncrementalCache>
 ) {
@@ -682,7 +682,7 @@ private fun ChangesCollector.processChangesUsingLookups(
 
     reporter.reportVerbose { "Start processing changes" }
 
-    val dirtyFiles = getDirtyFiles(allCaches, dataManager)
+    val dirtyFiles = getDirtyFiles(allCaches, lookupStorageManager)
     fsOperations.markInChunkOrDependents(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
 
     reporter.reportVerbose { "End of processing changes" }
@@ -690,11 +690,11 @@ private fun ChangesCollector.processChangesUsingLookups(
 
 private fun ChangesCollector.getDirtyFiles(
     caches: Iterable<IncrementalCacheCommon>,
-    dataManager: BuildDataManager
+    lookupStorageManager: JpsLookupStorageManager
 ): Set<File> {
     val reporter = JpsICReporter()
     val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(caches, reporter)
-    val dirtyFilesFromLookups = dataManager.withLookupStorage {
+    val dirtyFilesFromLookups = lookupStorageManager.withLookupStorage {
         mapLookupSymbolsToFiles(it, dirtyLookupSymbols, reporter)
     }
     return dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter)

jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt

@@ -70,6 +70,8 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
 
     val sourceFileToPathConverter: SourceFileToPathConverter = SourceFileToCanonicalPathConverter
 
+    val lookupStorageManager = JpsLookupStorageManager(dataManager, sourceFileToPathConverter)
+
     /**
      * Flag to prevent rebuilding twice.
      *
@@ -107,7 +109,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
             // try to perform a lookup
             // request rebuild if storage is corrupted
             try {
-                dataManager.withLookupStorage {
+                lookupStorageManager.withLookupStorage {
                     it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>"))
                 }
             } catch (e: Exception) {
@@ -189,13 +191,11 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
 
         KotlinBuilder.LOG.info("Rebuilding all Kotlin: $reason")
 
-        val dataManager = jpsContext.projectDescriptor.dataManager
-
         targetsIndex.chunks.forEach {
             markChunkForRebuildBeforeBuild(it)
         }
 
-        dataManager.cleanLookupStorage(KotlinBuilder.LOG)
+        lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG)
     }
 
     private fun markChunkForRebuildBeforeBuild(chunk: KotlinChunk) {
@@ -223,7 +223,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
 
     private fun clearLookupCache() {
         KotlinBuilder.LOG.info("Clearing lookup cache")
-        dataManager.cleanLookupStorage(KotlinBuilder.LOG)
+        lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG)
         initialLookupsCacheStateDiff.manager.writeVersion()
     }
 

jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt

@@ -10,7 +10,6 @@ import org.jetbrains.jps.builders.java.JavaBuilderExtension
 import org.jetbrains.jps.builders.java.dependencyView.Callbacks
 import org.jetbrains.jps.incremental.CompileContext
 import org.jetbrains.kotlin.incremental.LookupSymbol
-import org.jetbrains.kotlin.jps.incremental.withLookupStorage
 import java.io.File
 import java.util.concurrent.Executors
 import java.util.concurrent.Future
@@ -24,7 +23,7 @@ class KotlinJavaBuilderExtension : JavaBuilderExtension() {
 
 private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.ConstantAffectionResolver {
     private val pool = Executors.newSingleThreadExecutor()
-    private val dataManager = context.projectDescriptor.dataManager
+    private val kotlinContext by lazy { context.kotlin }
 
     override fun request(
         ownerClassName: String,
@@ -54,7 +53,7 @@ private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.Co
         }
         pool.submit {
             if (!future.isCancelled) {
-                dataManager.withLookupStorage { storage ->
+                kotlinContext.lookupStorageManager.withLookupStorage { storage ->
                     val paths = storage.get(LookupSymbol(name = fieldName, scope = ownerClassName))
                     future.result(paths.map { File(it) })
                 }

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt

@@ -22,36 +22,50 @@ import org.jetbrains.jps.builders.storage.StorageProvider
 import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.incremental.storage.StorageOwner
 import org.jetbrains.kotlin.incremental.LookupStorage
+import org.jetbrains.kotlin.incremental.storage.SourceFileToPathConverter
 import java.io.File
 import java.io.IOException
 
 private object LookupStorageLock
 
-fun BuildDataManager.cleanLookupStorage(log: Logger) {
+class JpsLookupStorageManager(
-    synchronized(LookupStorageLock) {
+    private val buildDataManager: BuildDataManager,
-        try {
+    sourcePathConverter: SourceFileToPathConverter
-            cleanTargetStorages(KotlinDataContainerTarget)
+) {
-        } catch (e: IOException) {
+    private val storageProvider = JpsLookupStorageProvider(sourcePathConverter)
-            if (!dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) {
+
-                log.debug("Could not clear lookup storage caches", e)
+    fun cleanLookupStorage(log: Logger) {
+        synchronized(LookupStorageLock) {
+            try {
+                buildDataManager.cleanTargetStorages(KotlinDataContainerTarget)
+            } catch (e: IOException) {
+                if (!buildDataManager.dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) {
+                    log.debug("Could not clear lookup storage caches", e)
+                }
             }
         }
     }
-}
 
-fun <T> BuildDataManager.withLookupStorage(fn: (LookupStorage) -> T): T {
+    fun <T> withLookupStorage(fn: (LookupStorage) -> T): T {
-    synchronized(LookupStorageLock) {
+        synchronized(LookupStorageLock) {
-        try {
+            try {
-            val lookupStorage = getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider)
+                val lookupStorage = buildDataManager.getStorage(KotlinDataContainerTarget, storageProvider)
-            return fn(lookupStorage)
+                return fn(lookupStorage)
-        } catch (e: IOException) {
+            } catch (e: IOException) {
-            throw BuildDataCorruptedException(e)
+                throw BuildDataCorruptedException(e)
+            }
         }
     }
-}
 
-private object JpsLookupStorageProvider : StorageProvider<JpsLookupStorage>() {
+    private class JpsLookupStorageProvider(
-    override fun createStorage(targetDataDir: File): JpsLookupStorage = JpsLookupStorage(targetDataDir)
+        private val sourcePathConverter: SourceFileToPathConverter
-}
+    ) : StorageProvider<JpsLookupStorage>() {
+        override fun createStorage(targetDataDir: File): JpsLookupStorage =
+            JpsLookupStorage(targetDataDir, sourcePathConverter)
+    }
 
-private class JpsLookupStorage(targetDataDir: File) : StorageOwner, LookupStorage(targetDataDir)
+    private class JpsLookupStorage(
+        targetDataDir: File,
+        sourcePathConverter: SourceFileToPathConverter
+    ) : StorageOwner, LookupStorage(targetDataDir, sourcePathConverter)
+}