Allow customizing source file path conversion in lookup storage

build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt

@@ -30,15 +30,18 @@ import java.io.IOException
 import java.util.*
 
 
-open class LookupStorage(targetDataDir: File) : BasicMapsOwner(targetDataDir) {
+open class LookupStorage(
+    targetDataDir: File,
+    sourcePathConverter: SourceFileToPathConverter
+) : BasicMapsOwner(targetDataDir) {
     companion object {
         private val DELETED_TO_SIZE_TRESHOLD = 0.5
         private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000
     }
 
     private val countersFile = "counters".storageFile
-    private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile))
+    private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, sourcePathConverter))
-    private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile))
+    private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, sourcePathConverter))
     private val lookupMap = registerMap(LookupMap("lookups".storageFile))
 
     @Volatile

build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt

@@ -18,26 +18,27 @@ package org.jetbrains.kotlin.incremental.storage
 
 import java.io.File
 
-internal class FileToIdMap(file: File) : BasicMap<File, Int>(file, FileKeyDescriptor, IntExternalizer) {
+internal class FileToIdMap(
-    override fun dumpKey(key: File): String = key.toString()
+    file: File,
-
+    private val sourcePathConverter: SourceFileToPathConverter
+) : BasicStringMap<Int>(file, IntExternalizer) {
     override fun dumpValue(value: Int): String = value.toString()
 
-    operator fun get(file: File): Int? = storage[file]
+    operator fun get(file: File): Int? = storage[sourcePathConverter.toPath(file)]
 
     operator fun set(file: File, id: Int) {
-        storage[file] = id
+        storage[sourcePathConverter.toPath(file)] = id
     }
 
     fun remove(file: File) {
-        storage.remove(file)
+        storage.remove(sourcePathConverter.toPath(file))
     }
 
     fun toMap(): Map<File, Int> {
         val result = HashMap<File, Int>()
         for (key in storage.keys) {
             val value = storage[key] ?: continue
-            result[key] = value
+            result[sourcePathConverter.toFile(key)] = value
         }
         return result
     }

build-common/src/org/jetbrains/kotlin/incremental/storage/FilesMap.kt

@@ -1,26 +0,0 @@
-/*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.incremental.storage
-
-import org.jetbrains.kotlin.incremental.dumpCollection
-import java.io.File
-
-class FilesMap(storageFile: File)
-    : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
-
-    operator fun set(sourceFile: File, outputFiles: Collection<File>) {
-        storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath }
-    }
-
-    operator fun get(sourceFile: File): Collection<File> =
-        storage[sourceFile.absolutePath].orEmpty().map(::File)
-
-    override fun dumpValue(value: Collection<String>) =
-        value.dumpCollection()
-
-    fun remove(file: File): Collection<File> =
-        get(file).also { storage.remove(file.absolutePath) }
-}

build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt

@@ -16,20 +16,24 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
+import com.intellij.util.io.EnumeratorStringDescriptor
 import com.intellij.util.io.ExternalIntegerKeyDescriptor
 import java.io.File
 
-internal class IdToFileMap(file: File) : BasicMap<Int, File>(file, ExternalIntegerKeyDescriptor(), FileKeyDescriptor) {
+internal class IdToFileMap(
+    file: File,
+    private val sourcePathConverter: SourceFileToPathConverter
+) : BasicMap<Int, String>(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE) {
     override fun dumpKey(key: Int): String = key.toString()
 
-    override fun dumpValue(value: File): String = value.toString()
+    override fun dumpValue(value: String): String = value
 
-    operator fun get(id: Int): File? = storage[id]
+    operator fun get(id: Int): File? = storage[id]?.let { sourcePathConverter.toFile(it) }
 
     operator fun contains(id: Int): Boolean = id in storage
 
     operator fun set(id: Int, file: File) {
-        storage[id] = file
+        storage[id] = sourcePathConverter.toPath(file)
     }
 
     fun remove(id: Int) {

build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt

@@ -177,20 +177,6 @@ object PathStringDescriptor : EnumeratorStringDescriptor() {
     override fun isEqual(val1: String, val2: String?) = FileUtil.pathsEqual(val1, val2)
 }
 
-object FileKeyDescriptor : KeyDescriptor<File> {
-    override fun read(input: DataInput): File = File(input.readUTF())
-
-    override fun save(output: DataOutput, value: File) {
-        output.writeUTF(value.canonicalPath)
-    }
-
-    override fun getHashCode(value: File?): Int =
-            FileUtil.FILE_HASHING_STRATEGY.computeHashCode(value)
-
-    override fun isEqual(val1: File?, val2: File?): Boolean =
-            FileUtil.FILE_HASHING_STRATEGY.equals(val1, val2)
-}
-
 open class CollectionExternalizer<T>(
         private val elementExternalizer: DataExternalizer<T>,
         private val newCollection: () -> MutableCollection<T>

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -35,7 +35,7 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
     private val lookupCacheDir = File(cachesRootDir, "lookups").apply { mkdirs() }
 
     val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter).apply { registerCache() }
-    val lookupCache: LookupStorage = LookupStorage(lookupCacheDir).apply { registerCache() }
+    val lookupCache: LookupStorage = LookupStorage(lookupCacheDir, PATH_CONVERTER).apply { registerCache() }
     abstract val platformCache: PlatformCache
 
     fun close(flush: Boolean = false): Boolean {

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt

@@ -204,7 +204,7 @@ abstract class AbstractIncrementalJpsTest(
                 return MakeResult(
                     log = logger.log,
                     makeFailed = false,
-                    mappingsDump = createMappingsDump(projectDescriptor),
+                    mappingsDump = createMappingsDump(projectDescriptor, kotlinCompileContext),
                     name = name
                 )
             }
@@ -327,9 +327,10 @@ abstract class AbstractIncrementalJpsTest(
     }
 
     private fun createMappingsDump(
-        project: ProjectDescriptor
+        project: ProjectDescriptor,
+        kotlinContext: KotlinCompileContext
     ) = createKotlinIncrementalCacheDump(project) + "\n\n\n" +
-            createLookupCacheDump(project) + "\n\n\n" +
+            createLookupCacheDump(kotlinContext) + "\n\n\n" +
             createCommonMappingsDump(project) + "\n\n\n" +
             createJavaMappingsDump(project)
 
@@ -348,13 +349,13 @@ abstract class AbstractIncrementalJpsTest(
         }
     }
 
-    private fun createLookupCacheDump(project: ProjectDescriptor): String {
+    private fun createLookupCacheDump(kotlinContext: KotlinCompileContext): String {
         val sb = StringBuilder()
         val p = Printer(sb)
         p.println("Begin of Lookup Maps")
         p.println()
 
-        project.dataManager.withLookupStorage { lookupStorage ->
+        kotlinContext.lookupStorageManager.withLookupStorage { lookupStorage ->
             lookupStorage.forceGC()
             p.print(lookupStorage.dump(lookupsDuringTest))
         }

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -45,7 +45,7 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.ICReporterBase
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
-import org.jetbrains.kotlin.jps.incremental.withLookupStorage
+import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
 import org.jetbrains.kotlin.jps.model.kotlinKind
 import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget
 import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget
@@ -249,7 +249,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
 
         val changesCollector = ChangesCollector()
         removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) }
-        val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, context.projectDescriptor.dataManager)
+        val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, kotlinContext.lookupStorageManager)
 
         fsOperations.markFilesForCurrentRound(affectedByRemovedClasses)
     }
@@ -471,12 +471,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
                 )
             }
 
-            updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder)
+            updateLookupStorage(lookupTracker, kotlinContext.lookupStorageManager, kotlinDirtyFilesHolder)
 
             if (!isChunkRebuilding) {
                 changesCollector.processChangesUsingLookups(
                     kotlinDirtyFilesHolder.allDirtyFiles,
-                    dataManager,
+                    kotlinContext.lookupStorageManager,
                     fsOperations,
                     incrementalCaches.values
                 )
@@ -643,13 +643,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
 
     private fun updateLookupStorage(
         lookupTracker: LookupTracker,
-        dataManager: BuildDataManager,
+        lookupStorageManager: JpsLookupStorageManager,
         dirtyFilesHolder: KotlinDirtySourceFilesHolder
     ) {
         if (lookupTracker !is LookupTrackerImpl)
             throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
 
-        dataManager.withLookupStorage { lookupStorage ->
+        lookupStorageManager.withLookupStorage { lookupStorage ->
             lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence())
             lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values)
         }
@@ -673,7 +673,7 @@ private class JpsICReporter : ICReporterBase() {
 
 private fun ChangesCollector.processChangesUsingLookups(
     compiledFiles: Set<File>,
-    dataManager: BuildDataManager,
+    lookupStorageManager: JpsLookupStorageManager,
     fsOperations: FSOperationsHelper,
     caches: Iterable<JpsIncrementalCache>
 ) {
@@ -682,7 +682,7 @@ private fun ChangesCollector.processChangesUsingLookups(
 
     reporter.reportVerbose { "Start processing changes" }
 
-    val dirtyFiles = getDirtyFiles(allCaches, dataManager)
+    val dirtyFiles = getDirtyFiles(allCaches, lookupStorageManager)
     fsOperations.markInChunkOrDependents(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
 
     reporter.reportVerbose { "End of processing changes" }
@@ -690,11 +690,11 @@ private fun ChangesCollector.processChangesUsingLookups(
 
 private fun ChangesCollector.getDirtyFiles(
     caches: Iterable<IncrementalCacheCommon>,
-    dataManager: BuildDataManager
+    lookupStorageManager: JpsLookupStorageManager
 ): Set<File> {
     val reporter = JpsICReporter()
     val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(caches, reporter)
-    val dirtyFilesFromLookups = dataManager.withLookupStorage {
+    val dirtyFilesFromLookups = lookupStorageManager.withLookupStorage {
         mapLookupSymbolsToFiles(it, dirtyLookupSymbols, reporter)
     }
     return dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter)

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt

@@ -70,6 +70,8 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
 
     val sourceFileToPathConverter: SourceFileToPathConverter = SourceFileToCanonicalPathConverter
 
+    val lookupStorageManager = JpsLookupStorageManager(dataManager, sourceFileToPathConverter)
+
     /**
      * Flag to prevent rebuilding twice.
      *
@@ -107,7 +109,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
             // try to perform a lookup
             // request rebuild if storage is corrupted
             try {
-                dataManager.withLookupStorage {
+                lookupStorageManager.withLookupStorage {
                     it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>"))
                 }
             } catch (e: Exception) {
@@ -189,13 +191,11 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
 
         KotlinBuilder.LOG.info("Rebuilding all Kotlin: $reason")
 
-        val dataManager = jpsContext.projectDescriptor.dataManager
-
         targetsIndex.chunks.forEach {
             markChunkForRebuildBeforeBuild(it)
         }
 
-        dataManager.cleanLookupStorage(KotlinBuilder.LOG)
+        lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG)
     }
 
     private fun markChunkForRebuildBeforeBuild(chunk: KotlinChunk) {
@@ -223,7 +223,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
 
     private fun clearLookupCache() {
         KotlinBuilder.LOG.info("Clearing lookup cache")
-        dataManager.cleanLookupStorage(KotlinBuilder.LOG)
+        lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG)
         initialLookupsCacheStateDiff.manager.writeVersion()
     }
 

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt

@@ -10,7 +10,6 @@ import org.jetbrains.jps.builders.java.JavaBuilderExtension
 import org.jetbrains.jps.builders.java.dependencyView.Callbacks
 import org.jetbrains.jps.incremental.CompileContext
 import org.jetbrains.kotlin.incremental.LookupSymbol
-import org.jetbrains.kotlin.jps.incremental.withLookupStorage
 import java.io.File
 import java.util.concurrent.Executors
 import java.util.concurrent.Future
@@ -24,7 +23,7 @@ class KotlinJavaBuilderExtension : JavaBuilderExtension() {
 
 private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.ConstantAffectionResolver {
     private val pool = Executors.newSingleThreadExecutor()
-    private val dataManager = context.projectDescriptor.dataManager
+    private val kotlinContext by lazy { context.kotlin }
 
     override fun request(
         ownerClassName: String,
@@ -54,7 +53,7 @@ private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.Co
         }
         pool.submit {
             if (!future.isCancelled) {
-                dataManager.withLookupStorage { storage ->
+                kotlinContext.lookupStorageManager.withLookupStorage { storage ->
                     val paths = storage.get(LookupSymbol(name = fieldName, scope = ownerClassName))
                     future.result(paths.map { File(it) })
                 }

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt

@@ -22,36 +22,50 @@ import org.jetbrains.jps.builders.storage.StorageProvider
 import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.incremental.storage.StorageOwner
 import org.jetbrains.kotlin.incremental.LookupStorage
+import org.jetbrains.kotlin.incremental.storage.SourceFileToPathConverter
 import java.io.File
 import java.io.IOException
 
 private object LookupStorageLock
 
-fun BuildDataManager.cleanLookupStorage(log: Logger) {
+class JpsLookupStorageManager(
-    synchronized(LookupStorageLock) {
+    private val buildDataManager: BuildDataManager,
-        try {
+    sourcePathConverter: SourceFileToPathConverter
-            cleanTargetStorages(KotlinDataContainerTarget)
+) {
-        } catch (e: IOException) {
+    private val storageProvider = JpsLookupStorageProvider(sourcePathConverter)
-            if (!dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) {
+
-                log.debug("Could not clear lookup storage caches", e)
+    fun cleanLookupStorage(log: Logger) {
+        synchronized(LookupStorageLock) {
+            try {
+                buildDataManager.cleanTargetStorages(KotlinDataContainerTarget)
+            } catch (e: IOException) {
+                if (!buildDataManager.dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) {
+                    log.debug("Could not clear lookup storage caches", e)
+                }
             }
         }
     }
-}
 
-fun <T> BuildDataManager.withLookupStorage(fn: (LookupStorage) -> T): T {
+    fun <T> withLookupStorage(fn: (LookupStorage) -> T): T {
-    synchronized(LookupStorageLock) {
+        synchronized(LookupStorageLock) {
-        try {
+            try {
-            val lookupStorage = getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider)
+                val lookupStorage = buildDataManager.getStorage(KotlinDataContainerTarget, storageProvider)
-            return fn(lookupStorage)
+                return fn(lookupStorage)
-        } catch (e: IOException) {
+            } catch (e: IOException) {
-            throw BuildDataCorruptedException(e)
+                throw BuildDataCorruptedException(e)
+            }
         }
     }
-}
 
-private object JpsLookupStorageProvider : StorageProvider<JpsLookupStorage>() {
+    private class JpsLookupStorageProvider(
-    override fun createStorage(targetDataDir: File): JpsLookupStorage = JpsLookupStorage(targetDataDir)
+        private val sourcePathConverter: SourceFileToPathConverter
-}
+    ) : StorageProvider<JpsLookupStorage>() {
+        override fun createStorage(targetDataDir: File): JpsLookupStorage =
+            JpsLookupStorage(targetDataDir, sourcePathConverter)
+    }
 
-private class JpsLookupStorage(targetDataDir: File) : StorageOwner, LookupStorage(targetDataDir)
+    private class JpsLookupStorage(
+        targetDataDir: File,
+        sourcePathConverter: SourceFileToPathConverter
+    ) : StorageOwner, LookupStorage(targetDataDir, sourcePathConverter)
+}