diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index d4279ddab91..ab92ac4af83 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -30,15 +30,18 @@ import java.io.IOException import java.util.* -open class LookupStorage(targetDataDir: File) : BasicMapsOwner(targetDataDir) { +open class LookupStorage( + targetDataDir: File, + sourcePathConverter: SourceFileToPathConverter +) : BasicMapsOwner(targetDataDir) { companion object { private val DELETED_TO_SIZE_TRESHOLD = 0.5 private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000 } private val countersFile = "counters".storageFile - private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile)) - private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile)) + private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, sourcePathConverter)) + private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, sourcePathConverter)) private val lookupMap = registerMap(LookupMap("lookups".storageFile)) @Volatile diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt index 274ff76290a..3b62f957390 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt @@ -18,26 +18,27 @@ package org.jetbrains.kotlin.incremental.storage import java.io.File -internal class FileToIdMap(file: File) : BasicMap(file, FileKeyDescriptor, IntExternalizer) { - override fun dumpKey(key: File): String = key.toString() - +internal class FileToIdMap( + file: File, + private val sourcePathConverter: SourceFileToPathConverter +) : BasicStringMap(file, IntExternalizer) { override fun dumpValue(value: Int): String = value.toString() - operator fun get(file: File): Int? = storage[file] + operator fun get(file: File): Int? = storage[sourcePathConverter.toPath(file)] operator fun set(file: File, id: Int) { - storage[file] = id + storage[sourcePathConverter.toPath(file)] = id } fun remove(file: File) { - storage.remove(file) + storage.remove(sourcePathConverter.toPath(file)) } fun toMap(): Map { val result = HashMap() for (key in storage.keys) { val value = storage[key] ?: continue - result[key] = value + result[sourcePathConverter.toFile(key)] = value } return result } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/FilesMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/FilesMap.kt deleted file mode 100644 index a1a650aaaa0..00000000000 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/FilesMap.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.incremental.storage - -import org.jetbrains.kotlin.incremental.dumpCollection -import java.io.File - -class FilesMap(storageFile: File) - : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer) { - - operator fun set(sourceFile: File, outputFiles: Collection) { - storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath } - } - - operator fun get(sourceFile: File): Collection = - storage[sourceFile.absolutePath].orEmpty().map(::File) - - override fun dumpValue(value: Collection) = - value.dumpCollection() - - fun remove(file: File): Collection = - get(file).also { storage.remove(file.absolutePath) } -} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt index 98f84e01d43..cc5e6ce5fae 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt @@ -16,20 +16,24 @@ package org.jetbrains.kotlin.incremental.storage +import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.ExternalIntegerKeyDescriptor import java.io.File -internal class IdToFileMap(file: File) : BasicMap(file, ExternalIntegerKeyDescriptor(), FileKeyDescriptor) { +internal class IdToFileMap( + file: File, + private val sourcePathConverter: SourceFileToPathConverter +) : BasicMap(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE) { override fun dumpKey(key: Int): String = key.toString() - override fun dumpValue(value: File): String = value.toString() + override fun dumpValue(value: String): String = value - operator fun get(id: Int): File? = storage[id] + operator fun get(id: Int): File? = storage[id]?.let { sourcePathConverter.toFile(it) } operator fun contains(id: Int): Boolean = id in storage operator fun set(id: Int, file: File) { - storage[id] = file + storage[id] = sourcePathConverter.toPath(file) } fun remove(id: Int) { diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt index 4cbe2ee2be9..795de067365 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt @@ -177,20 +177,6 @@ object PathStringDescriptor : EnumeratorStringDescriptor() { override fun isEqual(val1: String, val2: String?) = FileUtil.pathsEqual(val1, val2) } -object FileKeyDescriptor : KeyDescriptor { - override fun read(input: DataInput): File = File(input.readUTF()) - - override fun save(output: DataOutput, value: File) { - output.writeUTF(value.canonicalPath) - } - - override fun getHashCode(value: File?): Int = - FileUtil.FILE_HASHING_STRATEGY.computeHashCode(value) - - override fun isEqual(val1: File?, val2: File?): Boolean = - FileUtil.FILE_HASHING_STRATEGY.equals(val1, val2) -} - open class CollectionExternalizer( private val elementExternalizer: DataExternalizer, private val newCollection: () -> MutableCollection diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 239c78f9ad6..5fbcaa18b33 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -35,7 +35,7 @@ abstract class IncrementalCachesManager + kotlinContext.lookupStorageManager.withLookupStorage { lookupStorage -> lookupStorage.forceGC() p.print(lookupStorage.dump(lookupsDuringTest)) } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index ae3bb106cd9..db54b5978ad 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -45,7 +45,7 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.ICReporterBase import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache -import org.jetbrains.kotlin.jps.incremental.withLookupStorage +import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager import org.jetbrains.kotlin.jps.model.kotlinKind import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget @@ -249,7 +249,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val changesCollector = ChangesCollector() removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) } - val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, context.projectDescriptor.dataManager) + val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, kotlinContext.lookupStorageManager) fsOperations.markFilesForCurrentRound(affectedByRemovedClasses) } @@ -471,12 +471,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { ) } - updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder) + updateLookupStorage(lookupTracker, kotlinContext.lookupStorageManager, kotlinDirtyFilesHolder) if (!isChunkRebuilding) { changesCollector.processChangesUsingLookups( kotlinDirtyFilesHolder.allDirtyFiles, - dataManager, + kotlinContext.lookupStorageManager, fsOperations, incrementalCaches.values ) @@ -643,13 +643,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { private fun updateLookupStorage( lookupTracker: LookupTracker, - dataManager: BuildDataManager, + lookupStorageManager: JpsLookupStorageManager, dirtyFilesHolder: KotlinDirtySourceFilesHolder ) { if (lookupTracker !is LookupTrackerImpl) throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}") - dataManager.withLookupStorage { lookupStorage -> + lookupStorageManager.withLookupStorage { lookupStorage -> lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence()) lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values) } @@ -673,7 +673,7 @@ private class JpsICReporter : ICReporterBase() { private fun ChangesCollector.processChangesUsingLookups( compiledFiles: Set, - dataManager: BuildDataManager, + lookupStorageManager: JpsLookupStorageManager, fsOperations: FSOperationsHelper, caches: Iterable ) { @@ -682,7 +682,7 @@ private fun ChangesCollector.processChangesUsingLookups( reporter.reportVerbose { "Start processing changes" } - val dirtyFiles = getDirtyFiles(allCaches, dataManager) + val dirtyFiles = getDirtyFiles(allCaches, lookupStorageManager) fsOperations.markInChunkOrDependents(dirtyFiles.asIterable(), excludeFiles = compiledFiles) reporter.reportVerbose { "End of processing changes" } @@ -690,11 +690,11 @@ private fun ChangesCollector.processChangesUsingLookups( private fun ChangesCollector.getDirtyFiles( caches: Iterable, - dataManager: BuildDataManager + lookupStorageManager: JpsLookupStorageManager ): Set { val reporter = JpsICReporter() val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(caches, reporter) - val dirtyFilesFromLookups = dataManager.withLookupStorage { + val dirtyFilesFromLookups = lookupStorageManager.withLookupStorage { mapLookupSymbolsToFiles(it, dirtyLookupSymbols, reporter) } return dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter) diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt index ec3ff7385b1..604ac019265 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt @@ -70,6 +70,8 @@ class KotlinCompileContext(val jpsContext: CompileContext) { val sourceFileToPathConverter: SourceFileToPathConverter = SourceFileToCanonicalPathConverter + val lookupStorageManager = JpsLookupStorageManager(dataManager, sourceFileToPathConverter) + /** * Flag to prevent rebuilding twice. * @@ -107,7 +109,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) { // try to perform a lookup // request rebuild if storage is corrupted try { - dataManager.withLookupStorage { + lookupStorageManager.withLookupStorage { it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>")) } } catch (e: Exception) { @@ -189,13 +191,11 @@ class KotlinCompileContext(val jpsContext: CompileContext) { KotlinBuilder.LOG.info("Rebuilding all Kotlin: $reason") - val dataManager = jpsContext.projectDescriptor.dataManager - targetsIndex.chunks.forEach { markChunkForRebuildBeforeBuild(it) } - dataManager.cleanLookupStorage(KotlinBuilder.LOG) + lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG) } private fun markChunkForRebuildBeforeBuild(chunk: KotlinChunk) { @@ -223,7 +223,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) { private fun clearLookupCache() { KotlinBuilder.LOG.info("Clearing lookup cache") - dataManager.cleanLookupStorage(KotlinBuilder.LOG) + lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG) initialLookupsCacheStateDiff.manager.writeVersion() } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt index 63c9e0ba471..b3636498cec 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt @@ -10,7 +10,6 @@ import org.jetbrains.jps.builders.java.JavaBuilderExtension import org.jetbrains.jps.builders.java.dependencyView.Callbacks import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.kotlin.incremental.LookupSymbol -import org.jetbrains.kotlin.jps.incremental.withLookupStorage import java.io.File import java.util.concurrent.Executors import java.util.concurrent.Future @@ -24,7 +23,7 @@ class KotlinJavaBuilderExtension : JavaBuilderExtension() { private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.ConstantAffectionResolver { private val pool = Executors.newSingleThreadExecutor() - private val dataManager = context.projectDescriptor.dataManager + private val kotlinContext by lazy { context.kotlin } override fun request( ownerClassName: String, @@ -54,7 +53,7 @@ private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.Co } pool.submit { if (!future.isCancelled) { - dataManager.withLookupStorage { storage -> + kotlinContext.lookupStorageManager.withLookupStorage { storage -> val paths = storage.get(LookupSymbol(name = fieldName, scope = ownerClassName)) future.result(paths.map { File(it) }) } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt index c68c128b36f..45330f65cf0 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt @@ -22,36 +22,50 @@ import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.StorageOwner import org.jetbrains.kotlin.incremental.LookupStorage +import org.jetbrains.kotlin.incremental.storage.SourceFileToPathConverter import java.io.File import java.io.IOException private object LookupStorageLock -fun BuildDataManager.cleanLookupStorage(log: Logger) { - synchronized(LookupStorageLock) { - try { - cleanTargetStorages(KotlinDataContainerTarget) - } catch (e: IOException) { - if (!dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) { - log.debug("Could not clear lookup storage caches", e) +class JpsLookupStorageManager( + private val buildDataManager: BuildDataManager, + sourcePathConverter: SourceFileToPathConverter +) { + private val storageProvider = JpsLookupStorageProvider(sourcePathConverter) + + fun cleanLookupStorage(log: Logger) { + synchronized(LookupStorageLock) { + try { + buildDataManager.cleanTargetStorages(KotlinDataContainerTarget) + } catch (e: IOException) { + if (!buildDataManager.dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) { + log.debug("Could not clear lookup storage caches", e) + } } } } -} -fun BuildDataManager.withLookupStorage(fn: (LookupStorage) -> T): T { - synchronized(LookupStorageLock) { - try { - val lookupStorage = getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider) - return fn(lookupStorage) - } catch (e: IOException) { - throw BuildDataCorruptedException(e) + fun withLookupStorage(fn: (LookupStorage) -> T): T { + synchronized(LookupStorageLock) { + try { + val lookupStorage = buildDataManager.getStorage(KotlinDataContainerTarget, storageProvider) + return fn(lookupStorage) + } catch (e: IOException) { + throw BuildDataCorruptedException(e) + } } } -} -private object JpsLookupStorageProvider : StorageProvider() { - override fun createStorage(targetDataDir: File): JpsLookupStorage = JpsLookupStorage(targetDataDir) -} + private class JpsLookupStorageProvider( + private val sourcePathConverter: SourceFileToPathConverter + ) : StorageProvider() { + override fun createStorage(targetDataDir: File): JpsLookupStorage = + JpsLookupStorage(targetDataDir, sourcePathConverter) + } -private class JpsLookupStorage(targetDataDir: File) : StorageOwner, LookupStorage(targetDataDir) + private class JpsLookupStorage( + targetDataDir: File, + sourcePathConverter: SourceFileToPathConverter + ) : StorageOwner, LookupStorage(targetDataDir, sourcePathConverter) +}