azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow customizing source file path conversion in lookup storage

Browse Source
This commit is contained in:
Alexey Tsvetkov
2019-04-12 16:35:13 +03:00
parent 79337a6b96
commit 00de7b6c44
11 changed files with 80 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+6 -3
View File
@@ -30,15 +30,18 @@ import java.io.IOException
import java.util.*
open class LookupStorage(targetDataDir: File) : BasicMapsOwner(targetDataDir) {
open class LookupStorage(
targetDataDir: File,
sourcePathConverter: SourceFileToPathConverter
) : BasicMapsOwner(targetDataDir) {
companion object {
private val DELETED_TO_SIZE_TRESHOLD = 0.5
private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000
}
private val countersFile = "counters".storageFile
private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile))
private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile))
private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, sourcePathConverter))
private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, sourcePathConverter))
private val lookupMap = registerMap(LookupMap("lookups".storageFile))
@Volatile
build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt
+8 -7
View File
@@ -18,26 +18,27 @@ package org.jetbrains.kotlin.incremental.storage
import java.io.File
internal class FileToIdMap(file: File) : BasicMap<File, Int>(file, FileKeyDescriptor, IntExternalizer) {
override fun dumpKey(key: File): String = key.toString()
internal class FileToIdMap(
file: File,
private val sourcePathConverter: SourceFileToPathConverter
) : BasicStringMap<Int>(file, IntExternalizer) {
override fun dumpValue(value: Int): String = value.toString()
operator fun get(file: File): Int? = storage[file]
operator fun get(file: File): Int? = storage[sourcePathConverter.toPath(file)]
operator fun set(file: File, id: Int) {
storage[file] = id
storage[sourcePathConverter.toPath(file)] = id
}
fun remove(file: File) {
storage.remove(file)
storage.remove(sourcePathConverter.toPath(file))
}
fun toMap(): Map<File, Int> {
val result = HashMap<File, Int>()
for (key in storage.keys) {
val value = storage[key] ?: continue
result[key] = value
result[sourcePathConverter.toFile(key)] = value
}
return result
}
build-common/src/org/jetbrains/kotlin/incremental/storage/FilesMap.kt
-26
View File
@@ -1,26 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File
class FilesMap(storageFile: File)
: BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
operator fun set(sourceFile: File, outputFiles: Collection<File>) {
storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath }
}
operator fun get(sourceFile: File): Collection<File> =
storage[sourceFile.absolutePath].orEmpty().map(::File)
override fun dumpValue(value: Collection<String>) =
value.dumpCollection()
fun remove(file: File): Collection<File> =
get(file).also { storage.remove(file.absolutePath) }
}
build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt
+8 -4
View File
@@ -16,20 +16,24 @@
package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.EnumeratorStringDescriptor
import com.intellij.util.io.ExternalIntegerKeyDescriptor
import java.io.File
internal class IdToFileMap(file: File) : BasicMap<Int, File>(file, ExternalIntegerKeyDescriptor(), FileKeyDescriptor) {
internal class IdToFileMap(
file: File,
private val sourcePathConverter: SourceFileToPathConverter
) : BasicMap<Int, String>(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE) {
override fun dumpKey(key: Int): String = key.toString()
override fun dumpValue(value: File): String = value.toString()
override fun dumpValue(value: String): String = value
operator fun get(id: Int): File? = storage[id]
operator fun get(id: Int): File? = storage[id]?.let { sourcePathConverter.toFile(it) }
operator fun contains(id: Int): Boolean = id in storage
operator fun set(id: Int, file: File) {
storage[id] = file
storage[id] = sourcePathConverter.toPath(file)
}
fun remove(id: Int) {
build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt
-14
View File
@@ -177,20 +177,6 @@ object PathStringDescriptor : EnumeratorStringDescriptor() {
override fun isEqual(val1: String, val2: String?) = FileUtil.pathsEqual(val1, val2)
}
object FileKeyDescriptor : KeyDescriptor<File> {
override fun read(input: DataInput): File = File(input.readUTF())
override fun save(output: DataOutput, value: File) {
output.writeUTF(value.canonicalPath)
}
override fun getHashCode(value: File?): Int =
FileUtil.FILE_HASHING_STRATEGY.computeHashCode(value)
override fun isEqual(val1: File?, val2: File?): Boolean =
FileUtil.FILE_HASHING_STRATEGY.equals(val1, val2)
}
open class CollectionExternalizer<T>(
private val elementExternalizer: DataExternalizer<T>,
private val newCollection: () -> MutableCollection<T>
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt
+1 -1
View File
@@ -35,7 +35,7 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
private val lookupCacheDir = File(cachesRootDir, "lookups").apply { mkdirs() }
val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter).apply { registerCache() }
val lookupCache: LookupStorage = LookupStorage(lookupCacheDir).apply { registerCache() }
val lookupCache: LookupStorage = LookupStorage(lookupCacheDir, PATH_CONVERTER).apply { registerCache() }
abstract val platformCache: PlatformCache
fun close(flush: Boolean = false): Boolean {
jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt
+6 -5
View File
@@ -204,7 +204,7 @@ abstract class AbstractIncrementalJpsTest(
return MakeResult(
log = logger.log,
makeFailed = false,
mappingsDump = createMappingsDump(projectDescriptor),
mappingsDump = createMappingsDump(projectDescriptor, kotlinCompileContext),
name = name
)
}
@@ -327,9 +327,10 @@ abstract class AbstractIncrementalJpsTest(
}
private fun createMappingsDump(
project: ProjectDescriptor
project: ProjectDescriptor,
kotlinContext: KotlinCompileContext
) = createKotlinIncrementalCacheDump(project) + "\n\n\n" +
createLookupCacheDump(project) + "\n\n\n" +
createLookupCacheDump(kotlinContext) + "\n\n\n" +
createCommonMappingsDump(project) + "\n\n\n" +
createJavaMappingsDump(project)
@@ -348,13 +349,13 @@ abstract class AbstractIncrementalJpsTest(
}
}
private fun createLookupCacheDump(project: ProjectDescriptor): String {
private fun createLookupCacheDump(kotlinContext: KotlinCompileContext): String {
val sb = StringBuilder()
val p = Printer(sb)
p.println("Begin of Lookup Maps")
p.println()
project.dataManager.withLookupStorage { lookupStorage ->
kotlinContext.lookupStorageManager.withLookupStorage { lookupStorage ->
lookupStorage.forceGC()
p.print(lookupStorage.dump(lookupsDuringTest))
}
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+10 -10
View File
@@ -45,7 +45,7 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.ICReporterBase
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.withLookupStorage
import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
import org.jetbrains.kotlin.jps.model.kotlinKind
import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget
import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget
@@ -249,7 +249,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val changesCollector = ChangesCollector()
removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) }
val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, context.projectDescriptor.dataManager)
val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, kotlinContext.lookupStorageManager)
fsOperations.markFilesForCurrentRound(affectedByRemovedClasses)
}
@@ -471,12 +471,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
)
}
updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder)
updateLookupStorage(lookupTracker, kotlinContext.lookupStorageManager, kotlinDirtyFilesHolder)
if (!isChunkRebuilding) {
changesCollector.processChangesUsingLookups(
kotlinDirtyFilesHolder.allDirtyFiles,
dataManager,
kotlinContext.lookupStorageManager,
fsOperations,
incrementalCaches.values
)
@@ -643,13 +643,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
private fun updateLookupStorage(
lookupTracker: LookupTracker,
dataManager: BuildDataManager,
lookupStorageManager: JpsLookupStorageManager,
dirtyFilesHolder: KotlinDirtySourceFilesHolder
) {
if (lookupTracker !is LookupTrackerImpl)
throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
dataManager.withLookupStorage { lookupStorage ->
lookupStorageManager.withLookupStorage { lookupStorage ->
lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence())
lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values)
}
@@ -673,7 +673,7 @@ private class JpsICReporter : ICReporterBase() {
private fun ChangesCollector.processChangesUsingLookups(
compiledFiles: Set<File>,
dataManager: BuildDataManager,
lookupStorageManager: JpsLookupStorageManager,
fsOperations: FSOperationsHelper,
caches: Iterable<JpsIncrementalCache>
) {
@@ -682,7 +682,7 @@ private fun ChangesCollector.processChangesUsingLookups(
reporter.reportVerbose { "Start processing changes" }
val dirtyFiles = getDirtyFiles(allCaches, dataManager)
val dirtyFiles = getDirtyFiles(allCaches, lookupStorageManager)
fsOperations.markInChunkOrDependents(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
reporter.reportVerbose { "End of processing changes" }
@@ -690,11 +690,11 @@ private fun ChangesCollector.processChangesUsingLookups(
private fun ChangesCollector.getDirtyFiles(
caches: Iterable<IncrementalCacheCommon>,
dataManager: BuildDataManager
lookupStorageManager: JpsLookupStorageManager
): Set<File> {
val reporter = JpsICReporter()
val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(caches, reporter)
val dirtyFilesFromLookups = dataManager.withLookupStorage {
val dirtyFilesFromLookups = lookupStorageManager.withLookupStorage {
mapLookupSymbolsToFiles(it, dirtyLookupSymbols, reporter)
}
return dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter)
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt
+5 -5
View File
@@ -70,6 +70,8 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
val sourceFileToPathConverter: SourceFileToPathConverter = SourceFileToCanonicalPathConverter
val lookupStorageManager = JpsLookupStorageManager(dataManager, sourceFileToPathConverter)
/**
* Flag to prevent rebuilding twice.
*
@@ -107,7 +109,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
// try to perform a lookup
// request rebuild if storage is corrupted
try {
dataManager.withLookupStorage {
lookupStorageManager.withLookupStorage {
it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>"))
}
} catch (e: Exception) {
@@ -189,13 +191,11 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
KotlinBuilder.LOG.info("Rebuilding all Kotlin: $reason")
val dataManager = jpsContext.projectDescriptor.dataManager
targetsIndex.chunks.forEach {
markChunkForRebuildBeforeBuild(it)
}
dataManager.cleanLookupStorage(KotlinBuilder.LOG)
lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG)
}
private fun markChunkForRebuildBeforeBuild(chunk: KotlinChunk) {
@@ -223,7 +223,7 @@ class KotlinCompileContext(val jpsContext: CompileContext) {
private fun clearLookupCache() {
KotlinBuilder.LOG.info("Clearing lookup cache")
dataManager.cleanLookupStorage(KotlinBuilder.LOG)
lookupStorageManager.cleanLookupStorage(KotlinBuilder.LOG)
initialLookupsCacheStateDiff.manager.writeVersion()
}
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinJavaBuilderExtension.kt
+2 -3
View File
@@ -10,7 +10,6 @@ import org.jetbrains.jps.builders.java.JavaBuilderExtension
import org.jetbrains.jps.builders.java.dependencyView.Callbacks
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.kotlin.incremental.LookupSymbol
import org.jetbrains.kotlin.jps.incremental.withLookupStorage
import java.io.File
import java.util.concurrent.Executors
import java.util.concurrent.Future
@@ -24,7 +23,7 @@ class KotlinJavaBuilderExtension : JavaBuilderExtension() {
private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.ConstantAffectionResolver {
private val pool = Executors.newSingleThreadExecutor()
private val dataManager = context.projectDescriptor.dataManager
private val kotlinContext by lazy { context.kotlin }
override fun request(
ownerClassName: String,
@@ -54,7 +53,7 @@ private class KotlinLookupConstantSearch(context: CompileContext) : Callbacks.Co
}
pool.submit {
if (!future.isCancelled) {
dataManager.withLookupStorage { storage ->
kotlinContext.lookupStorageManager.withLookupStorage { storage ->
val paths = storage.get(LookupSymbol(name = fieldName, scope = ownerClassName))
future.result(paths.map { File(it) })
}
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsLookupStorage.kt
+34 -20
View File
@@ -22,36 +22,50 @@ import org.jetbrains.jps.builders.storage.StorageProvider
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.incremental.storage.StorageOwner
import org.jetbrains.kotlin.incremental.LookupStorage
import org.jetbrains.kotlin.incremental.storage.SourceFileToPathConverter
import java.io.File
import java.io.IOException
private object LookupStorageLock
fun BuildDataManager.cleanLookupStorage(log: Logger) {
synchronized(LookupStorageLock) {
try {
cleanTargetStorages(KotlinDataContainerTarget)
} catch (e: IOException) {
if (!dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) {
log.debug("Could not clear lookup storage caches", e)
class JpsLookupStorageManager(
private val buildDataManager: BuildDataManager,
sourcePathConverter: SourceFileToPathConverter
) {
private val storageProvider = JpsLookupStorageProvider(sourcePathConverter)
fun cleanLookupStorage(log: Logger) {
synchronized(LookupStorageLock) {
try {
buildDataManager.cleanTargetStorages(KotlinDataContainerTarget)
} catch (e: IOException) {
if (!buildDataManager.dataPaths.getTargetDataRoot(KotlinDataContainerTarget).deleteRecursively()) {
log.debug("Could not clear lookup storage caches", e)
}
}
}
}
}
fun <T> BuildDataManager.withLookupStorage(fn: (LookupStorage) -> T): T {
synchronized(LookupStorageLock) {
try {
val lookupStorage = getStorage(KotlinDataContainerTarget, JpsLookupStorageProvider)
return fn(lookupStorage)
} catch (e: IOException) {
throw BuildDataCorruptedException(e)
fun <T> withLookupStorage(fn: (LookupStorage) -> T): T {
synchronized(LookupStorageLock) {
try {
val lookupStorage = buildDataManager.getStorage(KotlinDataContainerTarget, storageProvider)
return fn(lookupStorage)
} catch (e: IOException) {
throw BuildDataCorruptedException(e)
}
}
}
}
private object JpsLookupStorageProvider : StorageProvider<JpsLookupStorage>() {
override fun createStorage(targetDataDir: File): JpsLookupStorage = JpsLookupStorage(targetDataDir)
}
private class JpsLookupStorageProvider(
private val sourcePathConverter: SourceFileToPathConverter
) : StorageProvider<JpsLookupStorage>() {
override fun createStorage(targetDataDir: File): JpsLookupStorage =
JpsLookupStorage(targetDataDir, sourcePathConverter)
}
private class JpsLookupStorage(targetDataDir: File) : StorageOwner, LookupStorage(targetDataDir)
private class JpsLookupStorage(
targetDataDir: File,
sourcePathConverter: SourceFileToPathConverter
) : StorageOwner, LookupStorage(targetDataDir, sourcePathConverter)
}