diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index 4967914b28f..45fa8477dc7 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -503,7 +503,7 @@ public class IncrementalCacheImpl( private fun remove(path: String) { storage.remove(path) - lookupTrackerImpl?.removeLookupsFrom(path) + lookupTrackerImpl?.removeLookupsFrom(File(path)) } } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt index 5a80420b2a5..9b973baef95 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt @@ -20,7 +20,8 @@ import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.ScopeKind import org.jetbrains.kotlin.jps.incremental.storage.BasicMapsOwner -import org.jetbrains.kotlin.jps.incremental.storage.FilesMap +import org.jetbrains.kotlin.jps.incremental.storage.FileToIdMap +import org.jetbrains.kotlin.jps.incremental.storage.IdToFileMap import org.jetbrains.kotlin.jps.incremental.storage.LookupMap import java.io.File @@ -29,30 +30,83 @@ object LOOKUP_TRACKER_STORAGE_PROVIDER : StorageProvider() { } class LookupTrackerImpl(private val targetDataDir: File) : BasicMapsOwner(), LookupTracker { + + companion object { + private val DELETED_TO_SIZE_TRESHOLD = 0.5 + private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000 + } + private val String.storageFile: File get() = File(targetDataDir, this + IncrementalCacheImpl.CACHE_EXTENSION) - private val filesMap = registerMap(FilesMap("files".storageFile)) - private val lookupMap = registerMap(LookupMap("lookups".storageFile, filesMap)) + private val countersFile = "counters".storageFile + private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile)) + private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile)) + private val lookupMap = registerMap(LookupMap("lookups".storageFile)) + private var size: Int = 0 + private var deletedCount: Int = 0 + + init { + if (countersFile.exists()) { + val lines = countersFile.readLines() + size = lines[0].toInt() + deletedCount = lines[1].toInt() + } + } override fun record(lookupContainingFile: String, lookupLine: Int?, lookupColumn: Int?, scopeFqName: String, scopeKind: ScopeKind, name: String) { - lookupMap.add(name, scopeFqName, lookupContainingFile) + val file = File(lookupContainingFile) + val fileId = fileToId[file] ?: addFile(file) + lookupMap.add(name, scopeFqName, fileId) + } + + fun removeLookupsFrom(file: File) { + val id = fileToId[file] ?: return + idToFile.remove(id) + fileToId.remove(file) + deletedCount++ } override fun clean() { - lookupMap.clean() - } + if (countersFile.exists()) { + countersFile.delete() + } - override fun close() { - lookupMap.close() + size = 0 + deletedCount = 0 + + super.clean() } override fun flush(memoryCachesOnly: Boolean) { - lookupMap.flush(memoryCachesOnly) + try { + removeGarbageIfNeeded() + countersFile.writeText("$size\n$deletedCount") + } + finally { + super.flush(memoryCachesOnly) + } } - fun removeLookupsFrom(path: String) { - filesMap.remove(path) + private fun addFile(file: File): Int { + val id = size++ + fileToId[file] = id + idToFile[id] = file + return id + } + + private fun removeGarbageIfNeeded() { + if (size <= MINIMUM_GARBAGE_COLLECTIBLE_SIZE && deletedCount.toDouble() / size <= DELETED_TO_SIZE_TRESHOLD) return + + for (hash in lookupMap.lookupHashes) { + lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet() + } + + size = 0 + deletedCount = 0 + idToFile.clean() + + fileToId.files.forEach { addFile(it) } } } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FileToIdMap.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FileToIdMap.kt new file mode 100644 index 00000000000..6517167089e --- /dev/null +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FileToIdMap.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.incremental.storage + +import java.io.File + +class FileToIdMap(file: File) : BasicMap(file, FILE_KEY_DESCRIPTOR, INT_EXTERNALIZER) { + override fun dumpKey(key: File): String = key.toString() + + override fun dumpValue(value: Int): String = value.toString() + + public operator fun get(file: File): Int? = storage[file] + + public operator fun set(file: File, id: Int) { + storage[file] = id + } + + public fun remove(file: File) { + storage.remove(file) + } + + public val files: Collection + get() = storage.keys +} diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FilesMap.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FilesMap.kt deleted file mode 100644 index 61329338eba..00000000000 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FilesMap.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.jps.incremental.storage - -import com.intellij.openapi.util.io.FileUtil -import com.intellij.util.io.ExternalIntegerKeyDescriptor -import java.io.File - -class FilesMap(file: File) : BasicMap>(file, ExternalIntegerKeyDescriptor(), PATH_COLLECTION_EXTERNALIZER) { - override fun dumpKey(key: Int): String = key.toString() - - override fun dumpValue(value: Collection): String = value.toString() - - public fun get(hash: Int): Collection? = storage[hash] - - public fun add(path: String): Int { - val hash = FileUtil.PATH_HASHING_STRATEGY.computeHashCode(path) - storage.append(hash) { it.writeUTF(path) } - return hash - } - - public fun remove(path: String) { - val hash = FileUtil.PATH_HASHING_STRATEGY.computeHashCode(path) - val collection = storage[hash] as? MutableCollection ?: return - collection.remove(path) - - if (collection.isNotEmpty()) { - storage.remove(hash) - } - } -} - - - diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/IdToFileMap.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/IdToFileMap.kt new file mode 100644 index 00000000000..48e93441363 --- /dev/null +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/IdToFileMap.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.incremental.storage + +import com.intellij.util.io.ExternalIntegerKeyDescriptor +import java.io.File + +class IdToFileMap(file: File) : BasicMap(file, ExternalIntegerKeyDescriptor(), FILE_EXTERNALIZER) { + override fun dumpKey(key: Int): String = key.toString() + + override fun dumpValue(value: File): String = value.toString() + + public operator fun get(id: Int): File? = storage[id] + + public operator fun contains(id: Int): Boolean = id in storage + + public operator fun set(id: Int, file: File) { + storage[id] = file + } + + public fun remove(id: Int) { + storage.remove(id) + } +} diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LookupMap.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LookupMap.kt index 374451c2609..8c8e1006150 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LookupMap.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LookupMap.kt @@ -18,21 +18,23 @@ package org.jetbrains.kotlin.jps.incremental.storage import java.io.File -class LookupMap( - storage: File, - private val filesMap: FilesMap -) : BasicMap>(storage, INT_PAIR_KEY_DESCRIPTOR, INT_SET_EXTERNALIZER) { +class LookupMap(storage: File) : BasicMap>(storage, INT_PAIR_KEY_DESCRIPTOR, INT_SET_EXTERNALIZER) { override fun dumpKey(key: IntPair): String = key.toString() override fun dumpValue(value: Set): String = value.toString() - public fun add(name: String, scope: String, path: String) { - val pathHash = filesMap.add(path) - storage.append(HashPair(name, scope)) { out -> out.writeInt(pathHash) } + public fun add(name: String, scope: String, fileId: Int) { + storage.append(HashPair(name, scope)) { out -> out.writeInt(fileId) } } - public fun get(name: String, scope: String): Set? = storage[HashPair(name, scope)] + public operator fun get(name: String, scope: String): Set? = storage[HashPair(name, scope)] + + public operator fun get(lookupHash: IntPair): Set? = storage[lookupHash] + + public operator fun set(key: IntPair, fileIds: Set) { + storage.set(key, fileIds) + } + + public val lookupHashes: Collection + get() = storage.keys } - - - diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/externalizers.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/externalizers.kt index 68618da5a45..00ed99663b4 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/externalizers.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/externalizers.kt @@ -26,6 +26,7 @@ import gnu.trove.decorator.TIntHashSetDecorator import java.io.DataInput import java.io.DataInputStream import java.io.DataOutput +import java.io.File import java.util.* object INT_PAIR_KEY_DESCRIPTOR : KeyDescriptor { @@ -236,4 +237,34 @@ object INT_SET_EXTERNALIZER : DataExternalizer> { return TIntHashSetDecorator(result) } +} + +object INT_EXTERNALIZER : DataExternalizer { + override fun read(`in`: DataInput): Int = `in`.readInt() + + override fun save(out: DataOutput, value: Int) { + out.writeInt(value) + } +} + +object FILE_EXTERNALIZER : DataExternalizer { + override fun read(`in`: DataInput): File = File(`in`.readUTF()) + + override fun save(out: DataOutput, value: File) { + out.writeUTF(value.canonicalPath) + } +} + +object FILE_KEY_DESCRIPTOR : KeyDescriptor { + override fun read(`in`: DataInput): File = File(`in`.readUTF()) + + override fun save(out: DataOutput, value: File) { + out.writeUTF(value.canonicalPath) + } + + override fun getHashCode(value: File?): Int = + FileUtil.FILE_HASHING_STRATEGY.computeHashCode(value) + + override fun isEqual(val1: File?, val2: File?): Boolean = + FileUtil.FILE_HASHING_STRATEGY.equals(val1, val2) } \ No newline at end of file