azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Hash files without collisions

Browse Source 
Original commit: 7ebc58c690
This commit is contained in:
Alexey Tsvetkov
2015-10-30 00:11:07 +03:00
parent a72be353e2
commit 4e99ad81a5
7 changed files with 186 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt
+1 -1
View File
@@ -503,7 +503,7 @@ public class IncrementalCacheImpl(
private fun remove(path: String) {
storage.remove(path)
lookupTrackerImpl?.removeLookupsFrom(path)
lookupTrackerImpl?.removeLookupsFrom(File(path))
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt
+65 -11
View File
@@ -20,7 +20,8 @@ import org.jetbrains.jps.builders.storage.StorageProvider
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.components.ScopeKind
import org.jetbrains.kotlin.jps.incremental.storage.BasicMapsOwner
import org.jetbrains.kotlin.jps.incremental.storage.FilesMap
import org.jetbrains.kotlin.jps.incremental.storage.FileToIdMap
import org.jetbrains.kotlin.jps.incremental.storage.IdToFileMap
import org.jetbrains.kotlin.jps.incremental.storage.LookupMap
import java.io.File
@@ -29,30 +30,83 @@ object LOOKUP_TRACKER_STORAGE_PROVIDER : StorageProvider<LookupTrackerImpl>() {
}
class LookupTrackerImpl(private val targetDataDir: File) : BasicMapsOwner(), LookupTracker {
companion object {
private val DELETED_TO_SIZE_TRESHOLD = 0.5
private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000
}
private val String.storageFile: File
get() = File(targetDataDir, this + IncrementalCacheImpl.CACHE_EXTENSION)
private val filesMap = registerMap(FilesMap("files".storageFile))
private val lookupMap = registerMap(LookupMap("lookups".storageFile, filesMap))
private val countersFile = "counters".storageFile
private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile))
private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile))
private val lookupMap = registerMap(LookupMap("lookups".storageFile))
private var size: Int = 0
private var deletedCount: Int = 0
init {
if (countersFile.exists()) {
val lines = countersFile.readLines()
size = lines[0].toInt()
deletedCount = lines[1].toInt()
}
}
override fun record(lookupContainingFile: String, lookupLine: Int?, lookupColumn: Int?, scopeFqName: String, scopeKind: ScopeKind, name: String) {
lookupMap.add(name, scopeFqName, lookupContainingFile)
val file = File(lookupContainingFile)
val fileId = fileToId[file] ?: addFile(file)
lookupMap.add(name, scopeFqName, fileId)
}
fun removeLookupsFrom(file: File) {
val id = fileToId[file] ?: return
idToFile.remove(id)
fileToId.remove(file)
deletedCount++
}
override fun clean() {
lookupMap.clean()
}
if (countersFile.exists()) {
countersFile.delete()
}
override fun close() {
lookupMap.close()
size = 0
deletedCount = 0
super.clean()
}
override fun flush(memoryCachesOnly: Boolean) {
lookupMap.flush(memoryCachesOnly)
try {
removeGarbageIfNeeded()
countersFile.writeText("$size\n$deletedCount")
}
finally {
super.flush(memoryCachesOnly)
}
}
fun removeLookupsFrom(path: String) {
filesMap.remove(path)
private fun addFile(file: File): Int {
val id = size++
fileToId[file] = id
idToFile[id] = file
return id
}
private fun removeGarbageIfNeeded() {
if (size <= MINIMUM_GARBAGE_COLLECTIBLE_SIZE && deletedCount.toDouble() / size <= DELETED_TO_SIZE_TRESHOLD) return
for (hash in lookupMap.lookupHashes) {
lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet()
}
size = 0
deletedCount = 0
idToFile.clean()
fileToId.files.forEach { addFile(it) }
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FileToIdMap.kt
+38
View File
@@ -0,0 +1,38 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.incremental.storage
import java.io.File
class FileToIdMap(file: File) : BasicMap<File, Int>(file, FILE_KEY_DESCRIPTOR, INT_EXTERNALIZER) {
override fun dumpKey(key: File): String = key.toString()
override fun dumpValue(value: Int): String = value.toString()
public operator fun get(file: File): Int? = storage[file]
public operator fun set(file: File, id: Int) {
storage[file] = id
}
public fun remove(file: File) {
storage.remove(file)
}
public val files: Collection<File>
get() = storage.keys
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FilesMap.kt
-48
View File
@@ -1,48 +0,0 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.incremental.storage
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.io.ExternalIntegerKeyDescriptor
import java.io.File
class FilesMap(file: File) : BasicMap<Int, Collection<String>>(file, ExternalIntegerKeyDescriptor(), PATH_COLLECTION_EXTERNALIZER) {
override fun dumpKey(key: Int): String = key.toString()
override fun dumpValue(value: Collection<String>): String = value.toString()
public fun get(hash: Int): Collection<String>? = storage[hash]
public fun add(path: String): Int {
val hash = FileUtil.PATH_HASHING_STRATEGY.computeHashCode(path)
storage.append(hash) { it.writeUTF(path) }
return hash
}
public fun remove(path: String) {
val hash = FileUtil.PATH_HASHING_STRATEGY.computeHashCode(path)
val collection = storage[hash] as? MutableCollection<String> ?: return
collection.remove(path)
if (collection.isNotEmpty()) {
storage.remove(hash)
}
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/IdToFileMap.kt
+38
View File
@@ -0,0 +1,38 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.incremental.storage
import com.intellij.util.io.ExternalIntegerKeyDescriptor
import java.io.File
class IdToFileMap(file: File) : BasicMap<Int, File>(file, ExternalIntegerKeyDescriptor(), FILE_EXTERNALIZER) {
override fun dumpKey(key: Int): String = key.toString()
override fun dumpValue(value: File): String = value.toString()
public operator fun get(id: Int): File? = storage[id]
public operator fun contains(id: Int): Boolean = id in storage
public operator fun set(id: Int, file: File) {
storage[id] = file
}
public fun remove(id: Int) {
storage.remove(id)
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LookupMap.kt
+13 -11
View File
@@ -18,21 +18,23 @@ package org.jetbrains.kotlin.jps.incremental.storage
import java.io.File
class LookupMap(
storage: File,
private val filesMap: FilesMap
) : BasicMap<IntPair, Set<Int>>(storage, INT_PAIR_KEY_DESCRIPTOR, INT_SET_EXTERNALIZER) {
class LookupMap(storage: File) : BasicMap<IntPair, Set<Int>>(storage, INT_PAIR_KEY_DESCRIPTOR, INT_SET_EXTERNALIZER) {
override fun dumpKey(key: IntPair): String = key.toString()
override fun dumpValue(value: Set<Int>): String = value.toString()
public fun add(name: String, scope: String, path: String) {
val pathHash = filesMap.add(path)
storage.append(HashPair(name, scope)) { out -> out.writeInt(pathHash) }
public fun add(name: String, scope: String, fileId: Int) {
storage.append(HashPair(name, scope)) { out -> out.writeInt(fileId) }
}
public fun get(name: String, scope: String): Set<Int>? = storage[HashPair(name, scope)]
public operator fun get(name: String, scope: String): Set<Int>? = storage[HashPair(name, scope)]
public operator fun get(lookupHash: IntPair): Set<Int>? = storage[lookupHash]
public operator fun set(key: IntPair, fileIds: Set<Int>) {
storage.set(key, fileIds)
}
public val lookupHashes: Collection<IntPair>
get() = storage.keys
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/externalizers.kt
+31
View File
@@ -26,6 +26,7 @@ import gnu.trove.decorator.TIntHashSetDecorator
import java.io.DataInput
import java.io.DataInputStream
import java.io.DataOutput
import java.io.File
import java.util.*
object INT_PAIR_KEY_DESCRIPTOR : KeyDescriptor<IntPair> {
@@ -236,4 +237,34 @@ object INT_SET_EXTERNALIZER : DataExternalizer<Set<Int>> {
return TIntHashSetDecorator(result)
}
}
object INT_EXTERNALIZER : DataExternalizer<Int> {
override fun read(`in`: DataInput): Int = `in`.readInt()
override fun save(out: DataOutput, value: Int) {
out.writeInt(value)
}
}
object FILE_EXTERNALIZER : DataExternalizer<File> {
override fun read(`in`: DataInput): File = File(`in`.readUTF())
override fun save(out: DataOutput, value: File) {
out.writeUTF(value.canonicalPath)
}
}
object FILE_KEY_DESCRIPTOR : KeyDescriptor<File> {
override fun read(`in`: DataInput): File = File(`in`.readUTF())
override fun save(out: DataOutput, value: File) {
out.writeUTF(value.canonicalPath)
}
override fun getHashCode(value: File?): Int =
FileUtil.FILE_HASHING_STRATEGY.computeHashCode(value)
override fun isEqual(val1: File?, val2: File?): Boolean =
FileUtil.FILE_HASHING_STRATEGY.equals(val1, val2)
}