Hash files without collisions

Original commit: 7ebc58c690

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -503,7 +503,7 @@ public class IncrementalCacheImpl(
 
         private fun remove(path: String) {
             storage.remove(path)
-            lookupTrackerImpl?.removeLookupsFrom(path)
+            lookupTrackerImpl?.removeLookupsFrom(File(path))
         }
     }
 

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt

@@ -20,7 +20,8 @@ import org.jetbrains.jps.builders.storage.StorageProvider
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.components.ScopeKind
 import org.jetbrains.kotlin.jps.incremental.storage.BasicMapsOwner
-import org.jetbrains.kotlin.jps.incremental.storage.FilesMap
+import org.jetbrains.kotlin.jps.incremental.storage.FileToIdMap
+import org.jetbrains.kotlin.jps.incremental.storage.IdToFileMap
 import org.jetbrains.kotlin.jps.incremental.storage.LookupMap
 import java.io.File
 
@@ -29,30 +30,83 @@ object LOOKUP_TRACKER_STORAGE_PROVIDER : StorageProvider<LookupTrackerImpl>() {
 }
 
 class LookupTrackerImpl(private val targetDataDir: File) : BasicMapsOwner(), LookupTracker {
+
+    companion object {
+        private val DELETED_TO_SIZE_TRESHOLD = 0.5
+        private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000
+    }
+
     private val String.storageFile: File
         get() = File(targetDataDir, this + IncrementalCacheImpl.CACHE_EXTENSION)
 
-    private val filesMap = registerMap(FilesMap("files".storageFile))
+    private val countersFile = "counters".storageFile
-    private val lookupMap = registerMap(LookupMap("lookups".storageFile, filesMap))
+    private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile))
+    private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile))
+    private val lookupMap = registerMap(LookupMap("lookups".storageFile))
+    private var size: Int = 0
+    private var deletedCount: Int = 0
+
+    init {
+        if (countersFile.exists()) {
+            val lines = countersFile.readLines()
+            size = lines[0].toInt()
+            deletedCount = lines[1].toInt()
+        }
+    }
 
     override fun record(lookupContainingFile: String, lookupLine: Int?, lookupColumn: Int?, scopeFqName: String, scopeKind: ScopeKind, name: String) {
-        lookupMap.add(name, scopeFqName, lookupContainingFile)
+        val file = File(lookupContainingFile)
+        val fileId = fileToId[file] ?: addFile(file)
+        lookupMap.add(name, scopeFqName, fileId)
+    }
+
+    fun removeLookupsFrom(file: File) {
+        val id = fileToId[file] ?: return
+        idToFile.remove(id)
+        fileToId.remove(file)
+        deletedCount++
     }
 
     override fun clean() {
-        lookupMap.clean()
+        if (countersFile.exists()) {
-    }
+            countersFile.delete()
+        }
 
-    override fun close() {
+        size = 0
-        lookupMap.close()
+        deletedCount = 0
+
+        super.clean()
     }
 
     override fun flush(memoryCachesOnly: Boolean) {
-        lookupMap.flush(memoryCachesOnly)
+        try {
+            removeGarbageIfNeeded()
+            countersFile.writeText("$size\n$deletedCount")
+        }
+        finally {
+            super.flush(memoryCachesOnly)
+        }
     }
 
-    fun removeLookupsFrom(path: String) {
+    private fun addFile(file: File): Int {
-        filesMap.remove(path)
+        val id = size++
+        fileToId[file] = id
+        idToFile[id] = file
+        return id
+    }
+
+    private fun removeGarbageIfNeeded() {
+        if (size <= MINIMUM_GARBAGE_COLLECTIBLE_SIZE && deletedCount.toDouble() / size <= DELETED_TO_SIZE_TRESHOLD) return
+
+        for (hash in lookupMap.lookupHashes) {
+            lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet()
+        }
+
+        size = 0
+        deletedCount = 0
+        idToFile.clean()
+
+        fileToId.files.forEach { addFile(it) }
     }
 }
 

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FileToIdMap.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental.storage
+
+import java.io.File
+
+class FileToIdMap(file: File) : BasicMap<File, Int>(file, FILE_KEY_DESCRIPTOR, INT_EXTERNALIZER) {
+    override fun dumpKey(key: File): String = key.toString()
+
+    override fun dumpValue(value: Int): String = value.toString()
+
+    public operator fun get(file: File): Int? = storage[file]
+
+    public operator fun set(file: File, id: Int) {
+        storage[file] = id
+    }
+
+    public fun remove(file: File) {
+        storage.remove(file)
+    }
+
+    public val files: Collection<File>
+        get() = storage.keys
+}

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/FilesMap.kt

@@ -1,48 +0,0 @@
-/*
- * Copyright 2010-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.jps.incremental.storage
-
-import com.intellij.openapi.util.io.FileUtil
-import com.intellij.util.io.ExternalIntegerKeyDescriptor
-import java.io.File
-
-class FilesMap(file: File) : BasicMap<Int, Collection<String>>(file, ExternalIntegerKeyDescriptor(), PATH_COLLECTION_EXTERNALIZER) {
-    override fun dumpKey(key: Int): String = key.toString()
-
-    override fun dumpValue(value: Collection<String>): String = value.toString()
-
-    public fun get(hash: Int): Collection<String>? = storage[hash]
-
-    public fun add(path: String): Int {
-        val hash = FileUtil.PATH_HASHING_STRATEGY.computeHashCode(path)
-        storage.append(hash) { it.writeUTF(path) }
-        return hash
-    }
-
-    public fun remove(path: String) {
-        val hash = FileUtil.PATH_HASHING_STRATEGY.computeHashCode(path)
-        val collection = storage[hash] as? MutableCollection<String> ?: return
-        collection.remove(path)
-
-        if (collection.isNotEmpty()) {
-            storage.remove(hash)
-        }
-    }
-}
-
-
-

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/IdToFileMap.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental.storage
+
+import com.intellij.util.io.ExternalIntegerKeyDescriptor
+import java.io.File
+
+class IdToFileMap(file: File) : BasicMap<Int, File>(file, ExternalIntegerKeyDescriptor(), FILE_EXTERNALIZER) {
+    override fun dumpKey(key: Int): String = key.toString()
+
+    override fun dumpValue(value: File): String = value.toString()
+
+    public operator fun get(id: Int): File? = storage[id]
+
+    public operator fun contains(id: Int): Boolean = id in storage
+
+    public operator fun set(id: Int, file: File) {
+        storage[id] = file
+    }
+
+    public fun remove(id: Int) {
+        storage.remove(id)
+    }
+}

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LookupMap.kt

@@ -18,21 +18,23 @@ package org.jetbrains.kotlin.jps.incremental.storage
 
 import java.io.File
 
-class LookupMap(
+class LookupMap(storage: File) : BasicMap<IntPair, Set<Int>>(storage, INT_PAIR_KEY_DESCRIPTOR, INT_SET_EXTERNALIZER) {
-        storage: File,
-        private val filesMap: FilesMap
-) : BasicMap<IntPair, Set<Int>>(storage, INT_PAIR_KEY_DESCRIPTOR, INT_SET_EXTERNALIZER) {
     override fun dumpKey(key: IntPair): String = key.toString()
 
     override fun dumpValue(value: Set<Int>): String = value.toString()
 
-    public fun add(name: String, scope: String, path: String) {
+    public fun add(name: String, scope: String, fileId: Int) {
-        val pathHash = filesMap.add(path)
+        storage.append(HashPair(name, scope)) { out -> out.writeInt(fileId) }
-        storage.append(HashPair(name, scope)) { out -> out.writeInt(pathHash) }
     }
 
-    public fun get(name: String, scope: String): Set<Int>? = storage[HashPair(name, scope)]
+    public operator fun get(name: String, scope: String): Set<Int>? = storage[HashPair(name, scope)]
+
+    public operator fun get(lookupHash: IntPair): Set<Int>? = storage[lookupHash]
+
+    public operator fun set(key: IntPair, fileIds: Set<Int>) {
+        storage.set(key, fileIds)
+    }
+
+    public val lookupHashes: Collection<IntPair>
+        get() = storage.keys
 }
-
-
-

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/externalizers.kt

@@ -26,6 +26,7 @@ import gnu.trove.decorator.TIntHashSetDecorator
 import java.io.DataInput
 import java.io.DataInputStream
 import java.io.DataOutput
+import java.io.File
 import java.util.*
 
 object INT_PAIR_KEY_DESCRIPTOR : KeyDescriptor<IntPair> {
@@ -237,3 +238,33 @@ object INT_SET_EXTERNALIZER : DataExternalizer<Set<Int>> {
         return TIntHashSetDecorator(result)
     }
 }
+
+object INT_EXTERNALIZER : DataExternalizer<Int> {
+    override fun read(`in`: DataInput): Int = `in`.readInt()
+
+    override fun save(out: DataOutput, value: Int) {
+        out.writeInt(value)
+    }
+}
+
+object FILE_EXTERNALIZER : DataExternalizer<File> {
+    override fun read(`in`: DataInput): File = File(`in`.readUTF())
+
+    override fun save(out: DataOutput, value: File) {
+        out.writeUTF(value.canonicalPath)
+    }
+}
+
+object FILE_KEY_DESCRIPTOR : KeyDescriptor<File> {
+    override fun read(`in`: DataInput): File = File(`in`.readUTF())
+
+    override fun save(out: DataOutput, value: File) {
+        out.writeUTF(value.canonicalPath)
+    }
+
+    override fun getHashCode(value: File?): Int =
+            FileUtil.FILE_HASHING_STRATEGY.computeHashCode(value)
+
+    override fun isEqual(val1: File?, val2: File?): Boolean =
+            FileUtil.FILE_HASHING_STRATEGY.equals(val1, val2)
+}