From d2881a7920178310f1148002181f5999285c8696 Mon Sep 17 00:00:00 2001
From: "Aleksei.Cherepanov" <aleksei.cherepanov@jetbrains.com>
Date: Mon, 28 Jun 2021 14:56:10 +0300
Subject: [PATCH] Improve performance of Lookup storage

Reduce size of lookup map after rebuild, reduce waiting time by replacing operations of read+write with append, also split remove garbage process into smaller operations in get

#KT-46804 Fixed
---
 .../kotlin/incremental/LookupStorage.kt       | 68 +++++++++----------
 .../kotlin/incremental/storage/LookupMap.kt   |  4 ++
 2 files changed, 38 insertions(+), 34 deletions(-)

diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
index 6287871aeb5..eb8324c54e1 100644
--- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.incremental
 
+import com.intellij.openapi.diagnostic.Logger
 import com.intellij.util.containers.MultiMap
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.incremental.components.LookupTracker
@@ -28,11 +29,14 @@ import org.jetbrains.kotlin.utils.keysToMap
 import java.io.File
 import java.io.IOException
 import java.util.*
+import kotlin.system.measureTimeMillis
 
 open class LookupStorage(
     targetDataDir: File,
     pathConverter: FileToPathConverter
 ) : BasicMapsOwner(targetDataDir) {
+    val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")
+
     companion object {
         private val DELETED_TO_SIZE_TRESHOLD = 0.5
         private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000
@@ -59,18 +63,30 @@ open class LookupStorage(
         } catch (e: Exception) {
             throw IOException("Could not read $countersFile", e)
         }
-
     }
 
     @Synchronized
     fun get(lookupSymbol: LookupSymbol): Collection<String> {
         val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
         val fileIds = lookupMap[key] ?: return emptySet()
+        val paths = mutableSetOf<String>()
+        val filtered = mutableSetOf<Int>()
+
+        for (fileId in fileIds) {
+            val path = idToFile[fileId]?.path
+
+            if (path != null) {
+                paths.add(path)
+                filtered.add(fileId)
+            }
 
-        return fileIds.mapNotNull {
-            // null means it's outdated
-            idToFile[it]?.path
         }
+
+        if (size > MINIMUM_GARBAGE_COLLECTIBLE_SIZE && filtered.size.toDouble() / fileIds.size.toDouble() < DELETED_TO_SIZE_TRESHOLD) {
+            lookupMap[key] = filtered
+        }
+
+        return paths
     }
 
     @Synchronized
@@ -81,8 +97,8 @@ open class LookupStorage(
             val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
             val paths = lookups[lookupSymbol]
             val fileIds = paths.mapTo(TreeSet()) { pathToId[it]!! }
-            fileIds.addAll(lookupMap[key] ?: emptySet())
-            lookupMap[key] = fileIds
+
+            lookupMap.append(key, fileIds)
         }
     }
 
@@ -121,8 +137,7 @@ open class LookupStorage(
 
                 countersFile.writeText("$size\n$deletedCount")
             }
-        }
-        finally {
+        } finally {
             super.flush(memoryCachesOnly)
         }
     }
@@ -138,41 +153,26 @@ open class LookupStorage(
     }
 
     private fun removeGarbageIfNeeded(force: Boolean = false) {
-        if (force || (size > MINIMUM_GARBAGE_COLLECTIBLE_SIZE && deletedCount.toDouble() / size > DELETED_TO_SIZE_TRESHOLD)) {
+        if (force && (size > MINIMUM_GARBAGE_COLLECTIBLE_SIZE && deletedCount.toDouble() / size > DELETED_TO_SIZE_TRESHOLD)) {
             doRemoveGarbage()
         }
     }
 
     private fun doRemoveGarbage() {
-        for (hash in lookupMap.keys) {
-            lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet()
-        }
-
-        val oldFileToId = fileToId.toMap()
-        val oldIdToNewId = HashMap<Int, Int>(oldFileToId.size)
-        idToFile.clean()
-        fileToId.clean()
-        size = 0
-        deletedCount = 0
-
-        for ((file, oldId) in oldFileToId.entries.sortedBy { it.key.path }) {
-            val newId = addFileIfNeeded(file)
-            oldIdToNewId[oldId] = newId
-        }
-
-        for (lookup in lookupMap.keys) {
-            val fileIds = lookupMap[lookup]!!.mapNotNull { oldIdToNewId[it] }.toSet()
-
-            if (fileIds.isEmpty()) {
-                lookupMap.remove(lookup)
-            }
-            else {
-                lookupMap[lookup] = fileIds
+        val timeInMillis = measureTimeMillis {
+            for (hash in lookupMap.keys) {
+                val dirtyFileIds = lookupMap[hash]!!
+                val filteredFileIds = dirtyFileIds.filter { it in idToFile }.toSet()
+                if (dirtyFileIds != filteredFileIds) lookupMap[hash] = filteredFileIds
             }
+
+            deletedCount = 0
         }
+        LOG.debug(">>Garbage removed in $timeInMillis ms")
     }
 
-    @TestOnly fun forceGC() {
+    @TestOnly
+    fun forceGC() {
         removeGarbageIfNeeded(force = true)
         flush(false)
     }
diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt
index 5b606336506..bf612acb13b 100644
--- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt
+++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt
@@ -27,6 +27,10 @@ class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(stor
         storage.append(LookupSymbolKey(name, scope), listOf(fileId))
     }
 
+    fun append(lookup: LookupSymbolKey, fileIds: Collection<Int>) {
+        storage.append(lookup, fileIds)
+    }
+
     operator fun get(key: LookupSymbolKey): Collection<Int>? = storage[key]
 
     operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) {