Save lookups to incremental cache

Original commit: c9663340c9

jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -22,19 +22,22 @@ import com.intellij.openapi.util.io.FileUtil
 import com.intellij.util.containers.MultiMap
 import gnu.trove.THashSet
 import org.jetbrains.jps.ModuleChunk
-import org.jetbrains.jps.builders.BuildTarget
+import org.jetbrains.jps.builders.*
-import org.jetbrains.jps.builders.DirtyFilesHolder
 import org.jetbrains.jps.builders.impl.BuildTargetRegistryImpl
 import org.jetbrains.jps.builders.impl.TargetOutputIndexImpl
 import org.jetbrains.jps.builders.java.JavaBuilderUtil
 import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
 import org.jetbrains.jps.builders.java.dependencyView.Mappings
+import org.jetbrains.jps.builders.storage.BuildDataPaths
 import org.jetbrains.jps.incremental.*
 import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.*
 import org.jetbrains.jps.incremental.fs.CompilationRound
 import org.jetbrains.jps.incremental.java.JavaBuilder
 import org.jetbrains.jps.incremental.messages.BuildMessage
 import org.jetbrains.jps.incremental.messages.CompilerMessage
+import org.jetbrains.jps.indices.IgnoredFileIndex
+import org.jetbrains.jps.indices.ModuleExcludeIndex
+import org.jetbrains.jps.model.JpsModel
 import org.jetbrains.jps.model.JpsProject
 import org.jetbrains.jps.model.JpsSimpleElement
 import org.jetbrains.jps.model.ex.JpsElementChildRoleBase
@@ -152,12 +155,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
 
         val project = projectDescriptor.project
 
-        val lookupTracker =
+        val lookupTracker = dataManager.getStorage(LOOKUP_TRACKER_TARGET, LOOKUP_TRACKER_STORAGE_PROVIDER)
-                project.container.getChild(LOOKUP_TRACKER)?.let {
-                    assert("true".equals(System.getProperty("kotlin.jps.tests"), ignoreCase = true)) { "LOOKUP_TRACKER allowed only for jps tests" }
-                    it.data
-                } ?: LookupTracker.DO_NOTHING
-
         val incrementalCaches = getIncrementalCaches(chunk, context)
         val environment = createCompileEnvironment(incrementalCaches, lookupTracker, context)
         if (!environment.success()) {

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -731,7 +731,7 @@ private object StringListExternalizer : DataExternalizer<List<String>> {
     }
 }
 
-private object PathCollectionExternalizer : DataExternalizer<Collection<String>> {
+object PathCollectionExternalizer : DataExternalizer<Collection<String>> {
     override fun save(out: DataOutput, value: Collection<String>) {
         for (str in value) {
             IOUtil.writeUTF(out, str)

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerImpl.kt

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental
+
+import com.intellij.util.io.KeyDescriptor
+import org.jetbrains.jps.builders.storage.StorageProvider
+import org.jetbrains.jps.incremental.storage.StorageOwner
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.incremental.components.ScopeKind
+import org.jetbrains.kotlin.jps.incremental.storage.BasicMap
+import java.io.DataInput
+import java.io.DataOutput
+import java.io.File
+
+object LOOKUP_TRACKER_STORAGE_PROVIDER : StorageProvider<LookupTrackerImpl>() {
+    override fun createStorage(targetDataDir: File): LookupTrackerImpl = LookupTrackerImpl(targetDataDir)
+}
+
+class LookupTrackerImpl(targetDataDir: File) : LookupTracker, StorageOwner {
+    private val lookupMap = LookupMap(File(targetDataDir, "lookups.tab"))
+
+    override fun record(lookupContainingFile: String, lookupLine: Int?, lookupColumn: Int?, scopeFqName: String, scopeKind: ScopeKind, name: String) {
+        lookupMap.add(name, scopeFqName, lookupContainingFile)
+    }
+
+    override fun clean() {
+        lookupMap.clean()
+    }
+
+    override fun close() {
+        lookupMap.close()
+    }
+
+    override fun flush(memoryCachesOnly: Boolean) {
+        lookupMap.flush(memoryCachesOnly)
+    }
+}
+
+private class LookupMap(file: File) : BasicMap<IntPair, Collection<String>>(file, INT_PAIR_KEY_DESCRIPTOR, PathCollectionExternalizer) {
+    override fun dumpKey(key: IntPair): String = key.toString()
+
+    override fun dumpValue(value: Collection<String>): String = value.toString()
+
+    public fun add(name: String, scope: String, path: String) {
+        storage.append(HashPair(name, scope)) { out -> out.writeUTF(path) }
+    }
+
+    public fun get(name: String, scope: String): Collection<String>? = storage[HashPair(name, scope)]
+}
+
+private data class IntPair(val first: Int, val second: Int) : Comparable<IntPair> {
+    override fun compareTo(other: IntPair): Int {
+        val firstCmp = first.compareTo(other.first)
+
+        if (firstCmp != 0) return firstCmp
+
+        return second.compareTo(other.second)
+    }
+}
+
+private fun HashPair(a: Any, b: Any): IntPair = IntPair(a.hashCode(), b.hashCode())
+
+private object INT_PAIR_KEY_DESCRIPTOR : KeyDescriptor<IntPair> {
+    override fun read(`in`: DataInput): IntPair {
+        val first = `in`.readInt()
+        val second = `in`.readInt()
+        return IntPair(first, second)
+    }
+
+    override fun save(out: DataOutput, value: IntPair?) {
+        if (value == null) return
+
+        out.writeInt(value.first)
+        out.writeInt(value.second)
+    }
+
+    override fun getHashCode(value: IntPair?): Int = value?.hashCode() ?: 0
+
+    override fun isEqual(val1: IntPair?, val2: IntPair?): Boolean = val1 == val2
+}

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupTrackerTarget.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental
+
+import org.jetbrains.jps.builders.*
+import org.jetbrains.jps.builders.storage.BuildDataPaths
+import org.jetbrains.jps.incremental.CompileContext
+import org.jetbrains.jps.indices.IgnoredFileIndex
+import org.jetbrains.jps.indices.ModuleExcludeIndex
+import org.jetbrains.jps.model.JpsModel
+import java.io.File
+
+private val KOTLIN_LOOKUP_TRACKER = "kotlin-lookup-tracker"
+
+object LOOKUP_TRACKER_TARGET_TYPE : BuildTargetType<LOOKUP_TRACKER_TARGET>(KOTLIN_LOOKUP_TRACKER) {
+    override fun computeAllTargets(model: JpsModel): List<LOOKUP_TRACKER_TARGET> = listOf(LOOKUP_TRACKER_TARGET)
+
+    override fun createLoader(model: JpsModel): BuildTargetLoader<LOOKUP_TRACKER_TARGET> =
+            object : BuildTargetLoader<LOOKUP_TRACKER_TARGET>() {
+                override fun createTarget(targetId: String): LOOKUP_TRACKER_TARGET? = LOOKUP_TRACKER_TARGET
+            }
+}
+
+object LOOKUP_TRACKER_TARGET : BuildTarget<BuildRootDescriptor>(LOOKUP_TRACKER_TARGET_TYPE) {
+    override fun getId(): String? = KOTLIN_LOOKUP_TRACKER
+    override fun getPresentableName(): String = KOTLIN_LOOKUP_TRACKER
+
+    override fun computeRootDescriptors(
+            model: JpsModel?,
+            index: ModuleExcludeIndex?,
+            ignoredFileIndex: IgnoredFileIndex?,
+            dataPaths: BuildDataPaths?
+    ): List<BuildRootDescriptor> = listOf()
+
+    override fun getOutputRoots(context: CompileContext): Collection<File> {
+        val dataManager = context.projectDescriptor.dataManager
+        val storageRoot = dataManager.dataPaths.dataStorageRoot
+        return listOf(File(storageRoot, KOTLIN_LOOKUP_TRACKER))
+    }
+
+    override fun findRootDescriptor(rootId: String?, rootIndex: BuildRootIndex?): BuildRootDescriptor? = null
+
+    override fun computeDependencies(
+            targetRegistry: BuildTargetRegistry?,
+            outputIndex: TargetOutputIndex?
+    ): Collection<BuildTarget<*>>? = listOf()
+}