Add incremental compilation tests with lookup cache enabled

compiler/util/src/org/jetbrains/kotlin/config/IncrementalCompilation.java

@@ -39,4 +39,14 @@ public class IncrementalCompilation {
     public static void enableIncrementalCompilation() {
         System.setProperty(INCREMENTAL_COMPILATION_PROPERTY, "true");
     }
+
+    @TestOnly
+    public static void disableExperimental() {
+        System.setProperty(IS_EXPERIMENTAL_PROPERTY, "false");
+    }
+
+    @TestOnly
+    public static void enableExperimental() {
+        System.setProperty(IS_EXPERIMENTAL_PROPERTY, "true");
+    }
 }

generators/src/org/jetbrains/kotlin/generators/tests/GenerateTests.kt

@@ -102,10 +102,7 @@ import org.jetbrains.kotlin.integration.AbstractAntTaskTest
 import org.jetbrains.kotlin.j2k.AbstractJavaToKotlinConverterForWebDemoTest
 import org.jetbrains.kotlin.j2k.AbstractJavaToKotlinConverterMultiFileTest
 import org.jetbrains.kotlin.j2k.AbstractJavaToKotlinConverterSingleFileTest
-import org.jetbrains.kotlin.jps.build.AbstractIncrementalCacheVersionChangedTest
+import org.jetbrains.kotlin.jps.build.*
-import org.jetbrains.kotlin.jps.build.AbstractIncrementalJpsTest
-import org.jetbrains.kotlin.jps.build.AbstractIncrementalLazyCachesTest
-import org.jetbrains.kotlin.jps.build.AbstractLookupTrackerTest
 import org.jetbrains.kotlin.jps.build.android.AbstractAndroidJpsTestCase
 import org.jetbrains.kotlin.jps.incremental.AbstractProtoComparisonTest
 import org.jetbrains.kotlin.js.test.semantics.*
@@ -891,6 +888,23 @@ fun main(args: Array<String>) {
         }
     }
 
+    testGroup("jps-plugin/test", "jps-plugin/testData") {
+        testClass<AbstractExperimentalIncrementalJpsTest>() {
+            model("incremental/multiModule", extension = null, excludeParentDirs = true)
+            model("incremental/pureKotlin", extension = null, excludeParentDirs = true)
+            model("incremental/withJava", extension = null, excludeParentDirs = true)
+            model("incremental/inlineFunCallSite", extension = null, excludeParentDirs = true)
+        }
+
+        testClass<AbstractExperimentalIncrementalLazyCachesTest>() {
+            model("incremental/lazyKotlinCaches", extension = null, excludeParentDirs = true)
+        }
+
+        testClass<AbstractExperimentalIncrementalCacheVersionChangedTest>() {
+            model("incremental/cacheVersionChanged", extension = null, excludeParentDirs = true)
+        }
+    }
+
     testGroup("jps-plugin/test", "jps-plugin/testData") {
         testClass<AbstractProtoComparisonTest>() {
             model("comparison/classSignatureChange", extension = null, excludeParentDirs = true)

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt

@@ -17,11 +17,13 @@
 package org.jetbrains.kotlin.jps.incremental
 
 import com.intellij.util.containers.MultiMap
+import org.jetbrains.annotations.TestOnly
 import org.jetbrains.jps.builders.storage.StorageProvider
 import org.jetbrains.kotlin.incremental.components.LocationInfo
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.components.ScopeKind
 import org.jetbrains.kotlin.jps.incremental.storage.*
+import org.jetbrains.kotlin.utils.Printer
 import java.io.File
 import java.util.*
 
@@ -136,6 +138,36 @@ class LookupStorage(private val targetDataDir: File) : BasicMapsOwner() {
             }
         }
     }
+
+    @TestOnly
+    public fun forceGC() {
+        removeGarbageIfNeeded(force = true)
+        flush(false)
+    }
+
+    @TestOnly
+    public fun dump(lookupSymbols: Set<LookupSymbol>): String {
+        flush(false)
+
+        val sb = StringBuilder()
+        val p = Printer(sb)
+        val lookupsStrings = lookupSymbols.groupBy { LookupHashPair(it.name, it.scope) }
+        val lookups = lookupMap.copyAsMap()
+
+        for ((lookup, fileIds) in lookups.entries.sortedBy { it.key }) {
+            val key = if (lookup in lookupsStrings) {
+                lookupsStrings[lookup]!!.map { "${it.scope}#${it.name}" }.sorted().joinToString(", ")
+            }
+            else {
+                lookup.toString()
+            }
+
+            val value = fileIds.map { idToFile[it]?.absolutePath ?: it.toString() }.sorted().joinToString(", ")
+            p.println("$key -> $value")
+        }
+
+        return sb.toString()
+    }
 }
 
 class LookupTrackerImpl(private val delegate: LookupTracker) : LookupTracker {

jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt

@@ -43,8 +43,12 @@ import org.jetbrains.jps.model.JpsModuleRootModificationUtil
 import org.jetbrains.jps.model.java.JpsJavaDependencyScope
 import org.jetbrains.jps.model.java.JpsJavaExtensionService
 import org.jetbrains.jps.util.JpsPathUtil
+import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.jps.build.classFilesComparison.assertEqualDirectories
+import org.jetbrains.kotlin.jps.incremental.LOOKUP_TRACKER_STORAGE_PROVIDER
+import org.jetbrains.kotlin.jps.incremental.LOOKUP_TRACKER_TARGET
+import org.jetbrains.kotlin.jps.incremental.LookupSymbol
 import org.jetbrains.kotlin.jps.incremental.getKotlinCache
 import org.jetbrains.kotlin.test.KotlinTestUtils
 import org.jetbrains.kotlin.utils.Printer
@@ -75,12 +79,16 @@ public abstract class AbstractIncrementalJpsTest(
         private val COMMANDS_AS_MESSAGE_PART = COMMANDS.joinToString("/") { "\".$it\"" }
     }
 
+    protected open val enableExperimentalIncrementalCompilation = false
+
     protected var testDataDir: File by Delegates.notNull()
 
     protected var workDir: File by Delegates.notNull()
 
     protected var projectDescriptor: ProjectDescriptor by Delegates.notNull()
 
+    protected var lookupsDuringTest: MutableSet<LookupSymbol> by Delegates.notNull()
+
     protected val mapWorkingToOriginalFile: MutableMap<File, File> = hashMapOf()
 
     private fun enableDebugLogging() {
@@ -98,6 +106,11 @@ public abstract class AbstractIncrementalJpsTest(
     override fun setUp() {
         super.setUp()
         System.setProperty("kotlin.jps.tests", "true")
+        lookupsDuringTest = hashSetOf()
+
+        if (enableExperimentalIncrementalCompilation) {
+            IncrementalCompilation.enableExperimental()
+        }
 
         if (DEBUG_LOGGING_ENABLED) {
             enableDebugLogging()
@@ -106,13 +119,18 @@ public abstract class AbstractIncrementalJpsTest(
 
     override fun tearDown() {
         System.clearProperty("kotlin.jps.tests")
+
+        if (enableExperimentalIncrementalCompilation) {
+            IncrementalCompilation.disableExperimental()
+        }
+
         super.tearDown()
     }
 
     protected open val mockConstantSearch: Callbacks.ConstantAffectionResolver?
         get() = null
 
-    protected open fun createLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING
+    private fun createLookupTracker(): TestLookupTracker = TestLookupTracker()
 
     protected open fun checkLookups(@Suppress("UNUSED_PARAMETER") lookupTracker: LookupTracker, compiledFiles: Set<File>) {
     }
@@ -136,6 +154,9 @@ public abstract class AbstractIncrementalJpsTest(
                 checkLookups(lookupTracker, logger.compiledFiles)
             }
 
+            val lookups = lookupTracker.lookups.map { LookupSymbol(it.name, it.scopeFqName) }
+            lookupsDuringTest.addAll(lookups)
+
             if (!buildResult.isSuccessful) {
                 val errorMessages =
                         buildResult
@@ -314,6 +335,7 @@ public abstract class AbstractIncrementalJpsTest(
 
     private fun createMappingsDump(project: ProjectDescriptor) =
             createKotlinIncrementalCacheDump(project) + "\n\n\n" +
+            createLookupCacheDump(project) + "\n\n\n" +
             createCommonMappingsDump(project) + "\n\n\n" +
             createJavaMappingsDump(project)
 
@@ -327,6 +349,21 @@ public abstract class AbstractIncrementalJpsTest(
         }.toString()
     }
 
+    private fun createLookupCacheDump(project: ProjectDescriptor): String {
+        val sb = StringBuilder()
+        val p = Printer(sb)
+        p.println("Begin of Lookup Maps")
+        p.println()
+
+        val lookupStorage = project.dataManager.getStorage(LOOKUP_TRACKER_TARGET, LOOKUP_TRACKER_STORAGE_PROVIDER)
+        lookupStorage.forceGC()
+        p.print(lookupStorage.dump(lookupsDuringTest))
+
+        p.println()
+        p.println("End of Lookup Maps")
+        return sb.toString()
+    }
+
     private fun createCommonMappingsDump(project: ProjectDescriptor): String {
         val resultBuf = StringBuilder()
         val result = Printer(resultBuf)

jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt

@@ -35,8 +35,6 @@ abstract class AbstractLookupTrackerTest : AbstractIncrementalJpsTest(
     // ignore KDoc like comments which starts with `/**`, example: /** text */
     val COMMENT_WITH_LOOKUP_INFO = "/\\*[^*]+\\*/".toRegex()
 
-    override fun createLookupTracker(): LookupTracker = TestLookupTracker()
-
     override fun checkLookups(lookupTracker: LookupTracker, compiledFiles: Set<File>) {
         if (lookupTracker !is TestLookupTracker) throw AssertionError("Expected TestLookupTracker, but: ${lookupTracker.javaClass}")
 
@@ -119,22 +117,22 @@ abstract class AbstractLookupTrackerTest : AbstractIncrementalJpsTest(
     }
 }
 
-private data class LookupInfo(
+class TestLookupTracker : LookupTracker {
-        val lookupContainingFile: String,
-        val lookupLine: Int,
-        val lookupColumn: Int,
-        val scopeFqName: String,
-        val scopeKind: ScopeKind,
-        val name: String
-)
-
-private class TestLookupTracker : LookupTracker {
     val lookups = arrayListOf<LookupInfo>()
 
     override fun record(locationInfo: LocationInfo, scopeFqName: String, scopeKind: ScopeKind, name: String) {
         val (line, column) = locationInfo.position
         lookups.add(LookupInfo(locationInfo.filePath, line, column, scopeFqName, scopeKind, name))
     }
+
+    data class LookupInfo(
+            val lookupContainingFile: String,
+            val lookupLine: Int,
+            val lookupColumn: Int,
+            val scopeFqName: String,
+            val scopeKind: ScopeKind,
+            val name: String
+    )
 }
 
 

jps-plugin/test/org/jetbrains/kotlin/jps/build/experimentalIncrementalCompilationTests.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.build
+
+abstract class AbstractExperimentalIncrementalJpsTest : AbstractIncrementalJpsTest() {
+    override val enableExperimentalIncrementalCompilation = true
+}
+
+abstract class AbstractExperimentalIncrementalLazyCachesTest : AbstractIncrementalLazyCachesTest() {
+    override val enableExperimentalIncrementalCompilation = true
+}
+
+abstract class AbstractExperimentalIncrementalCacheVersionChangedTest : AbstractIncrementalCacheVersionChangedTest() {
+    override val enableExperimentalIncrementalCompilation = true
+}