IDE perf tests for Kotlin/Native projects

Issue #MMPP-201

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -11,7 +11,6 @@ import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.daemon.impl.IdentifierHighlighterPassFactory
 import com.intellij.codeInsight.lookup.LookupElement
-import com.intellij.ide.highlighter.ModuleFileType
 import com.intellij.ide.startup.impl.StartupManagerImpl
 import com.intellij.openapi.actionSystem.IdeActions
 import com.intellij.openapi.application.ApplicationManager
@@ -20,26 +19,23 @@ import com.intellij.openapi.editor.EditorFactory
 import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsManagerImpl
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.fileEditor.FileEditorManager
-import com.intellij.openapi.fileEditor.impl.FileEditorManagerImpl
 import com.intellij.openapi.module.ModuleManager
-import com.intellij.openapi.module.ModuleTypeId
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.project.ex.ProjectManagerEx
-import com.intellij.openapi.project.impl.ProjectImpl
 import com.intellij.openapi.projectRoots.JavaSdk
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
-import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.startup.StartupManager
 import com.intellij.openapi.vcs.changes.ChangeListManager
 import com.intellij.openapi.vcs.changes.ChangeListManagerImpl
-import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.psi.PsiFile
 import com.intellij.psi.PsiManager
-import com.intellij.testFramework.*
+import com.intellij.testFramework.EditorTestUtil
+import com.intellij.testFramework.RunAll
+import com.intellij.testFramework.TestDataProvider
+import com.intellij.testFramework.UsefulTestCase
 import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl
 import com.intellij.util.ArrayUtilRt
 import com.intellij.util.ThrowableRunnable
@@ -51,7 +47,6 @@ import org.jetbrains.kotlin.idea.framework.KotlinSdkType
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
 import org.jetbrains.kotlin.idea.project.getAndCacheLanguageLevelByDependencies
-import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
 import org.jetbrains.kotlin.idea.test.invalidateLibraryCache
 import org.jetbrains.kotlin.idea.testFramework.*
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
@@ -98,11 +93,15 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         }
     }
 
-    protected fun warmUpProject(stats: Stats) {
-        val project = perfOpenHelloWorld(stats, WARM_UP)
+    protected fun warmUpProject(stats: Stats, vararg filesToHighlight: String, openProject: () -> Project) {
+        assertTrue(filesToHighlight.isNotEmpty())
+
+        val project = openProject()
         try {
-            val perfHighlightFile = perfHighlightFile(project, "src/HelloMain.kt", stats, WARM_UP)
-            assertTrue("kotlin project has been not imported properly", perfHighlightFile.isNotEmpty())
+            filesToHighlight.forEach {
+                val perfHighlightFile = perfHighlightFile(project, it, stats, WARM_UP)
+                assertTrue("kotlin project has been not imported properly", perfHighlightFile.isNotEmpty())
+            }
         } finally {
             closeProject(project)
             myApplication.setDataProvider(null)
@@ -130,22 +129,12 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         return if (lastIndexOf >= 0) fileName.substring(lastIndexOf + 1) else fileName
     }
 
-    protected fun perfOpenKotlinProjectFast(stats: Stats) =
-        perfOpenKotlinProject(stats, fast = true)
-
-    protected fun perfOpenKotlinProject(stats: Stats, fast: Boolean = false) {
-        myProject = innerPerfOpenProject("kotlin", stats = stats, path = "../perfTestProject", note = "", fast = fast)
-    }
-
-    protected fun perfOpenHelloWorld(stats: Stats, note: String = ""): Project =
-        innerPerfOpenProject("helloKotlin", stats, note, path = "idea/testData/perfTest/helloKotlin", simpleModule = true)
-
-    protected fun innerPerfOpenProject(
+    protected fun perfOpenProject(
         name: String,
         stats: Stats,
         note: String,
         path: String,
-        simpleModule: Boolean = false,
+        openAction: ProjectOpenAction,
         fast: Boolean = false
     ): Project {
         val projectPath = File(path).canonicalPath
@@ -154,7 +143,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
 
         val warmUpIterations = if (fast) 0 else 5
         val iterations = if (fast) 1 else 5
-        val projectManagerEx = ProjectManagerEx.getInstanceEx()
 
         var lastProject: Project? = null
         var counter = 0
@@ -166,27 +154,10 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             iterations(iterations)
             checkStability(!fast)
             test {
-                val project = if (!simpleModule) {
-                    val project = loadProjectWithName(name = name, path = path)
-                    assertNotNull("project $name at $path is not loaded", project)
-                    val projectRootManager = ProjectRootManager.getInstance(project!!)
-
-                    runWriteAction {
-                        with(projectRootManager) {
-                            projectSdk = jdk18
-                        }
-                    }
-                    assertTrue("project $name at $path is not opened", projectManagerEx.openProject(project))
-                    project
-                } else {
-                    val project = projectManagerEx.loadAndOpenProject(projectPath)!!
-                    initKotlinProject(project, projectPath, name)
-                    project
-                }
-
-                (project as ProjectImpl).registerComponentImplementation(
-                    FileEditorManager::class.java,
-                    FileEditorManagerImpl::class.java
+                val project = openAction.openProject(
+                    projectPath = path,
+                    projectName = name,
+                    jdk = jdk18
                 )
 
                 dispatchAllInvocationEvents()
@@ -537,27 +508,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         }
     }
 
-    private fun initKotlinProject(
-        project: Project,
-        projectPath: String,
-        name: String
-    ) {
-        val modulePath = "$projectPath/$name${ModuleFileType.DOT_DEFAULT_EXTENSION}"
-        val projectFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(File(projectPath))!!
-        val srcFile = projectFile.findChild("src")!!
-        val module = runWriteAction {
-            val projectRootManager = ProjectRootManager.getInstance(project)
-            with(projectRootManager) {
-                projectSdk = jdk18
-            }
-            val moduleManager = ModuleManager.getInstance(project)
-            val module = moduleManager.newModule(modulePath, ModuleTypeId.JAVA_MODULE)
-            PsiTestUtil.addSourceRoot(module, srcFile)
-            module
-        }
-        ConfigLibraryUtil.configureKotlinRuntimeAndSdk(module, jdk18)
-    }
-
     protected fun perfHighlightFile(name: String, stats: Stats): List<HighlightInfo> =
         perfHighlightFile(project(), name, stats)
 

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt

@@ -0,0 +1,229 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf
+
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.module.Module
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.rootManager
+import com.intellij.openapi.roots.LibraryOrderEntry
+import com.intellij.openapi.roots.OrderRootType
+import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.gradle.KotlinSourceSet.Companion.COMMON_TEST_SOURCE_SET_NAME
+import org.jetbrains.kotlin.ide.konan.NativeLibraryKind
+import org.jetbrains.kotlin.idea.caches.project.isMPPModule
+import org.jetbrains.kotlin.idea.configuration.klib.KotlinNativeLibraryNameUtil.parseIDELibraryName
+import org.jetbrains.kotlin.idea.facet.KotlinFacet
+import org.jetbrains.kotlin.idea.framework.CommonLibraryKind
+import org.jetbrains.kotlin.idea.framework.detectLibraryKind
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
+import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.GRADLE_PROJECT
+import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.util.projectStructure.allModules
+import org.jetbrains.kotlin.library.KOTLIN_STDLIB_NAME
+import org.jetbrains.kotlin.platform.konan.isNative
+import java.io.File
+
+class PerformanceNativeProjectsTest : AbstractPerformanceProjectsTest() {
+
+    companion object {
+        private const val GRADLE_VERSION = "6.0.1"
+        private const val KOTLIN_PLUGIN_VERSION = "1.4.0-dev-3273" // TODO: use 1.4-M1 when it's published
+        
+        private var warmedUp: Boolean = false
+    }
+
+    override fun setUp() {
+        super.setUp()
+
+        // warm up: open simple small project
+        if (!warmedUp) {
+            val projectTemplate = "HelloWorld"
+            val enableCommonizer = true
+
+            val projectName = "${projectName(projectTemplate, enableCommonizer)}-$WARM_UP"
+
+            // don't share this stats instance with another one used in "Hello World" test
+            Stats(projectName).use { stats ->
+                warmUpProject(stats, "src/iosX64Main/kotlin/HelloMain.kt") {
+                    perfOpenTemplateGradleProject(stats, projectTemplate, enableCommonizer, WARM_UP)
+                }
+            }
+
+            warmedUp = true
+        }
+    }
+
+    fun testHelloWorldProjectWithCommonizer() = doTestHighlighting(
+        "HelloWorld",
+        true,
+        "src/iosX64Main/kotlin/HelloMain.kt",
+        "src/iosX64Main/kotlin/HelloMain2.kt",
+        "src/iosX64Main/kotlin/HelloMain3.kt",
+        "src/iosX64Test/kotlin/HelloTest.kt",
+        "src/iosX64Test/kotlin/HelloTest2.kt",
+        "src/iosX64Test/kotlin/HelloTest3.kt"
+    )
+
+    fun testHelloWorldProjectWithoutCommonizer() = doTestHighlighting(
+        "HelloWorld",
+        false,
+        "src/iosX64Main/kotlin/HelloMain.kt",
+        "src/iosX64Main/kotlin/HelloMain2.kt",
+        "src/iosX64Main/kotlin/HelloMain3.kt",
+        "src/iosX64Test/kotlin/HelloTest.kt",
+        "src/iosX64Test/kotlin/HelloTest2.kt",
+        "src/iosX64Test/kotlin/HelloTest3.kt"
+    )
+
+    fun testCvsParserProjectWithCommonizer() = doTestHighlighting(
+        "CsvParser",
+        true,
+        "src/iosX64Main/kotlin/CsvParser.kt",
+        "src/iosX64Main/kotlin/CsvParser2.kt",
+        "src/iosX64Main/kotlin/CsvParser3.kt"
+    )
+
+    fun testCvsParserProjectWithoutCommonizer() = doTestHighlighting(
+        "CsvParser",
+        false,
+        "src/iosX64Main/kotlin/CsvParser.kt",
+        "src/iosX64Main/kotlin/CsvParser2.kt",
+        "src/iosX64Main/kotlin/CsvParser3.kt"
+    )
+
+    fun testUIKitAppProjectWithCommonizer() = doTestHighlighting(
+        "UIKitApp",
+        true,
+        "src/iosX64Main/kotlin/UIKitApp.kt",
+        "src/iosX64Main/kotlin/UIKitApp2.kt",
+        "src/iosX64Main/kotlin/UIKitApp3.kt"
+    )
+
+    fun testUIKitAppProjectWithoutCommonizer() = doTestHighlighting(
+        "UIKitApp",
+        false,
+        "src/iosX64Main/kotlin/UIKitApp.kt",
+        "src/iosX64Main/kotlin/UIKitApp2.kt",
+        "src/iosX64Main/kotlin/UIKitApp3.kt"
+    )
+
+    private fun doTestHighlighting(
+        templateName: String,
+        enableCommonizer: Boolean,
+        vararg filesToHighlight: String
+    ) {
+        assertTrue(filesToHighlight.isNotEmpty())
+
+        val projectName = projectName(templateName, enableCommonizer)
+        tcSuite(projectName) {
+            Stats(projectName).use { stats ->
+                myProject = perfOpenTemplateGradleProject(stats, templateName, enableCommonizer)
+
+                // highlight
+                filesToHighlight.forEach { perfHighlightFile(it, stats) }
+            }
+        }
+    }
+
+    private fun perfOpenTemplateGradleProject(
+        stats: Stats,
+        templateName: String,
+        enableCommonizer: Boolean,
+        note: String = ""
+    ): Project {
+        val templateRoot = File("idea/testData/perfTest/native/").resolve(templateName)
+        val projectRoot = FileUtil.createTempDirectory("project", "", false)
+
+        templateRoot.walkTopDown()
+            .filter { it.isFile }
+            .forEach { source ->
+                val destination = projectRoot.resolve(source.relativeTo(templateRoot))
+
+                val filename = source.name
+                if (filename == "build.gradle.kts" || filename == "gradle-wrapper.properties" || filename == "gradle.properties") {
+                    val text = source.readText()
+                        .replace("{{kotlin_plugin_version}}", KOTLIN_PLUGIN_VERSION)
+                        .replace("{{gradle_version}}", GRADLE_VERSION)
+                        .replace("{{disable_commonizer}}", (!enableCommonizer).toString())
+
+                    destination.parentFile.mkdirs()
+                    destination.writeText(text)
+                } else {
+                    source.copyTo(destination)
+                }
+            }
+
+        val projectName = projectName(templateName, enableCommonizer, separator = '-')
+
+        val project = perfOpenProject(
+            name = projectName,
+            stats = stats,
+            note = note,
+            path = projectRoot.absolutePath,
+            openAction = GRADLE_PROJECT
+        )
+        runProjectSanityChecks(project)
+
+        return project
+    }
+
+    // goal: make sure that the project imported from Gradle is valid
+    private fun runProjectSanityChecks(project: Project) {
+
+        val nativeModules: Map<Module, Set<String>> = runReadAction {
+            project.allModules().mapNotNull { module ->
+                val facetSettings = KotlinFacet.get(module)?.configuration?.settings ?: return@mapNotNull null
+                if (!facetSettings.isMPPModule || !facetSettings.targetPlatform.isNative()) return@mapNotNull null
+
+                // workaround to skip top-level common test module, which in fact does not get any Kotlin/Native KLIB libraries
+                // but accidentally gets Kotlin facet with Native platform
+                if (module.name == "${project.name}.$COMMON_TEST_SOURCE_SET_NAME") return@mapNotNull null
+
+                val nativeLibraries = module.rootManager.orderEntries
+                    .asSequence()
+                    .filterIsInstance<LibraryOrderEntry>()
+                    .mapNotNull { it.library }
+                    .filter { library ->
+                        val libraryKind = detectLibraryKind(library.getFiles(OrderRootType.CLASSES))
+                        libraryKind == NativeLibraryKind
+                                // TODO: remove this check for CommonLibraryKind when detection of K/N KLIBs in
+                                //  org.jetbrains.kotlin.ide.konan.KotlinNativePluginUtilKt.isKonanLibraryRoot
+                                //  is correctly implemented
+                                || libraryKind == CommonLibraryKind
+                    }
+                    .mapNotNull inner@{ library ->
+                        val libraryNameParts = parseIDELibraryName(library.name.orEmpty()) ?: return@inner null
+                        val (_, pureLibraryName, platformPart) = libraryNameParts
+                        pureLibraryName + if (platformPart != null) " [$platformPart]" else ""
+                    }
+                    .toSet()
+
+                module to nativeLibraries
+            }.toMap()
+        }
+
+        assertTrue("No Native modules found in project $project", nativeModules.isNotEmpty())
+
+        nativeModules.forEach { (module, nativeLibraries) ->
+            assertTrue(
+                "$KOTLIN_STDLIB_NAME not found for for Native module $module in project $project",
+                KOTLIN_STDLIB_NAME in nativeLibraries
+            )
+
+            assertTrue(
+                "No Native libraries except for $KOTLIN_STDLIB_NAME for Native module $module in project $project",
+                (nativeLibraries - KOTLIN_STDLIB_NAME).isNotEmpty()
+            )
+
+            logMessage { "$module has ${nativeLibraries.size} native libraries" }
+        }
+    }
+
+    private fun projectName(templateName: String, enableCommonizer: Boolean, separator: Char = ' ') =
+        "$templateName project ${if (enableCommonizer) "with" else "without"} commonizer".replace(' ', separator)
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -14,11 +14,14 @@ import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.TEST_KEY
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
 import org.jetbrains.kotlin.idea.testFramework.Fixture
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile
+import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.EXISTING_IDEA_PROJECT
+import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.SIMPLE_JAVA_MODULE
 import java.util.concurrent.atomic.AtomicLong
 import kotlin.test.assertNotEquals
 
@@ -53,8 +56,7 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         super.setUp()
         // warm up: open simple small project
         if (!warmedUp) {
-            warmUpProject(hwStats)
-
+            warmUpProject(hwStats, "src/HelloMain.kt") { perfOpenHelloWorld(hwStats, WARM_UP) }
             warmedUp = true
         }
     }
@@ -207,6 +209,28 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         }
     }
 
+    private fun perfOpenKotlinProjectFast(stats: Stats) = perfOpenKotlinProject(stats, fast = true)
+
+    private fun perfOpenKotlinProject(stats: Stats, fast: Boolean = false) {
+        myProject = perfOpenProject(
+            name = "kotlin",
+            stats = stats,
+            note = "",
+            path = "../perfTestProject",
+            openAction = EXISTING_IDEA_PROJECT,
+            fast = fast
+        )
+    }
+
+    private fun perfOpenHelloWorld(stats: Stats, note: String = ""): Project =
+        perfOpenProject(
+            name = "helloKotlin",
+            stats = stats,
+            note = note,
+            path = "idea/testData/perfTest/helloKotlin",
+            openAction = SIMPLE_JAVA_MODULE
+        )
+
     private fun perfScriptDependenciesBuildGradleKts(it: Stats) {
         perfScriptDependencies("build.gradle.kts", stats = it)
     }

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/ProjectOpenAction.kt

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.testFramework
+
+import com.intellij.ide.highlighter.ModuleFileType
+import com.intellij.openapi.application.runWriteAction
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.module.ModuleTypeId
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ex.ProjectManagerEx
+import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.openapi.roots.ProjectRootManager
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.testFramework.PsiTestUtil
+import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
+import java.io.File
+import kotlin.test.assertNotNull
+import kotlin.test.assertTrue
+
+enum class ProjectOpenAction {
+    SIMPLE_JAVA_MODULE {
+        override fun openProject(projectPath: String, projectName: String, jdk: Sdk): Project {
+            val project = ProjectManagerEx.getInstanceEx().loadAndOpenProject(projectPath)!!
+
+            val modulePath = "$projectPath/$name${ModuleFileType.DOT_DEFAULT_EXTENSION}"
+            val projectFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(File(projectPath))!!
+            val srcFile = projectFile.findChild("src")!!
+
+            val module = runWriteAction {
+                ProjectRootManager.getInstance(project).projectSdk = jdk
+
+                val module = ModuleManager.getInstance(project).newModule(modulePath, ModuleTypeId.JAVA_MODULE)
+                PsiTestUtil.addSourceRoot(module, srcFile)
+
+                module
+            }
+
+            ConfigLibraryUtil.configureKotlinRuntimeAndSdk(module, jdk)
+
+            return project
+        }
+    },
+
+    EXISTING_IDEA_PROJECT {
+        override fun openProject(projectPath: String, projectName: String, jdk: Sdk): Project {
+            val projectManagerEx = ProjectManagerEx.getInstanceEx()
+
+            val project = loadProjectWithName(projectPath, projectName)
+            assertNotNull(project, "project $projectName at $projectPath is not loaded")
+
+            runWriteAction {
+                ProjectRootManager.getInstance(project).projectSdk = jdk
+            }
+
+            assertTrue(projectManagerEx.openProject(project), "project $projectName at $projectPath is not opened")
+
+            return project
+        }
+    },
+
+    GRADLE_PROJECT {
+        override fun openProject(projectPath: String, projectName: String, jdk: Sdk): Project {
+            val project = ProjectManagerEx.getInstanceEx().loadAndOpenProject(projectPath)!!
+
+            runWriteAction {
+                ProjectRootManager.getInstance(project).projectSdk = jdk
+            }
+
+            refreshGradleProject(projectPath, project)
+
+            return project
+        }
+    };
+
+    abstract fun openProject(projectPath: String, projectName: String, jdk: Sdk): Project
+}