IDE perf tests for K/N: keep Kotlin & Gradle versions in *.properties

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt

@@ -19,13 +19,17 @@ import org.jetbrains.kotlin.idea.configuration.klib.KotlinNativeLibraryNameUtil.
 import org.jetbrains.kotlin.idea.facet.KotlinFacet
 import org.jetbrains.kotlin.idea.framework.CommonLibraryKind
 import org.jetbrains.kotlin.idea.framework.detectLibraryKind
+import org.jetbrains.kotlin.idea.perf.PerformanceNativeProjectsTest.TestProject.*
+import org.jetbrains.kotlin.idea.perf.PerformanceNativeProjectsTest.TestTarget.IOS
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
 import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.GRADLE_PROJECT
 import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
 import org.jetbrains.kotlin.idea.util.projectStructure.allModules
 import org.jetbrains.kotlin.library.KOTLIN_STDLIB_NAME
 import org.jetbrains.kotlin.platform.konan.isNative
+import org.junit.Assume.assumeTrue
 import org.junit.Ignore
 import java.io.File
 
@@ -33,26 +37,64 @@ import java.io.File
 class PerformanceNativeProjectsTest : AbstractPerformanceProjectsTest() {
 
     companion object {
-        private const val GRADLE_VERSION = "6.0.1"
+        private const val TEST_DATA_PATH = "idea/testData/perfTest/native"
-        private const val KOTLIN_PLUGIN_VERSION = "1.4.0-dev-3273" // TODO: use 1.4-M1 when it's published
 
         private var warmedUp: Boolean = false
     }
 
+    private enum class TestTarget {
+        IOS;
+
+        val alias get() = name.toLowerCase()
+        val enabled get() = true
+    }
+
+    private enum class TestProject(
+        val templateName: String,
+        val filesToHighlight: List<String>
+    ) {
+        HELLO_WORLD(
+            templateName = "HelloWorld",
+            filesToHighlight = listOf(
+                "src/main/HelloMain.kt", "src/main/HelloMain2.kt", "src/main/HelloMain3.kt",
+                "src/test/HelloTest.kt", "src/test/HelloTest2.kt", "src/test/HelloTest3.kt"
+            )
+        ),
+        CSV_PARSER(
+            templateName = "CsvParser",
+            filesToHighlight = listOf("src/main/CsvParser.kt", "src/main/CsvParser2.kt", "src/main/CsvParser3.kt")
+        ),
+        UI_KIT_APP(
+            templateName = "UIKitApp",
+            filesToHighlight = listOf("src/main/UIKitApp.kt", "src/main/UIKitApp2.kt", "src/main/UIKitApp3.kt")
+        );
+
+        init {
+            assertTrue(filesToHighlight.isNotEmpty())
+        }
+    }
+
     override fun setUp() {
         super.setUp()
 
         // warm up: open simple small project
         if (!warmedUp) {
-            val projectTemplate = "HelloWorld"
+            val testProject = HELLO_WORLD
             val enableCommonizer = true
 
-            val projectName = "${projectName(projectTemplate, enableCommonizer)}-$WARM_UP"
+            TestTarget.values().forEach { testTarget ->
+                if (!testTarget.enabled) {
+                    logMessage { "Warm-up for test target $testTarget is disabled" }
+                    return@forEach
+                }
 
-            // don't share this stats instance with another one used in "Hello World" test
+                val projectName = "${projectName(testTarget, testProject, enableCommonizer)} $WARM_UP"
-            Stats(projectName).use { stats ->
+
-                warmUpProject(stats, "src/iosX64Main/kotlin/HelloMain.kt") {
+                // don't share this stats instance with another one used in "Hello World" test
-                    perfOpenTemplateGradleProject(stats, projectTemplate, enableCommonizer, WARM_UP)
+                Stats(projectName).use { stats ->
+                    warmUpProject(stats, testProject.filesToHighlight.first()) {
+                        perfOpenTemplateGradleProject(stats, testTarget, testProject, enableCommonizer, WARM_UP)
+                    }
                 }
             }
 
@@ -60,107 +102,106 @@ class PerformanceNativeProjectsTest : AbstractPerformanceProjectsTest() {
         }
     }
 
-    fun testHelloWorldProjectWithCommonizer() = doTestHighlighting(
+    fun testIosHelloWorldProjectWithCommonizer() = doTestHighlighting(IOS, HELLO_WORLD, enableCommonizer = true)
-        "HelloWorld",
+    fun testIosHelloWorldProjectWithoutCommonizer() = doTestHighlighting(IOS, HELLO_WORLD, enableCommonizer = false)
-        true,
-        "src/iosX64Main/kotlin/HelloMain.kt",
-        "src/iosX64Main/kotlin/HelloMain2.kt",
-        "src/iosX64Main/kotlin/HelloMain3.kt",
-        "src/iosX64Test/kotlin/HelloTest.kt",
-        "src/iosX64Test/kotlin/HelloTest2.kt",
-        "src/iosX64Test/kotlin/HelloTest3.kt"
-    )
 
-    fun testHelloWorldProjectWithoutCommonizer() = doTestHighlighting(
+    fun testIosCvsParserProjectWithCommonizer() = doTestHighlighting(IOS, CSV_PARSER, enableCommonizer = true)
-        "HelloWorld",
+    fun testIosCvsParserProjectWithoutCommonizer() = doTestHighlighting(IOS, CSV_PARSER, enableCommonizer = false)
-        false,
-        "src/iosX64Main/kotlin/HelloMain.kt",
-        "src/iosX64Main/kotlin/HelloMain2.kt",
-        "src/iosX64Main/kotlin/HelloMain3.kt",
-        "src/iosX64Test/kotlin/HelloTest.kt",
-        "src/iosX64Test/kotlin/HelloTest2.kt",
-        "src/iosX64Test/kotlin/HelloTest3.kt"
-    )
 
-    fun testCvsParserProjectWithCommonizer() = doTestHighlighting(
+    fun testIosUIKitAppProjectWithCommonizer() = doTestHighlighting(IOS, UI_KIT_APP, enableCommonizer = true)
-        "CsvParser",
+    fun testIosUIKitAppProjectWithoutCommonizer() = doTestHighlighting(IOS, UI_KIT_APP, enableCommonizer = false)
-        true,
-        "src/iosX64Main/kotlin/CsvParser.kt",
-        "src/iosX64Main/kotlin/CsvParser2.kt",
-        "src/iosX64Main/kotlin/CsvParser3.kt"
-    )
-
-    fun testCvsParserProjectWithoutCommonizer() = doTestHighlighting(
-        "CsvParser",
-        false,
-        "src/iosX64Main/kotlin/CsvParser.kt",
-        "src/iosX64Main/kotlin/CsvParser2.kt",
-        "src/iosX64Main/kotlin/CsvParser3.kt"
-    )
-
-    fun testUIKitAppProjectWithCommonizer() = doTestHighlighting(
-        "UIKitApp",
-        true,
-        "src/iosX64Main/kotlin/UIKitApp.kt",
-        "src/iosX64Main/kotlin/UIKitApp2.kt",
-        "src/iosX64Main/kotlin/UIKitApp3.kt"
-    )
-
-    fun testUIKitAppProjectWithoutCommonizer() = doTestHighlighting(
-        "UIKitApp",
-        false,
-        "src/iosX64Main/kotlin/UIKitApp.kt",
-        "src/iosX64Main/kotlin/UIKitApp2.kt",
-        "src/iosX64Main/kotlin/UIKitApp3.kt"
-    )
 
     private fun doTestHighlighting(
-        templateName: String,
+        testTarget: TestTarget,
-        enableCommonizer: Boolean,
+        testProject: TestProject,
-        vararg filesToHighlight: String
+        enableCommonizer: Boolean
     ) {
-        assertTrue(filesToHighlight.isNotEmpty())
+        assumeTrue(testTarget.enabled)
 
-        val projectName = projectName(templateName, enableCommonizer)
+        val projectName = projectName(testTarget, testProject, enableCommonizer)
         tcSuite(projectName) {
             Stats(projectName).use { stats ->
-                myProject = perfOpenTemplateGradleProject(stats, templateName, enableCommonizer)
+                myProject = perfOpenTemplateGradleProject(stats, testTarget, testProject, enableCommonizer)
 
                 // highlight
-                filesToHighlight.forEach { perfHighlightFile(it, stats) }
+                testProject.filesToHighlight.forEach { perfHighlightFile(it, stats) }
             }
         }
     }
 
     private fun perfOpenTemplateGradleProject(
         stats: Stats,
-        templateName: String,
+        testTarget: TestTarget,
+        testProject: TestProject,
         enableCommonizer: Boolean,
         note: String = ""
     ): Project {
-        val templateRoot = File("idea/testData/perfTest/native/").resolve(templateName)
+        val nativeTestsRoot = File(TEST_DATA_PATH)
+
+        val commonRoot = nativeTestsRoot.resolve("_common")
+        val targetRoot = nativeTestsRoot.resolve("_${testTarget.alias}")
+        val templateRoot = nativeTestsRoot.resolve(testProject.templateName)
+
         val projectRoot = FileUtil.createTempDirectory("project", "", false)
 
-        templateRoot.walkTopDown()
+        commonRoot.walkTopDown()
+            .onEnter { !it.name.startsWith('.') } // exclude any directory starting with dot '.'
             .filter { it.isFile }
-            .forEach { source ->
+            .forEach { sourceFile ->
-                val destination = projectRoot.resolve(source.relativeTo(templateRoot))
+                val destinationFileName = with(sourceFile.name) {
-
+                    when {
-                val filename = source.name
+                        // choose the right variant based on whether commonizer is enabled or not
-                if (filename == "build.gradle.kts" || filename == "gradle-wrapper.properties" || filename == "gradle.properties") {
+                        endsWith(".with-commonizer") -> if (!enableCommonizer) return@forEach else removeSuffix(".with-commonizer")
-                    val text = source.readText()
+                        endsWith(".without-commonizer") -> if (enableCommonizer) return@forEach else removeSuffix(".without-commonizer")
-                        .replace("{{kotlin_plugin_version}}", KOTLIN_PLUGIN_VERSION)
+                        else -> this
-                        .replace("{{gradle_version}}", GRADLE_VERSION)
+                    }
-                        .replace("{{disable_commonizer}}", (!enableCommonizer).toString())
-
-                    destination.parentFile.mkdirs()
-                    destination.writeText(text)
-                } else {
-                    source.copyTo(destination)
                 }
+
+                val destinationFile = projectRoot.resolve(sourceFile.resolveSibling(destinationFileName).relativeTo(commonRoot))
+                sourceFile.copyTo(destinationFile)
             }
 
-        val projectName = projectName(templateName, enableCommonizer, separator = '-')
+        targetRoot.copyRecursively(projectRoot)
+        templateRoot.copyRecursively(projectRoot)
+
+        // merge all files with ".header", ".middle", ".footer" suffixes
+        projectRoot.walkTopDown()
+            .filter { it.isFile && it.name.endsWith(".header") }
+            .forEach { headerFile ->
+                // locate middle and footer files
+                val destinationFileName = headerFile.name.removeSuffix(".header")
+
+                val middleFile = headerFile.resolveSibling("$destinationFileName.middle").also(::assertExists)
+                val footerFile = headerFile.resolveSibling("$destinationFileName.footer").also(::assertExists)
+
+                val destinationFile = headerFile.resolveSibling(destinationFileName).also(::assertDoesntExist)
+                destinationFile.writeText(headerFile.readText() + middleFile.readText() + footerFile.readText())
+
+                headerFile.delete()
+                middleFile.delete()
+                footerFile.delete()
+            }
+
+        // check no unmerged ".middle" and ".footer" files left
+        projectRoot.walkTopDown()
+            .filter { it.isFile && (it.name.endsWith(".middle") || it.name.endsWith(".footer")) }
+            .map { it.relativeTo(projectRoot).path }
+            .toList()
+            .sorted()
+            .takeIf { it.isNotEmpty() }
+            ?.let { unmergedFiles ->
+                fail("Some files have not been merged in project root directory: $projectRoot: ${unmergedFiles.joinToString()}")
+            }
+
+        // check all necessary files are there
+        listOf("build.gradle.kts", "settings.gradle.kts", "gradle.properties")
+            .filter { !projectRoot.resolve(it).exists() }
+            .takeIf { it.isNotEmpty() }
+            ?.let { missedFiles ->
+                fail("Some important files are missed in project root directory $projectRoot: ${missedFiles.joinToString()}")
+            }
+
+        val projectName = projectName(testTarget, testProject, enableCommonizer, fileSystemFriendlyName = true)
 
         val project = perfOpenProject(
             name = projectName,
@@ -222,10 +263,17 @@ class PerformanceNativeProjectsTest : AbstractPerformanceProjectsTest() {
                 (nativeLibraries - KOTLIN_STDLIB_NAME).isNotEmpty()
             )
 
-            logMessage { "$module has ${nativeLibraries.size} native libraries" }
+            logMessage { "Native $module has ${nativeLibraries.size} native libraries" }
         }
     }
 
-    private fun projectName(templateName: String, enableCommonizer: Boolean, separator: Char = ' ') =
+    private fun projectName(
-        "$templateName project ${if (enableCommonizer) "with" else "without"} commonizer".replace(' ', separator)
+        testTarget: TestTarget,
+        testProject: TestProject,
+        enableCommonizer: Boolean,
+        fileSystemFriendlyName: Boolean = false
+    ): String {
+        val name = "${testProject.templateName} ($testTarget) ${if (enableCommonizer) "with" else "without"} commonizer"
+        return if (fileSystemFriendlyName) suggestOsNeutralFileName(name) else name
+    }
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.idea.perf
 import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsToMs
 import org.jetbrains.kotlin.idea.perf.profilers.*
 import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
 import org.jetbrains.kotlin.util.PerformanceCounter
 import java.io.*
 import kotlin.system.measureNanoTime
@@ -40,7 +41,7 @@ class Stats(
 
     private fun statFilePrefix() = if (name.isNotEmpty()) "-${plainname()}" else ""
 
-    private fun plainname() = name.toLowerCase().replace(' ', '-').replace('/', '_')
+    private fun plainname() = suggestOsNeutralFileName(name)
 
     private fun pathToResource(resource: String) = "build/$resource"
 

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/util.kt

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.testFramework
+
+// much stricter version of com.intellij.util.PathUtil.suggestFileName(java.lang.String)
+// returns OS-neutral file name, which remains the same when generated on different OSes
+fun suggestOsNeutralFileName(name: String) = name.asSequence().map { ch ->
+    if (ch < 32.toChar() || ch in "<>:\"|?*.\\/;" || ch.isWhitespace()) '_' else ch
+}.joinToString(separator = "")

idea/testData/perfTest/native/CsvParser/.gitignore

@@ -1,4 +0,0 @@
-build
-.idea
-.gradle
-gradlew*

idea/testData/perfTest/native/CsvParser/build.gradle.kts

@@ -1,15 +0,0 @@
-plugins {
-    kotlin("multiplatform") version "{{kotlin_plugin_version}}"
-}
-
-repositories {
-    mavenCentral()
-    maven("https://dl.bintray.com/kotlin/kotlin-dev")
-    maven("https://dl.bintray.com/kotlin/kotlin-eap")
-}
-
-kotlin {
-    ios() {
-        compilations["main"].enableEndorsedLibs = true
-    }
-}

idea/testData/perfTest/native/CsvParser/build.gradle.kts.footer

@@ -0,0 +1,6 @@
+kotlin.targets.filterIsInstance<org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget>().forEach { target ->
+    target.compilations.forEach { compilation ->
+        // enable Kotlin/Native endorsed libs in every compilation
+        compilation.enableEndorsedLibs = true
+    }
+}

idea/testData/perfTest/native/CsvParser/gradle.properties

@@ -1,2 +0,0 @@
-kotlin.mpp.enableGranularSourceSetsMetadata=true
-kotlin.native.enableDependencyPropagation={{disable_commonizer}}

idea/testData/perfTest/native/CsvParser/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +0,0 @@
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-{{gradle_version}}-bin.zip
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists

idea/testData/perfTest/native/CsvParser/settings.gradle.kts

@@ -1,7 +0,0 @@
-pluginManagement {
-    repositories {
-        gradlePluginPortal()
-        maven("https://dl.bintray.com/kotlin/kotlin-dev")
-        maven("https://dl.bintray.com/kotlin/kotlin-eap")
-    }
-}

idea/testData/perfTest/native/HelloWorld/.gitignore

@@ -1,4 +0,0 @@
-build
-.idea
-.gradle
-gradlew*

idea/testData/perfTest/native/HelloWorld/build.gradle.kts

@@ -1,13 +0,0 @@
-plugins {
-    kotlin("multiplatform") version "{{kotlin_plugin_version}}"
-}
-
-repositories {
-    mavenCentral()
-    maven("https://dl.bintray.com/kotlin/kotlin-dev")
-    maven("https://dl.bintray.com/kotlin/kotlin-eap")
-}
-
-kotlin {
-    ios()
-}

idea/testData/perfTest/native/HelloWorld/build.gradle.kts.footer

@@ -0,0 +1 @@
+// no project-specific settings

idea/testData/perfTest/native/HelloWorld/gradle.properties

@@ -1,2 +0,0 @@
-kotlin.mpp.enableGranularSourceSetsMetadata=true
-kotlin.native.enableDependencyPropagation={{disable_commonizer}}

idea/testData/perfTest/native/HelloWorld/settings.gradle.kts

@@ -1,7 +0,0 @@
-pluginManagement {
-    repositories {
-        gradlePluginPortal()
-        maven("https://dl.bintray.com/kotlin/kotlin-dev")
-        maven("https://dl.bintray.com/kotlin/kotlin-eap")
-    }
-}

idea/testData/perfTest/native/UIKitApp/.gitignore

@@ -1,4 +0,0 @@
-build
-.idea
-.gradle
-gradlew*

idea/testData/perfTest/native/UIKitApp/build.gradle.kts

@@ -1,13 +0,0 @@
-plugins {
-    kotlin("multiplatform") version "{{kotlin_plugin_version}}"
-}
-
-repositories {
-    mavenCentral()
-    maven("https://dl.bintray.com/kotlin/kotlin-dev")
-    maven("https://dl.bintray.com/kotlin/kotlin-eap")
-}
-
-kotlin {
-    ios()
-}

idea/testData/perfTest/native/UIKitApp/build.gradle.kts.footer

@@ -0,0 +1 @@
+// no project-specific settings

idea/testData/perfTest/native/UIKitApp/gradle.properties

@@ -1,2 +0,0 @@
-kotlin.mpp.enableGranularSourceSetsMetadata=true
-kotlin.native.enableDependencyPropagation={{disable_commonizer}}

idea/testData/perfTest/native/UIKitApp/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +0,0 @@
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-{{gradle_version}}-bin.zip
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists

idea/testData/perfTest/native/UIKitApp/settings.gradle.kts

@@ -1,7 +0,0 @@
-pluginManagement {
-    repositories {
-        gradlePluginPortal()
-        maven("https://dl.bintray.com/kotlin/kotlin-dev")
-        maven("https://dl.bintray.com/kotlin/kotlin-eap")
-    }
-}

idea/testData/perfTest/native/_common/build.gradle.kts.header

@@ -0,0 +1,25 @@
+buildscript {
+    repositories {
+        mavenCentral()
+        maven("https://dl.bintray.com/kotlin/kotlin-dev")
+        maven("https://dl.bintray.com/kotlin/kotlin-eap")
+    }
+
+    val kotlin_version: String by rootProject
+    dependencies {
+        classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version")
+    }
+}
+
+plugins {
+    kotlin("multiplatform")
+}
+
+repositories {
+    mavenCentral()
+    maven("https://dl.bintray.com/kotlin/kotlin-dev")
+    maven("https://dl.bintray.com/kotlin/kotlin-eap")
+}
+
+// "kotlin" section goes under this line
+

idea/testData/perfTest/native/_common/gradle.properties.with-commonizer

@@ -0,0 +1,4 @@
+kotlin.mpp.enableGranularSourceSetsMetadata=true
+kotlin.native.enableDependencyPropagation=false
+
+kotlin_version=1.4.0-dev-3273

idea/testData/perfTest/native/_common/gradle.properties.without-commonizer

@@ -0,0 +1,4 @@
+kotlin.mpp.enableGranularSourceSetsMetadata=true
+kotlin.native.enableDependencyPropagation=true
+
+kotlin_version=1.4.0-dev-3273

idea/testData/perfTest/native/_common/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-{{gradle_version}}-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.0.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

idea/testData/perfTest/native/_common/settings.gradle.kts

@@ -0,0 +1,16 @@
+pluginManagement {
+    val kotlin_version: String by settings
+    resolutionStrategy {
+        eachPlugin {
+            if (requested.id.name == "multiplatform") {
+                useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version")
+            }
+        }
+    }
+
+    repositories {
+        gradlePluginPortal()
+        maven("https://dl.bintray.com/kotlin/kotlin-dev")
+        maven("https://dl.bintray.com/kotlin/kotlin-eap")
+    }
+}

idea/testData/perfTest/native/_ios/build.gradle.kts.middle

@@ -0,0 +1,10 @@
+kotlin {
+    ios()
+    iosX64 {
+        compilations["main"].defaultSourceSet.kotlin.srcDir("src/main")
+        compilations["test"].defaultSourceSet.kotlin.srcDir("src/test")
+    }
+}
+
+// project-specific settings go under this line
+